public void put(WebURL url) throws DatabaseException { /* * The key that is used for storing URLs determines the order * they are crawled. Lower key values results in earlier crawling. * Here our keys are 6 bytes. The first byte comes from the URL priority. * The second byte comes from depth of crawl at which this URL is first found. * The rest of the 4 bytes come from the docid of the URL. As a result, * URLs with lower priority numbers will be crawled earlier. If priority * numbers are the same, those found at lower depths will be crawled earlier. * If depth is also equal, those found earlier (therefore, smaller docid) will * be crawled earlier. */ byte[] keyData = new byte[6]; keyData[0] = url.getPriority(); keyData[1] = (url.getDepth() > Byte.MAX_VALUE ? Byte.MAX_VALUE : (byte) url.getDepth()); Util.putIntInByteArray(url.getDocid(), keyData, 2); DatabaseEntry value = new DatabaseEntry(); webURLBinding.objectToEntry(url, value); Transaction txn; if (resumable) { txn = env.beginTransaction(null, null); } else { txn = null; } urlsDB.put(txn, new DatabaseEntry(keyData), value); if (resumable) { if (txn != null) { txn.commit(); } } }
public List<WebURL> get(int max) throws DatabaseException { synchronized (mutex) { int matches = 0; List<WebURL> results = new ArrayList<WebURL>(max); Cursor cursor = null; OperationStatus result; DatabaseEntry key = new DatabaseEntry(); DatabaseEntry value = new DatabaseEntry(); Transaction txn; if (resumable) { txn = env.beginTransaction(null, null); } else { txn = null; } try { cursor = urlsDB.openCursor(txn, null); result = cursor.getFirst(key, value, null); while (matches < max && result == OperationStatus.SUCCESS) { if (value.getData().length > 0) { results.add(webURLBinding.entryToObject(value)); matches++; } result = cursor.getNext(key, value, null); } } catch (DatabaseException e) { if (txn != null) { txn.abort(); txn = null; } throw e; } finally { if (cursor != null) { cursor.close(); } if (txn != null) { txn.commit(); } } return results; } }