public void put(WebURL url) throws DatabaseException {
/*
* The key that is used for storing URLs determines the order
* they are crawled. Lower key values results in earlier crawling.
* Here our keys are 6 bytes. The first byte comes from the URL priority.
* The second byte comes from depth of crawl at which this URL is first found.
* The rest of the 4 bytes come from the docid of the URL. As a result,
* URLs with lower priority numbers will be crawled earlier. If priority
* numbers are the same, those found at lower depths will be crawled earlier.
* If depth is also equal, those found earlier (therefore, smaller docid) will
* be crawled earlier.
*/
byte[] keyData = new byte[6];
keyData[0] = url.getPriority();
keyData[1] = (url.getDepth() > Byte.MAX_VALUE ? Byte.MAX_VALUE : (byte) url.getDepth());
Util.putIntInByteArray(url.getDocid(), keyData, 2);
DatabaseEntry value = new DatabaseEntry();
webURLBinding.objectToEntry(url, value);
Transaction txn;
if (resumable) {
txn = env.beginTransaction(null, null);
} else {
txn = null;
}
urlsDB.put(txn, new DatabaseEntry(keyData), value);
if (resumable) {
if (txn != null) {
txn.commit();
}
}
}
public List<WebURL> get(int max) throws DatabaseException {
synchronized (mutex) {
int matches = 0;
List<WebURL> results = new ArrayList<WebURL>(max);
Cursor cursor = null;
OperationStatus result;
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry value = new DatabaseEntry();
Transaction txn;
if (resumable) {
txn = env.beginTransaction(null, null);
} else {
txn = null;
}
try {
cursor = urlsDB.openCursor(txn, null);
result = cursor.getFirst(key, value, null);
while (matches < max && result == OperationStatus.SUCCESS) {
if (value.getData().length > 0) {
results.add(webURLBinding.entryToObject(value));
matches++;
}
result = cursor.getNext(key, value, null);
}
} catch (DatabaseException e) {
if (txn != null) {
txn.abort();
txn = null;
}
throw e;
} finally {
if (cursor != null) {
cursor.close();
}
if (txn != null) {
txn.commit();
}
}
return results;
}
}
public void delete(int count) throws DatabaseException {
synchronized (mutex) {
int matches = 0;
Cursor cursor = null;
OperationStatus result;
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry value = new DatabaseEntry();
Transaction txn;
if (resumable) {
txn = env.beginTransaction(null, null);
} else {
txn = null;
}
try {
cursor = urlsDB.openCursor(txn, null);
result = cursor.getFirst(key, value, null);
while (matches < count && result == OperationStatus.SUCCESS) {
cursor.delete();
matches++;
result = cursor.getNext(key, value, null);
}
} catch (DatabaseException e) {
if (txn != null) {
txn.abort();
txn = null;
}
throw e;
} finally {
if (cursor != null) {
cursor.close();
}
if (txn != null) {
txn.commit();
}
}
}
}
@Test
public void testOpenReadOnly() throws Throwable {
try {
EnvironmentConfig envConfig = TestUtils.initEnvConfig();
envConfig.setTransactional(true);
envConfig.setAllowCreate(true);
env = create(envHome, envConfig);
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
Transaction txn = env.beginTransaction(null, null);
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setTransactional(true);
dbConfig.setAllowCreate(true);
Database myDb = env.openDatabase(txn, "testDB2", dbConfig);
key.setData(TestUtils.getTestArray(0));
data.setData(TestUtils.getTestArray(0));
try {
myDb.put(txn, key, data);
} catch (DatabaseException DBE) {
fail("unexpected DatabaseException during put");
}
txn.commit();
myDb.close();
dbConfig = new DatabaseConfig();
dbConfig.setTransactional(true);
dbConfig.setReadOnly(true);
txn = env.beginTransaction(null, null);
myDb = env.openDatabase(txn, "testDB2", dbConfig);
assertTrue(myDb.isTransactional());
assertTrue(myDb.getConfig().getTransactional());
key.setData(TestUtils.getTestArray(0));
data.setData(TestUtils.getTestArray(0));
try {
myDb.put(txn, key, data);
fail("expected UnsupportedOperationException " + "because open RDONLY");
} catch (UnsupportedOperationException expected) {
}
key.setData(TestUtils.getTestArray(0));
data.setData(TestUtils.getTestArray(0));
assertEquals(OperationStatus.SUCCESS, myDb.get(txn, key, data, LockMode.DEFAULT));
Cursor cursor = myDb.openCursor(txn, null);
assertEquals(OperationStatus.SUCCESS, cursor.getFirst(key, data, LockMode.DEFAULT));
try {
cursor.delete();
fail("expected Exception from delete on RD_ONLY db");
} catch (UnsupportedOperationException e) {
}
key.setData(TestUtils.getTestArray(1));
data.setData(TestUtils.getTestArray(1));
try {
myDb.put(txn, key, data);
fail("expected UnsupportedOperationException because open RDONLY");
} catch (UnsupportedOperationException expected) {
}
cursor.close();
txn.commit();
myDb.close();
close(env);
} catch (Throwable t) {
t.printStackTrace();
throw t;
}
}
@Test
public void testIsTransactional() throws Throwable {
try {
/* Open environment in transactional mode.*/
EnvironmentConfig envConfig = TestUtils.initEnvConfig();
envConfig.setTransactional(true);
envConfig.setAllowCreate(true);
env = create(envHome, envConfig);
/* Create a db, open transactionally with implied auto-commit. */
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setAllowCreate(true);
dbConfig.setTransactional(true);
Database myDb = env.openDatabase(null, "testDB", dbConfig);
assertTrue(myDb.isTransactional());
assertTrue(myDb.getConfig().getTransactional());
myDb.close();
/* Open an existing db, can open it non-transactionally. */
dbConfig.setTransactional(false);
myDb = env.openDatabase(null, "testDB", null);
assertFalse(myDb.isTransactional());
assertFalse(myDb.getConfig().getTransactional());
myDb.close();
/* Open another db, pass an explicit transaction. */
dbConfig.setTransactional(true);
Transaction txn = env.beginTransaction(null, null);
myDb = env.openDatabase(txn, "testDB2", dbConfig);
assertTrue(myDb.isTransactional());
assertTrue(myDb.getConfig().getTransactional());
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
key.setData(TestUtils.getTestArray(0));
data.setData(TestUtils.getTestArray(0));
try {
myDb.put(null, key, data);
} catch (DatabaseException DBE) {
fail("didn't expect DatabaseException, implied autocommit");
}
key.setData(TestUtils.getTestArray(1));
data.setData(TestUtils.getTestArray(1));
try {
myDb.put(txn, key, data);
} catch (DatabaseException DBE) {
fail("didn't expect DatabaseException with txn passed");
}
try {
myDb.get(txn, key, data, LockMode.DEFAULT);
} catch (DatabaseException DBE) {
fail("didn't expect DatabaseException with txn passed");
}
txn.commit();
try {
myDb.get(null, key, data, LockMode.DEFAULT);
} catch (DatabaseException DBE) {
fail("didn't expect DatabaseException because no txn passed");
}
myDb.close();
close(env);
} catch (Throwable t) {
t.printStackTrace();
throw t;
}
}
/** Test open and close of an environment. */
public void testCacheStats() throws DatabaseException {
EnvironmentConfig envConfig = TestUtils.initEnvConfig();
envConfig.setTransactional(true);
envConfig.setConfigParam(EnvironmentParams.NODE_MAX.getName(), "6");
envConfig.setAllowCreate(true);
env = new Environment(envHome, envConfig);
EnvironmentStats stat = env.getStats(TestUtils.FAST_STATS);
env.close();
env = null;
assertEquals(0, stat.getNCacheMiss());
assertEquals(0, stat.getNNotResident());
// Try to open and close again, now that the environment exists
envConfig.setAllowCreate(false);
envConfig.setConfigParam(EnvironmentParams.JE_LOGGING_LEVEL.getName(), "CONFIG");
env = new Environment(envHome, envConfig);
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setTransactional(true);
dbConfig.setAllowCreate(true);
Database db = env.openDatabase(null, "foo", dbConfig);
db.put(null, new DatabaseEntry(new byte[0]), new DatabaseEntry(new byte[0]));
Transaction txn = env.beginTransaction(null, null);
db.put(txn, new DatabaseEntry(new byte[0]), new DatabaseEntry(new byte[0]));
stat = env.getStats(TestUtils.FAST_STATS);
MemoryBudget mb = DbInternal.envGetEnvironmentImpl(env).getMemoryBudget();
assertEquals(mb.getCacheMemoryUsage(), stat.getCacheTotalBytes());
assertEquals(mb.getLogBufferBudget(), stat.getBufferBytes());
assertEquals(mb.getTreeMemoryUsage() + mb.getTreeAdminMemoryUsage(), stat.getDataBytes());
assertEquals(mb.getLockMemoryUsage(), stat.getLockBytes());
assertEquals(mb.getAdminMemoryUsage(), stat.getAdminBytes());
assertTrue(stat.getBufferBytes() > 0);
assertTrue(stat.getDataBytes() > 0);
assertTrue(stat.getLockBytes() > 0);
assertTrue(stat.getAdminBytes() > 0);
assertEquals(
stat.getCacheTotalBytes(),
stat.getBufferBytes() + stat.getDataBytes() + stat.getLockBytes() + stat.getAdminBytes());
assertEquals(12, stat.getNCacheMiss());
assertEquals(12, stat.getNNotResident());
/* Test deprecated getCacheDataBytes method. */
final EnvironmentStats finalStat = stat;
final long expectCacheDataBytes = mb.getCacheMemoryUsage() - mb.getLogBufferBudget();
(new Runnable() {
@Deprecated
public void run() {
assertEquals(expectCacheDataBytes, finalStat.getCacheDataBytes());
}
})
.run();
txn.abort();
db.close();
env.close();
env = null;
}
