/** Check that primary put() writes to each secondary that is open. */
private void checkSecondaryUpdate(
Transaction txn,
Database priDb,
int val,
Database secDb,
boolean expectSecDbVal,
Database secDb2,
boolean expectSecDb2Val)
throws DatabaseException {
OperationStatus status;
DatabaseEntry data = new DatabaseEntry();
int secVal = KEY_OFFSET + val;
status = priDb.put(txn, entry(val), entry(val));
assertSame(OperationStatus.SUCCESS, status);
status = secDb.get(txn, entry(secVal), data, LockMode.DEFAULT);
assertSame(expectSecDbVal ? OperationStatus.SUCCESS : OperationStatus.NOTFOUND, status);
status = secDb2.get(txn, entry(secVal), data, LockMode.DEFAULT);
assertSame(expectSecDb2Val ? OperationStatus.SUCCESS : OperationStatus.NOTFOUND, status);
status = priDb.delete(txn, entry(val));
assertSame(OperationStatus.SUCCESS, status);
}
public void merge() throws Exception {
LOG.info("start merge");
Database crawldbDatabase = env.openDatabase(null, "crawldb", BerkeleyDBUtils.defaultDBConfig);
/*合并fetch库*/
LOG.info("merge fetch database");
Database fetchDatabase = env.openDatabase(null, "fetch", BerkeleyDBUtils.defaultDBConfig);
Cursor fetchCursor = fetchDatabase.openCursor(null, null);
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry value = new DatabaseEntry();
while (fetchCursor.getNext(key, value, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
crawldbDatabase.put(null, key, value);
}
fetchCursor.close();
fetchDatabase.close();
/*合并link库*/
LOG.info("merge link database");
Database linkDatabase = env.openDatabase(null, "link", BerkeleyDBUtils.defaultDBConfig);
Cursor linkCursor = linkDatabase.openCursor(null, null);
while (linkCursor.getNext(key, value, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
if (!(crawldbDatabase.get(null, key, value, LockMode.DEFAULT) == OperationStatus.SUCCESS)) {
crawldbDatabase.put(null, key, value);
}
}
linkCursor.close();
linkDatabase.close();
LOG.info("end merge");
crawldbDatabase.sync();
crawldbDatabase.close();
env.removeDatabase(null, "fetch");
LOG.debug("remove fetch database");
env.removeDatabase(null, "link");
LOG.debug("remove link database");
}
@Override
public long[] findRela(long gid) {
DatabaseEntry key = longToEntry(gid);
DatabaseEntry res = new DatabaseEntry();
RelationBDB.get(null, key, res, null);
return entryToAddr(res);
}
public boolean contains(Transaction txn, K key, LockMode lockMode) throws DatabaseException {
DatabaseEntry keyEntry = new DatabaseEntry();
DatabaseEntry dataEntry = NO_RETURN_ENTRY;
keyBinding.objectToEntry(key, keyEntry);
OperationStatus status = db.get(txn, keyEntry, dataEntry, lockMode);
return (status == OperationStatus.SUCCESS);
}
@Override
public byte[] get(byte[] key) {
final DatabaseEntry dv = new DatabaseEntry();
byte[] val;
if (bdb.get(null, new DatabaseEntry(key), dv, lockMode) != opSuccess) {
return null;
}
val = dv.getData();
bytesIn.addAndGet(key.length + val.length);
gets.incrementAndGet();
return val;
}
@Override
public synchronized int getBallot(Long instance) {
keyBinding.objectToEntry(instance, key);
Integer ballot = null;
OperationStatus status = db.get(null, key, ballot_data, LockMode.DEFAULT);
if (status == OperationStatus.SUCCESS) {
ballot = ballotBinding.entryToObject(ballot_data);
}
if (logger.isDebugEnabled()) {
logger.debug("DB get ballot " + ballot + " for instance " + instance + " " + status.name());
}
return ballot.intValue();
}
@Override
public synchronized Decision getDecision(Long instance) {
keyBinding.objectToEntry(instance, key);
Decision decision = null;
OperationStatus status = db.get(null, key, data, LockMode.DEFAULT);
if (status == OperationStatus.SUCCESS) {
decision = dataBinding.entryToObject(data);
}
if (logger.isDebugEnabled()) {
logger.debug("DB get " + decision + " " + status.name());
}
return decision;
}
@Override
public synchronized boolean containsDecision(Long instance) {
boolean found = false;
keyBinding.objectToEntry(instance, key);
OperationStatus status = db.get(null, key, data, LockMode.DEFAULT);
if (status == OperationStatus.SUCCESS) {
found = true;
}
if (logger.isDebugEnabled()) {
logger.debug("DB contains " + instance + " " + found + " (" + status.name() + ")");
}
return found;
}
public static void main(String[] args) {
Environment env = null;
Database db = null;
EnvironmentConfig envconfig = new EnvironmentConfig();
envconfig.setAllowCreate(true);
try {
env = new Environment(new File("D://bdb"), envconfig);
DatabaseConfig dbconfig = new DatabaseConfig();
dbconfig.setAllowCreate(true);
db = env.openDatabase(null, "dbac.db", dbconfig);
String key = "mykey";
DatabaseEntry thekey = new DatabaseEntry();
thekey.setData(key.getBytes("utf-8"));
Long value = new Long(123456);
DatabaseEntry thevalue = new DatabaseEntry();
EntryBinding myBinging = TupleBinding.getPrimitiveBinding(Long.class);
myBinging.objectToEntry(value, thevalue);
// LongBinding myLongBinging=(LongBinding)TupleBinding.getPrimitiveBinding(Long.class);
// myLongBinging.objectToEntry(value, thevalue);
db.put(null, thekey, thevalue);
DatabaseEntry valueEntry = new DatabaseEntry();
OperationStatus status = db.get(null, thekey, valueEntry, LockMode.DEFAULT);
if (status == OperationStatus.SUCCESS) {
// Long number=myLongBinging.entryToObject(valueEntry);
Long number = (Long) myBinging.entryToObject(valueEntry);
System.out.println(env.getDatabaseNames());
System.out.println(number);
}
} catch (EnvironmentLockedException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
} finally {
if (db != null) {
try {
db.close();
} catch (DatabaseException e) {
e.printStackTrace();
}
}
if (env != null) {
try {
env.cleanLog();
env.close();
} catch (DatabaseException e) {
e.printStackTrace();
}
}
}
}
/** @see org.geogit.storage.ObjectDatabase#exists(org.geogit.api.ObjectId) */
@Override
public boolean exists(final ObjectId id) {
Preconditions.checkNotNull(id, "id");
DatabaseEntry key = new DatabaseEntry(id.getRawValue());
DatabaseEntry data = new DatabaseEntry();
// tell db not to retrieve data
data.setPartial(0, 0, true);
final LockMode lockMode = LockMode.DEFAULT;
CurrentTransaction.getInstance(env);
OperationStatus status = objectDb.get(txn.getTransaction(), key, data, lockMode);
return SUCCESS == status;
}
public boolean isLocked() throws Exception {
boolean isLocked = false;
lockDatabase = env.openDatabase(null, "lock", BerkeleyDBUtils.defaultDBConfig);
DatabaseEntry key = new DatabaseEntry("lock".getBytes("utf-8"));
DatabaseEntry value = new DatabaseEntry();
if (lockDatabase.get(null, key, value, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
String lockInfo = new String(value.getData(), "utf-8");
if (lockInfo.equals("locked")) {
isLocked = true;
}
}
lockDatabase.close();
return isLocked;
}
/** @see org.geogit.storage.ObjectDatabase#getRaw(org.geogit.api.ObjectId) */
@Override
protected InputStream getRawInternal(final ObjectId id) throws IOException {
Preconditions.checkNotNull(id, "id");
DatabaseEntry key = new DatabaseEntry(id.getRawValue());
DatabaseEntry data = new DatabaseEntry();
final LockMode lockMode = LockMode.READ_COMMITTED;
Transaction transaction = txn.getTransaction();
OperationStatus operationStatus = objectDb.get(transaction, key, data, lockMode);
if (NOTFOUND.equals(operationStatus)) {
throw new IllegalArgumentException("Object does not exist: " + id.toString());
}
final byte[] cData = data.getData();
return new ByteArrayInputStream(cData);
}
private void readData(Environment env, int lastRecordVal) {
Database readDb = null;
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
try {
readDb = openDatabase(env);
for (int i = 0; i < lastRecordVal; i++) {
LongBinding.longToEntry(i, key);
assertEquals(OperationStatus.SUCCESS, db.get(null, key, data, LockMode.DEFAULT));
}
} finally {
if (readDb != null) {
readDb.close();
}
}
}
/*
* Test that a replica using the jdb files copied from the master can join
* the group.
*/
@Test
public void testCopyEnvJoin() throws Throwable {
createGroup(1);
assertTrue(repEnvInfo[0].isMaster());
/* Create some data on the master. */
populateDB(repEnvInfo[0].getEnv(), "testDB", 1000);
/* Close the Environment before copy. */
repEnvInfo[0].closeEnv();
/* First check there is no jdb files in the second replica. */
File repEnvHome = repEnvInfo[1].getEnvHome();
File[] envFiles = repEnvHome.listFiles();
for (File envFile : envFiles) {
if (envFile.getName().contains(".jdb")) {
throw new IllegalStateException("Replica home should not contain any jdb files");
}
}
/* Copy the jdb files from the master to the replica. */
SharedTestUtils.copyFiles(repEnvInfo[0].getEnvHome(), repEnvInfo[1].getEnvHome());
/* Reopen the master. */
repEnvInfo[0].openEnv();
assertTrue(repEnvInfo[0].isMaster());
/* Open the replica. */
repEnvInfo[1].openEnv();
assertTrue(repEnvInfo[1].isReplica());
/* Read the data to make sure data is correctly copied. */
Database db = repEnvInfo[1].getEnv().openDatabase(null, "testDB", dbconfig);
for (int i = 0; i < 1000; i++) {
IntegerBinding.intToEntry(i, key);
db.get(null, key, data, null);
assertEquals(i, (int) LongBinding.entryToLong(data));
}
db.close();
}
/**
* gets object from databse
*
* @param key
* @return
* @throws Exception
*/
public Object get(Object key) throws Exception {
Serializer serializer = new Serializer();
byte[] keyBytes = serializer.serialize(key);
final DatabaseEntry keyEntry = new DatabaseEntry(keyBytes);
Object result;
final DatabaseEntry dataEntry = new DatabaseEntry();
final Transaction transaction = myEnv.beginTransaction(null, null);
final OperationStatus res = myDatabase.get(transaction, keyEntry, dataEntry, null);
if (res != OperationStatus.SUCCESS) {
// throw new Exception("Error retrieving from database");
return null;
} else {
result = serializer.deserialize(dataEntry.getData());
}
transaction.commit();
return result;
}
/** Test that null can be passed for the LockMode to all get methods. */
public void testNullLockMode() throws DatabaseException {
SecondaryDatabase secDb = initDb();
Database priDb = secDb.getPrimaryDatabase();
Transaction txn = txnBegin();
DatabaseEntry key = entry(0);
DatabaseEntry data = entry(0);
DatabaseEntry secKey = entry(KEY_OFFSET);
DatabaseEntry found = new DatabaseEntry();
DatabaseEntry found2 = new DatabaseEntry();
DatabaseEntry found3 = new DatabaseEntry();
assertEquals(OperationStatus.SUCCESS, priDb.put(txn, key, data));
assertEquals(OperationStatus.SUCCESS, priDb.put(txn, entry(1), data));
assertEquals(OperationStatus.SUCCESS, priDb.put(txn, entry(2), entry(2)));
/* Database operations. */
assertEquals(OperationStatus.SUCCESS, priDb.get(txn, key, found, null));
assertEquals(OperationStatus.SUCCESS, priDb.getSearchBoth(txn, key, data, null));
assertEquals(OperationStatus.SUCCESS, secDb.get(txn, secKey, found, null));
assertEquals(OperationStatus.SUCCESS, secDb.get(txn, secKey, found, found2, null));
assertEquals(OperationStatus.SUCCESS, secDb.getSearchBoth(txn, secKey, key, found, null));
/* Cursor operations. */
txnCommit(txn);
txn = txnBeginCursor();
Cursor cursor = priDb.openCursor(txn, null);
SecondaryCursor secCursor = secDb.openSecondaryCursor(txn, null);
assertEquals(OperationStatus.SUCCESS, cursor.getSearchKey(key, found, null));
assertEquals(OperationStatus.SUCCESS, cursor.getSearchBoth(key, data, null));
assertEquals(OperationStatus.SUCCESS, cursor.getSearchKeyRange(key, found, null));
assertEquals(OperationStatus.SUCCESS, cursor.getSearchBothRange(key, data, null));
assertEquals(OperationStatus.SUCCESS, cursor.getFirst(found, found2, null));
assertEquals(OperationStatus.SUCCESS, cursor.getNext(found, found2, null));
assertEquals(OperationStatus.SUCCESS, cursor.getPrev(found, found2, null));
assertEquals(OperationStatus.NOTFOUND, cursor.getNextDup(found, found2, null));
assertEquals(OperationStatus.NOTFOUND, cursor.getPrevDup(found, found2, null));
assertEquals(OperationStatus.SUCCESS, cursor.getNextNoDup(found, found2, null));
assertEquals(OperationStatus.SUCCESS, cursor.getPrevNoDup(found, found2, null));
assertEquals(OperationStatus.SUCCESS, cursor.getLast(found, found2, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getSearchKey(secKey, found, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getSearchKeyRange(secKey, found, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getFirst(found, found2, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getNext(found, found2, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getPrev(found, found2, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getNextDup(found, found2, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getPrevDup(found, found2, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getNextNoDup(found, found2, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getPrevNoDup(found, found2, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getLast(found, found2, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getSearchKey(secKey, found, found2, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getSearchBoth(secKey, data, found, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getSearchKeyRange(secKey, found, found2, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getSearchBothRange(secKey, data, found, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getFirst(found, found2, found3, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getNext(found, found2, found3, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getPrev(found, found2, found3, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getNextDup(found, found2, found3, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getPrevDup(found, found2, found3, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getNextNoDup(found, found2, found3, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getPrevNoDup(found, found2, found3, null));
assertEquals(OperationStatus.SUCCESS, secCursor.getLast(found, found2, found3, null));
secCursor.close();
cursor.close();
txnCommit(txn);
secDb.close();
priDb.close();
env.close();
env = null;
}
@Override
public boolean hasKey(byte[] key) {
return bdb.get(null, new DatabaseEntry(key), new DatabaseEntry(), lockMode) == opSuccess;
}
@Test
public void testOpenReadOnly() throws Throwable {
try {
EnvironmentConfig envConfig = TestUtils.initEnvConfig();
envConfig.setTransactional(true);
envConfig.setAllowCreate(true);
env = create(envHome, envConfig);
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
Transaction txn = env.beginTransaction(null, null);
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setTransactional(true);
dbConfig.setAllowCreate(true);
Database myDb = env.openDatabase(txn, "testDB2", dbConfig);
key.setData(TestUtils.getTestArray(0));
data.setData(TestUtils.getTestArray(0));
try {
myDb.put(txn, key, data);
} catch (DatabaseException DBE) {
fail("unexpected DatabaseException during put");
}
txn.commit();
myDb.close();
dbConfig = new DatabaseConfig();
dbConfig.setTransactional(true);
dbConfig.setReadOnly(true);
txn = env.beginTransaction(null, null);
myDb = env.openDatabase(txn, "testDB2", dbConfig);
assertTrue(myDb.isTransactional());
assertTrue(myDb.getConfig().getTransactional());
key.setData(TestUtils.getTestArray(0));
data.setData(TestUtils.getTestArray(0));
try {
myDb.put(txn, key, data);
fail("expected UnsupportedOperationException " + "because open RDONLY");
} catch (UnsupportedOperationException expected) {
}
key.setData(TestUtils.getTestArray(0));
data.setData(TestUtils.getTestArray(0));
assertEquals(OperationStatus.SUCCESS, myDb.get(txn, key, data, LockMode.DEFAULT));
Cursor cursor = myDb.openCursor(txn, null);
assertEquals(OperationStatus.SUCCESS, cursor.getFirst(key, data, LockMode.DEFAULT));
try {
cursor.delete();
fail("expected Exception from delete on RD_ONLY db");
} catch (UnsupportedOperationException e) {
}
key.setData(TestUtils.getTestArray(1));
data.setData(TestUtils.getTestArray(1));
try {
myDb.put(txn, key, data);
fail("expected UnsupportedOperationException because open RDONLY");
} catch (UnsupportedOperationException expected) {
}
cursor.close();
txn.commit();
myDb.close();
close(env);
} catch (Throwable t) {
t.printStackTrace();
throw t;
}
}
@Test
public void testIsTransactional() throws Throwable {
try {
/* Open environment in transactional mode.*/
EnvironmentConfig envConfig = TestUtils.initEnvConfig();
envConfig.setTransactional(true);
envConfig.setAllowCreate(true);
env = create(envHome, envConfig);
/* Create a db, open transactionally with implied auto-commit. */
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setAllowCreate(true);
dbConfig.setTransactional(true);
Database myDb = env.openDatabase(null, "testDB", dbConfig);
assertTrue(myDb.isTransactional());
assertTrue(myDb.getConfig().getTransactional());
myDb.close();
/* Open an existing db, can open it non-transactionally. */
dbConfig.setTransactional(false);
myDb = env.openDatabase(null, "testDB", null);
assertFalse(myDb.isTransactional());
assertFalse(myDb.getConfig().getTransactional());
myDb.close();
/* Open another db, pass an explicit transaction. */
dbConfig.setTransactional(true);
Transaction txn = env.beginTransaction(null, null);
myDb = env.openDatabase(txn, "testDB2", dbConfig);
assertTrue(myDb.isTransactional());
assertTrue(myDb.getConfig().getTransactional());
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
key.setData(TestUtils.getTestArray(0));
data.setData(TestUtils.getTestArray(0));
try {
myDb.put(null, key, data);
} catch (DatabaseException DBE) {
fail("didn't expect DatabaseException, implied autocommit");
}
key.setData(TestUtils.getTestArray(1));
data.setData(TestUtils.getTestArray(1));
try {
myDb.put(txn, key, data);
} catch (DatabaseException DBE) {
fail("didn't expect DatabaseException with txn passed");
}
try {
myDb.get(txn, key, data, LockMode.DEFAULT);
} catch (DatabaseException DBE) {
fail("didn't expect DatabaseException with txn passed");
}
txn.commit();
try {
myDb.get(null, key, data, LockMode.DEFAULT);
} catch (DatabaseException DBE) {
fail("didn't expect DatabaseException because no txn passed");
}
myDb.close();
close(env);
} catch (Throwable t) {
t.printStackTrace();
throw t;
}
}
private void experienceLogFlushTask(String sleepTime, boolean flushBeforeCrash) throws Throwable {
try {
createRepEnvInfo(sleepTime);
ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
long startTime = System.currentTimeMillis();
StatsConfig stConfig = new StatsConfig();
stConfig.setClear(true);
/* Flush the existed dirty data before we do writes. */
for (int i = 0; i < repEnvInfo.length; i++) {
repEnvInfo[i].getEnv().sync();
repEnvInfo[i].getEnv().getStats(stConfig);
}
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setAllowCreate(true);
dbConfig.setTransactional(true);
Database db = master.openDatabase(null, dbName, dbConfig);
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
for (int i = 1; i <= 100; i++) {
IntegerBinding.intToEntry(i, key);
StringBinding.stringToEntry(value, data);
db.put(null, key, data);
}
assertTrue(System.currentTimeMillis() - startTime < 15000);
Thread.sleep(15000);
long endTime = System.currentTimeMillis();
for (int i = 0; i < repEnvInfo.length; i++) {
EnvironmentStats envStats = repEnvInfo[i].getEnv().getStats(stConfig);
LogFlusher flusher = repEnvInfo[i].getRepNode().getLogFlusher();
if (flushBeforeCrash) {
/* Make sure the LogFlushTask has been invoked. */
assertTrue(flusher.getFlushTask().scheduledExecutionTime() > startTime);
assertTrue(flusher.getFlushTask().scheduledExecutionTime() < endTime);
/*
* Since the log file size is not so big, we can't assure
* all the data will be written in the same log file, but
* we can sure that a flush does happen.
*/
assertTrue(envStats.getNSequentialWrites() >= 1);
assertTrue(envStats.getNLogFSyncs() == 1);
} else {
/*
* Make sure the LogFlushTask is not invoked after making
* the changes.
*/
assertTrue(flusher.getFlushTask().scheduledExecutionTime() < startTime);
assertTrue(envStats.getNSequentialWrites() == 0);
assertTrue(envStats.getNLogFSyncs() == 0);
}
assertTrue(envStats.getNFSyncs() == 0);
}
File[] envHomes = new File[3];
/* Close the replicas without doing a checkpoint. */
for (int i = 0; i < repEnvInfo.length; i++) {
envHomes[i] = repEnvInfo[i].getEnvHome();
repEnvInfo[i].getRepImpl().abnormalClose();
}
/*
* Open a read only standalone Environment on the replicas to see
* whether the data has been synced to the disk.
*/
EnvironmentConfig newConfig = new EnvironmentConfig();
newConfig.setAllowCreate(false);
newConfig.setReadOnly(true);
newConfig.setTransactional(true);
for (int i = 0; i < repEnvInfo.length; i++) {
Environment env = new Environment(envHomes[i], newConfig);
dbConfig.setAllowCreate(false);
dbConfig.setReadOnly(true);
try {
db = env.openDatabase(null, dbName, dbConfig);
} catch (DatabaseNotFoundException e) {
/*
* If the system crashes before the flush, the database is
* not synced to the disk, so this database can't be found
* at all, it's expected.
*/
assertFalse(flushBeforeCrash);
}
if (flushBeforeCrash) {
assertTrue(db.count() == 100);
for (int index = 1; index <= 100; index++) {
IntegerBinding.intToEntry(index, key);
OperationStatus status = db.get(null, key, data, null);
if (flushBeforeCrash) {
assertTrue(status == OperationStatus.SUCCESS);
assertEquals(value, StringBinding.entryToString(data));
}
}
}
if (flushBeforeCrash) {
db.close();
}
env.close();
}
} catch (Throwable t) {
t.printStackTrace();
throw t;
}
}
/**
* Synchronize publications with pubmed using pmid
*
* @throws Exception if an error occurs
*/
public void execute() throws Exception {
// Needed so that STAX can find it's implementation classes
ClassLoader cl = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
Database db = null;
Transaction txn = null;
try {
if (osAlias == null) {
throw new BuildException("osAlias attribute is not set");
}
if (outputFile == null) {
throw new BuildException("outputFile attribute is not set");
}
// environment is transactional
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setTransactional(true);
envConfig.setAllowCreate(true);
Environment env = new Environment(new File(cacheDirName), envConfig);
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setTransactional(true);
dbConfig.setAllowCreate(true);
dbConfig.setSortedDuplicates(true);
db = env.openDatabase(null, "publications_db", dbConfig);
txn = env.beginTransaction(null, null);
LOG.info("Starting EntrezPublicationsRetriever");
Writer writer = new FileWriter(outputFile);
ObjectStore os = ObjectStoreFactory.getObjectStore(osAlias);
Set<Integer> idsToFetch = new HashSet<Integer>();
itemFactory = new ItemFactory(os.getModel(), "-1_");
writer.write(FullRenderer.getHeader() + ENDL);
for (Iterator<Publication> iter = getPublications(os).iterator(); iter.hasNext(); ) {
String pubMedId = iter.next().getPubMedId();
Integer pubMedIdInteger;
try {
pubMedIdInteger = Integer.valueOf(pubMedId);
} catch (NumberFormatException e) {
// not a pubmed id
continue;
}
if (seenPubMeds.contains(pubMedIdInteger)) {
continue;
}
DatabaseEntry key = new DatabaseEntry(pubMedId.getBytes());
DatabaseEntry data = new DatabaseEntry();
if (db.get(txn, key, data, null).equals(OperationStatus.SUCCESS)) {
try {
ByteArrayInputStream mapInputStream = new ByteArrayInputStream(data.getData());
ObjectInputStream deserializer = new ObjectInputStream(mapInputStream);
Map<String, Object> pubMap = (Map) deserializer.readObject();
writeItems(writer, mapToItems(itemFactory, pubMap));
seenPubMeds.add(pubMedIdInteger);
} catch (EOFException e) {
// ignore and fetch it again
System.err.println(
"found in cache, but igored due to cache problem: " + pubMedIdInteger);
}
} else {
idsToFetch.add(pubMedIdInteger);
}
}
Iterator<Integer> idIter = idsToFetch.iterator();
Set<Integer> thisBatch = new HashSet<Integer>();
while (idIter.hasNext()) {
Integer pubMedIdInteger = idIter.next();
thisBatch.add(pubMedIdInteger);
if (thisBatch.size() == BATCH_SIZE || !idIter.hasNext() && thisBatch.size() > 0) {
try {
// the server may return less publications than we ask for, so keep a Map
Map<String, Map<String, Object>> fromServerMap = null;
for (int i = 0; i < MAX_TRIES; i++) {
BufferedReader br = new BufferedReader(getReader(thisBatch));
StringBuffer buf = new StringBuffer();
String line;
while ((line = br.readLine()) != null) {
buf.append(line + "\n");
}
fromServerMap = new HashMap<String, Map<String, Object>>();
Throwable throwable = null;
try {
if (loadFullRecord) {
SAXParser.parse(
new InputSource(new StringReader(buf.toString())),
new FullRecordHandler(fromServerMap),
false);
} else {
SAXParser.parse(
new InputSource(new StringReader(buf.toString())),
new SummaryRecordHandler(fromServerMap),
false);
}
} catch (Throwable e) {
LOG.error("Couldn't parse PubMed XML", e);
// try again or re-throw the Throwable
throwable = e;
}
if (i == MAX_TRIES) {
throw new RuntimeException(
"failed to parse: " + buf.toString() + " - tried " + MAX_TRIES + " times",
throwable);
} else {
if (throwable != null) {
// try again
continue;
}
}
for (String id : fromServerMap.keySet()) {
writeItems(writer, mapToItems(itemFactory, fromServerMap.get(id)));
}
addToDb(txn, db, fromServerMap);
break;
}
thisBatch.clear();
} finally {
txn.commit();
// start a new transaction incase there is an exception while parsing
txn = env.beginTransaction(null, null);
}
}
}
writeItems(writer, authorMap.values());
writeItems(writer, meshTerms.values());
writer.write(FullRenderer.getFooter() + ENDL);
writer.flush();
writer.close();
} catch (Throwable e) {
throw new RuntimeException("failed to get all publications", e);
} finally {
txn.commit();
db.close();
Thread.currentThread().setContextClassLoader(cl);
}
}
public void testPutAndDelete() throws DatabaseException {
SecondaryDatabase secDb = initDb();
Database priDb = secDb.getPrimaryDatabase();
DatabaseEntry data = new DatabaseEntry();
DatabaseEntry key = new DatabaseEntry();
OperationStatus status;
Transaction txn = txnBegin();
/* Database.put() */
status = priDb.put(txn, entry(1), entry(2));
assertSame(OperationStatus.SUCCESS, status);
status = secDb.get(txn, entry(102), key, data, LockMode.DEFAULT);
assertSame(OperationStatus.SUCCESS, status);
assertDataEquals(entry(1), key);
assertDataEquals(entry(2), data);
/* Database.putNoOverwrite() */
status = priDb.putNoOverwrite(txn, entry(1), entry(1));
assertSame(OperationStatus.KEYEXIST, status);
status = secDb.get(txn, entry(102), key, data, LockMode.DEFAULT);
assertSame(OperationStatus.SUCCESS, status);
assertDataEquals(entry(1), key);
assertDataEquals(entry(2), data);
/* Database.put() overwrite */
status = priDb.put(txn, entry(1), entry(3));
assertSame(OperationStatus.SUCCESS, status);
status = secDb.get(txn, entry(102), key, data, LockMode.DEFAULT);
assertSame(OperationStatus.NOTFOUND, status);
status = secDb.get(txn, entry(103), key, data, LockMode.DEFAULT);
assertSame(OperationStatus.SUCCESS, status);
assertDataEquals(entry(1), key);
assertDataEquals(entry(3), data);
/* Database.delete() */
status = priDb.delete(txn, entry(1));
assertSame(OperationStatus.SUCCESS, status);
status = priDb.delete(txn, entry(1));
assertSame(OperationStatus.NOTFOUND, status);
status = secDb.get(txn, entry(103), key, data, LockMode.DEFAULT);
assertSame(OperationStatus.NOTFOUND, status);
/* SecondaryDatabase.delete() */
status = priDb.put(txn, entry(1), entry(1));
assertSame(OperationStatus.SUCCESS, status);
status = priDb.put(txn, entry(2), entry(1));
assertSame(OperationStatus.SUCCESS, status);
status = secDb.get(txn, entry(101), key, data, LockMode.DEFAULT);
assertSame(OperationStatus.SUCCESS, status);
assertDataEquals(entry(1), key);
assertDataEquals(entry(1), data);
status = secDb.delete(txn, entry(101));
assertSame(OperationStatus.SUCCESS, status);
status = secDb.delete(txn, entry(101));
assertSame(OperationStatus.NOTFOUND, status);
status = secDb.get(txn, entry(101), key, data, LockMode.DEFAULT);
assertSame(OperationStatus.NOTFOUND, status);
status = priDb.get(txn, entry(1), data, LockMode.DEFAULT);
assertSame(OperationStatus.NOTFOUND, status);
status = priDb.get(txn, entry(2), data, LockMode.DEFAULT);
assertSame(OperationStatus.NOTFOUND, status);
/*
* Database.putNoDupData() cannot be called since the primary cannot be
* configured for duplicates.
*/
/* Primary and secondary are empty now. */
/* Get a txn for a cursor. */
txnCommit(txn);
txn = txnBeginCursor();
Cursor priCursor = null;
SecondaryCursor secCursor = null;
try {
priCursor = priDb.openCursor(txn, null);
secCursor = secDb.openSecondaryCursor(txn, null);
/* Cursor.putNoOverwrite() */
status = priCursor.putNoOverwrite(entry(1), entry(2));
assertSame(OperationStatus.SUCCESS, status);
status = secCursor.getSearchKey(entry(102), key, data, LockMode.DEFAULT);
assertSame(OperationStatus.SUCCESS, status);
assertDataEquals(entry(1), key);
assertDataEquals(entry(2), data);
/* Cursor.putCurrent() */
status = priCursor.putCurrent(entry(3));
assertSame(OperationStatus.SUCCESS, status);
status = secCursor.getSearchKey(entry(102), key, data, LockMode.DEFAULT);
assertSame(OperationStatus.NOTFOUND, status);
status = secCursor.getSearchKey(entry(103), key, data, LockMode.DEFAULT);
assertSame(OperationStatus.SUCCESS, status);
assertDataEquals(entry(1), key);
assertDataEquals(entry(3), data);
/* Cursor.delete() */
status = priCursor.delete();
assertSame(OperationStatus.SUCCESS, status);
status = priCursor.delete();
assertSame(OperationStatus.KEYEMPTY, status);
status = secCursor.getSearchKey(entry(103), key, data, LockMode.DEFAULT);
assertSame(OperationStatus.NOTFOUND, status);
status = priCursor.getSearchKey(entry(1), data, LockMode.DEFAULT);
assertSame(OperationStatus.NOTFOUND, status);
/* Cursor.put() */
status = priCursor.put(entry(1), entry(4));
assertSame(OperationStatus.SUCCESS, status);
status = secCursor.getSearchKey(entry(104), key, data, LockMode.DEFAULT);
assertSame(OperationStatus.SUCCESS, status);
assertDataEquals(entry(1), key);
assertDataEquals(entry(4), data);
/* SecondaryCursor.delete() */
status = secCursor.delete();
assertSame(OperationStatus.SUCCESS, status);
status = secCursor.delete();
assertSame(OperationStatus.KEYEMPTY, status);
status = secCursor.getCurrent(new DatabaseEntry(), key, data, LockMode.DEFAULT);
assertSame(OperationStatus.KEYEMPTY, status);
status = secCursor.getSearchKey(entry(104), key, data, LockMode.DEFAULT);
assertSame(OperationStatus.NOTFOUND, status);
status = priCursor.getSearchKey(entry(1), data, LockMode.DEFAULT);
assertSame(OperationStatus.NOTFOUND, status);
/*
* Cursor.putNoDupData() cannot be called since the primary cannot
* be configured for duplicates.
*/
/* Primary and secondary are empty now. */
} finally {
if (secCursor != null) {
secCursor.close();
}
if (priCursor != null) {
priCursor.close();
}
}
txnCommit(txn);
secDb.close();
priDb.close();
}
