private int recursive(
Transaction txn,
final String endProt,
Stack<String> path,
int bestDepth,
final int maxDepth) {
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
StringBinding.stringToEntry(path.lastElement(), key);
if (this.database.get(txn, key, data, null) == OperationStatus.SUCCESS) {
Set<String> partners = this.bindingSet.entryToObject(data);
for (String prot2 : partners) {
int depth = -1;
if (prot2.equals(endProt)) {
depth = path.size() - 1;
} else if (path.size() < maxDepth && !path.contains(prot2)) {
path.push(prot2);
depth = recursive(txn, endProt, path, bestDepth, maxDepth);
path.pop();
}
if (depth != -1 && (bestDepth == -1 || bestDepth > depth)) {
bestDepth = depth;
}
}
}
return bestDepth;
}
/* Insert 100 records begins with the beginKey. */
private void doWork(Environment master, String dbName, int beginKey) throws Exception {
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setAllowCreate(true);
dbConfig.setTransactional(true);
/* Insert/Update the records of the database. */
Database db = master.openDatabase(null, dbName, dbConfig);
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
for (int i = 0; i < 100; i++) {
IntegerBinding.intToEntry(beginKey + i, key);
StringBinding.stringToEntry("herococo", data);
db.put(null, key, data);
}
db.close();
/*
* Do a sync at the end of the stage to make sure master and
* replica have the same data set.
*/
VLSN commitVLSN = RepTestUtils.syncGroupToLastCommit(repEnvInfo, repEnvInfo.length);
RepTestUtils.checkNodeEquality(commitVLSN, false, repEnvInfo);
}
/* Do the real reading work. */
private void doRead(Environment env, DatabaseConfig dbConfig, ArrayList<TestObject> list)
throws Exception {
Database db = env.openDatabase(null, dbName, dbConfig);
Cursor cursor = db.openCursor(null, null);
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
while (OperationStatus.SUCCESS == cursor.getNext(key, data, null)) {
list.add(new TestObject(IntegerBinding.entryToInt(key), StringBinding.entryToString(data)));
}
cursor.close();
db.close();
}
private void load(Transaction txn, LineIterator r) {
long nLines = 0L;
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
Pattern tab = Pattern.compile("[\t ]");
Cursor c = this.database.openCursor(txn, null);
while (r.hasNext()) {
if (nLines++ % 10000 == 0) {
info("Lines : " + nLines);
}
String line = r.next();
if (line.isEmpty() || line.startsWith("#")) continue;
String tokens[] = tab.split(line);
for (int i = 0; i < 2; ++i) {
String a = tokens[i == 0 ? 0 : 1];
String b = tokens[i == 0 ? 1 : 0];
StringBinding.stringToEntry(a, key);
OperationStatus status = c.getSearchKey(key, data, LockMode.DEFAULT);
Set<String> partners;
if (status == OperationStatus.SUCCESS) {
partners = bindingSet.entryToObject(data);
} else {
partners = new HashSet<String>(1);
}
partners.add(b);
bindingSet.objectToEntry(partners, data);
if (status == OperationStatus.SUCCESS) {
status = c.putCurrent(data);
} else {
status = c.put(key, data);
}
if (status != OperationStatus.SUCCESS) {
throw new PicardException("Cannot insert data in bdb " + a + "/" + b);
}
if (a.equals(b)) break; // self-self
}
}
CloserUtil.close(c);
}
public void openEnv() {
try {
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setReadOnly(readOnly);
envConfig.setAllowCreate(!readOnly);
env = new Environment(envHome, envConfig);
if (sleepTime < 0) {
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setAllowCreate(!readOnly);
Database db = env.openDatabase(null, "testDB", dbConfig);
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
for (int i = 1; i <= 50; i++) {
IntegerBinding.intToEntry(i, key);
StringBinding.stringToEntry("herococo", data);
db.put(null, key, data);
}
db.close();
} else {
Thread.sleep(sleepTime);
}
} catch (EnvironmentLockedException e) {
/*
* Exit the process with value 4, the exception is expected in
* this casse, don't dump it out.
*/
System.exit(4);
} catch (Exception e) {
/* Dump unexpected exception, exit process with value 5. */
e.printStackTrace();
System.exit(5);
} finally {
if (env != null) {
env.close();
}
}
}
public synchronized Sequence getSequence(String name) throws DatabaseException {
checkOpen();
if (storeConfig.getReadOnly()) {
throw new IllegalStateException("Store is read-only");
}
Sequence seq = sequenceMap.get(name);
if (seq == null) {
if (sequenceDb == null) {
String dbName = storePrefix + SEQUENCE_DB;
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setTransactional(storeConfig.getTransactional());
dbConfig.setAllowCreate(true);
sequenceDb = env.openDatabase(null, dbName, dbConfig);
}
DatabaseEntry entry = new DatabaseEntry();
StringBinding.stringToEntry(name, entry);
seq = sequenceDb.openSequence(null, entry, getSequenceConfig(name));
sequenceMap.put(name, seq);
}
return seq;
}
private void experienceLogFlushTask(String sleepTime, boolean flushBeforeCrash) throws Throwable {
try {
createRepEnvInfo(sleepTime);
ReplicatedEnvironment master = RepTestUtils.joinGroup(repEnvInfo);
long startTime = System.currentTimeMillis();
StatsConfig stConfig = new StatsConfig();
stConfig.setClear(true);
/* Flush the existed dirty data before we do writes. */
for (int i = 0; i < repEnvInfo.length; i++) {
repEnvInfo[i].getEnv().sync();
repEnvInfo[i].getEnv().getStats(stConfig);
}
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setAllowCreate(true);
dbConfig.setTransactional(true);
Database db = master.openDatabase(null, dbName, dbConfig);
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry data = new DatabaseEntry();
for (int i = 1; i <= 100; i++) {
IntegerBinding.intToEntry(i, key);
StringBinding.stringToEntry(value, data);
db.put(null, key, data);
}
assertTrue(System.currentTimeMillis() - startTime < 15000);
Thread.sleep(15000);
long endTime = System.currentTimeMillis();
for (int i = 0; i < repEnvInfo.length; i++) {
EnvironmentStats envStats = repEnvInfo[i].getEnv().getStats(stConfig);
LogFlusher flusher = repEnvInfo[i].getRepNode().getLogFlusher();
if (flushBeforeCrash) {
/* Make sure the LogFlushTask has been invoked. */
assertTrue(flusher.getFlushTask().scheduledExecutionTime() > startTime);
assertTrue(flusher.getFlushTask().scheduledExecutionTime() < endTime);
/*
* Since the log file size is not so big, we can't assure
* all the data will be written in the same log file, but
* we can sure that a flush does happen.
*/
assertTrue(envStats.getNSequentialWrites() >= 1);
assertTrue(envStats.getNLogFSyncs() == 1);
} else {
/*
* Make sure the LogFlushTask is not invoked after making
* the changes.
*/
assertTrue(flusher.getFlushTask().scheduledExecutionTime() < startTime);
assertTrue(envStats.getNSequentialWrites() == 0);
assertTrue(envStats.getNLogFSyncs() == 0);
}
assertTrue(envStats.getNFSyncs() == 0);
}
File[] envHomes = new File[3];
/* Close the replicas without doing a checkpoint. */
for (int i = 0; i < repEnvInfo.length; i++) {
envHomes[i] = repEnvInfo[i].getEnvHome();
repEnvInfo[i].getRepImpl().abnormalClose();
}
/*
* Open a read only standalone Environment on the replicas to see
* whether the data has been synced to the disk.
*/
EnvironmentConfig newConfig = new EnvironmentConfig();
newConfig.setAllowCreate(false);
newConfig.setReadOnly(true);
newConfig.setTransactional(true);
for (int i = 0; i < repEnvInfo.length; i++) {
Environment env = new Environment(envHomes[i], newConfig);
dbConfig.setAllowCreate(false);
dbConfig.setReadOnly(true);
try {
db = env.openDatabase(null, dbName, dbConfig);
} catch (DatabaseNotFoundException e) {
/*
* If the system crashes before the flush, the database is
* not synced to the disk, so this database can't be found
* at all, it's expected.
*/
assertFalse(flushBeforeCrash);
}
if (flushBeforeCrash) {
assertTrue(db.count() == 100);
for (int index = 1; index <= 100; index++) {
IntegerBinding.intToEntry(index, key);
OperationStatus status = db.get(null, key, data, null);
if (flushBeforeCrash) {
assertTrue(status == OperationStatus.SUCCESS);
assertEquals(value, StringBinding.entryToString(data));
}
}
}
if (flushBeforeCrash) {
db.close();
}
env.close();
}
} catch (Throwable t) {
t.printStackTrace();
throw t;
}
}
@Override
public int doWork(String[] args) {
int maxDepth = 3;
File dbHome = null;
com.github.lindenb.jvarkit.util.cli.GetOpt opt =
new com.github.lindenb.jvarkit.util.cli.GetOpt();
int c;
while ((c = opt.getopt(args, getGetOptDefault() + "D:M:")) != -1) {
switch (c) {
case 'D':
{
dbHome = new File(opt.getOptArg());
break;
}
case 'M':
{
maxDepth = Integer.parseInt(opt.getOptArg());
break;
}
default:
{
switch (handleOtherOptions(c, opt, args)) {
case EXIT_FAILURE:
return -1;
case EXIT_SUCCESS:
return 0;
default:
break;
}
}
}
}
if (dbHome == null) {
error("Undefined DB-Home");
}
environment = null;
EnvironmentConfig envCfg = new EnvironmentConfig();
this.database = null;
Transaction txn = null;
try {
envCfg.setAllowCreate(true);
this.environment = new Environment(dbHome, envCfg);
DatabaseConfig cfg = new DatabaseConfig();
cfg.setAllowCreate(true);
cfg.setTemporary(true);
this.database = environment.openDatabase(txn, "interactions", cfg);
if (opt.getOptInd() == args.length) {
info("reading stdin");
LineIterator r = IOUtils.openStdinForLineIterator();
load(txn, r);
CloserUtil.close(r);
} else {
for (int optind = opt.getOptInd(); optind < args.length; ++optind) {
String filename = args[optind];
info("reading " + filename);
LineIterator r = IOUtils.openURIForLineIterator(filename);
load(txn, r);
CloserUtil.close(r);
}
}
DatabaseEntry key1 = new DatabaseEntry();
DatabaseEntry data1 = new DatabaseEntry();
DatabaseEntry key2 = new DatabaseEntry();
DatabaseEntry data2 = new DatabaseEntry();
Cursor c1 = this.database.openCursor(txn, null);
Cursor c2 = this.database.openCursor(txn, null);
while (c1.getNext(key1, data1, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
String prot1 = StringBinding.entryToString(key1);
boolean first = true;
while ((first
? c2.getFirst(key2, data2, LockMode.DEFAULT)
: c2.getNext(key2, data2, LockMode.DEFAULT))
== OperationStatus.SUCCESS) {
first = false;
String prot2 = StringBinding.entryToString(key2);
if (prot2.compareTo(prot1) <= 0) continue;
Stack<String> path = new Stack<String>();
path.push(prot1);
int depth = recursive(txn, prot2, path, -1, maxDepth);
if (depth != -1) {
System.out.println(prot1 + "\t" + prot2 + "\t" + depth);
} else {
// System.out.println(prot1+"\t"+prot2+"\t"+depth);
}
if (System.out.checkError()) break;
}
}
CloserUtil.close(c2);
CloserUtil.close(c1);
} catch (Exception err) {
error(err);
} finally {
CloserUtil.close(this.database);
CloserUtil.close(this.environment);
}
return 0;
}