public boolean createSecondaryKey(
SecondaryDatabase secondaryDb,
DatabaseEntry keyEntry,
DatabaseEntry dataEntry,
DatabaseEntry resultEntry) {
/*
* Convert the data entry to a MyData object, extract the secondary
* key value from it, and then convert it to the resulting
* secondary key entry.
*/
MyData data = dataBinding.entryToObject(dataEntry);
String key = data.getMessage();
if (key != null) {
secKeyBinding.objectToEntry(key, resultEntry);
return true;
} else {
/*
* The message property of MyData is optional, so if it is null
* then return false to prevent it from being indexed. Note
* that if a required key is missing or an error occurs, an
* exception should be thrown by this method.
*/
return false;
}
}
@Override
public synchronized void putBallot(Long instance, int ballot) {
keyBinding.objectToEntry(instance, key);
ballotBinding.objectToEntry(ballot, ballot_data);
OperationStatus status = db.put(null, key, ballot_data);
if (logger.isDebugEnabled()) {
logger.debug("DB put ballot " + ballot + " for instance " + instance + " " + status.name());
}
}
@Override
public synchronized void putDecision(Long instance, Decision decision) {
keyBinding.objectToEntry(instance, key);
dataBinding.objectToEntry(decision, data);
OperationStatus status = db.put(null, key, data);
if (logger.isDebugEnabled()) {
logger.debug("DB put " + decision + " " + status.name());
}
}
/** Debug method */
public synchronized void listAll() {
Cursor cursor = db.openCursor(null, null);
while (cursor.getNext(key, data, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
Long instance = keyBinding.entryToObject(key);
Decision decision = dataBinding.entryToObject(data);
System.out.println("instance " + instance + " -> " + decision + "");
}
cursor.close();
}
@Override
public synchronized Decision getDecision(Long instance) {
keyBinding.objectToEntry(instance, key);
Decision decision = null;
OperationStatus status = db.get(null, key, data, LockMode.DEFAULT);
if (status == OperationStatus.SUCCESS) {
decision = dataBinding.entryToObject(data);
}
if (logger.isDebugEnabled()) {
logger.debug("DB get " + decision + " " + status.name());
}
return decision;
}
@Override
public synchronized int getBallot(Long instance) {
keyBinding.objectToEntry(instance, key);
Integer ballot = null;
OperationStatus status = db.get(null, key, ballot_data, LockMode.DEFAULT);
if (status == OperationStatus.SUCCESS) {
ballot = ballotBinding.entryToObject(ballot_data);
}
if (logger.isDebugEnabled()) {
logger.debug("DB get ballot " + ballot + " for instance " + instance + " " + status.name());
}
return ballot.intValue();
}
public static void main(String[] args) {
Environment env = null;
Database db = null;
EnvironmentConfig envconfig = new EnvironmentConfig();
envconfig.setAllowCreate(true);
try {
env = new Environment(new File("D://bdb"), envconfig);
DatabaseConfig dbconfig = new DatabaseConfig();
dbconfig.setAllowCreate(true);
db = env.openDatabase(null, "dbac.db", dbconfig);
String key = "mykey";
DatabaseEntry thekey = new DatabaseEntry();
thekey.setData(key.getBytes("utf-8"));
Long value = new Long(123456);
DatabaseEntry thevalue = new DatabaseEntry();
EntryBinding myBinging = TupleBinding.getPrimitiveBinding(Long.class);
myBinging.objectToEntry(value, thevalue);
// LongBinding myLongBinging=(LongBinding)TupleBinding.getPrimitiveBinding(Long.class);
// myLongBinging.objectToEntry(value, thevalue);
db.put(null, thekey, thevalue);
DatabaseEntry valueEntry = new DatabaseEntry();
OperationStatus status = db.get(null, thekey, valueEntry, LockMode.DEFAULT);
if (status == OperationStatus.SUCCESS) {
// Long number=myLongBinging.entryToObject(valueEntry);
Long number = (Long) myBinging.entryToObject(valueEntry);
System.out.println(env.getDatabaseNames());
System.out.println(number);
}
} catch (EnvironmentLockedException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
} finally {
if (db != null) {
try {
db.close();
} catch (DatabaseException e) {
e.printStackTrace();
}
}
if (env != null) {
try {
env.cleanLog();
env.close();
} catch (DatabaseException e) {
e.printStackTrace();
}
}
}
}
/**
* Delete all CrawlURIs matching the given expression.
*
* @param match
* @param queue
* @param headKey
* @return count of deleted items
* @throws DatabaseException
* @throws DatabaseException
*/
public long deleteMatchingFromQueue(String match, String queue, DatabaseEntry headKey)
throws DatabaseException {
long deletedCount = 0;
Pattern pattern = Pattern.compile(match);
DatabaseEntry key = headKey;
DatabaseEntry value = new DatabaseEntry();
Cursor cursor = null;
try {
cursor = pendingUrisDB.openCursor(null, null);
OperationStatus result = cursor.getSearchKeyRange(headKey, value, null);
while (result == OperationStatus.SUCCESS) {
if (value.getData().length > 0) {
CrawlURI curi = (CrawlURI) crawlUriBinding.entryToObject(value);
if (!curi.getClassKey().equals(queue)) {
// rolled into next queue; finished with this queue
break;
}
if (pattern.matcher(curi.toString()).matches()) {
cursor.delete();
deletedCount++;
}
}
result = cursor.getNext(key, value, null);
}
} finally {
if (cursor != null) {
cursor.close();
}
}
return deletedCount;
}
public boolean delete(Transaction txn, K key) throws DatabaseException {
DatabaseEntry keyEntry = new DatabaseEntry();
keyBinding.objectToEntry(key, keyEntry);
OperationStatus status = db.delete(txn, keyEntry);
return (status == OperationStatus.SUCCESS);
}
/**
* Get the next nearest item after the given key. Relies on external discipline -- we'll look at
* the queues count of how many items it has -- to avoid asking for something from a range where
* there are no associated items -- otherwise could get first item of next 'queue' by mistake.
*
* <p>TODO: hold within a queue's range
*
* @param headKey Key prefix that demarks the beginning of the range in <code>pendingUrisDB</code>
* we're interested in.
* @return CrawlURI.
* @throws DatabaseException
*/
public CrawlURI get(DatabaseEntry headKey) throws DatabaseException {
DatabaseEntry result = new DatabaseEntry();
// From Linda Lee of sleepycat:
// "You want to check the status returned from Cursor.getSearchKeyRange
// to make sure that you have OperationStatus.SUCCESS. In that case,
// you have found a valid data record, and result.getData()
// (called by internally by the binding code, in this case) will be
// non-null. The other possible status return is
// OperationStatus.NOTFOUND, in which case no data record matched
// the criteria. "
OperationStatus status = getNextNearestItem(headKey, result);
CrawlURI retVal = null;
if (status != OperationStatus.SUCCESS) {
LOGGER.severe(
"See '1219854 NPE je-2.0 "
+ "entryToObject...'. OperationStatus "
+ " was not SUCCESS: "
+ status
+ ", headKey "
+ BdbWorkQueue.getPrefixClassKey(headKey.getData()));
return null;
}
try {
retVal = (CrawlURI) crawlUriBinding.entryToObject(result);
} catch (ClassCastException cce) {
Object obj = crawlUriBinding.entryToObject(result);
LOGGER.log(
Level.SEVERE,
"see [#HER-1283]: deserialized "
+ obj.getClass()
+ " has ClassLoader "
+ obj.getClass().getClassLoader().getClass(),
cce);
return null;
} catch (RuntimeExceptionWrapper rw) {
LOGGER.log(
Level.SEVERE,
"expected object missing in queue " + BdbWorkQueue.getPrefixClassKey(headKey.getData()),
rw);
return null;
}
retVal.setHolderKey(headKey);
return retVal;
}
public boolean contains(Transaction txn, K key, LockMode lockMode) throws DatabaseException {
DatabaseEntry keyEntry = new DatabaseEntry();
DatabaseEntry dataEntry = NO_RETURN_ENTRY;
keyBinding.objectToEntry(key, keyEntry);
OperationStatus status = db.get(txn, keyEntry, dataEntry, lockMode);
return (status == OperationStatus.SUCCESS);
}
@Override
public synchronized boolean containsDecision(Long instance) {
boolean found = false;
keyBinding.objectToEntry(instance, key);
OperationStatus status = db.get(null, key, data, LockMode.DEFAULT);
if (status == OperationStatus.SUCCESS) {
found = true;
}
if (logger.isDebugEnabled()) {
logger.debug("DB contains " + instance + " " + found + " (" + status.name() + ")");
}
return found;
}
/**
* Utility method to perform action for all pending CrawlURI instances.
*
* @param c Closure action to perform
* @throws DatabaseException
*/
protected void forAllPendingDo(Closure c) throws DatabaseException {
DatabaseEntry key = new DatabaseEntry();
DatabaseEntry value = new DatabaseEntry();
Cursor cursor = pendingUrisDB.openCursor(null, null);
while (cursor.getNext(key, value, null) == OperationStatus.SUCCESS) {
if (value.getData().length == 0) {
continue;
}
CrawlURI item = (CrawlURI) crawlUriBinding.entryToObject(value);
c.execute(item);
}
cursor.close();
}
private <V> EntityCursor<V> cursor(
Transaction txn,
K fromKey,
boolean fromInclusive,
K toKey,
boolean toInclusive,
ValueAdapter<V> adapter,
CursorConfig config)
throws DatabaseException {
DatabaseEntry fromEntry = null;
if (fromKey != null) {
fromEntry = new DatabaseEntry();
keyBinding.objectToEntry(fromKey, fromEntry);
}
DatabaseEntry toEntry = null;
if (toKey != null) {
toEntry = new DatabaseEntry();
keyBinding.objectToEntry(toKey, toEntry);
}
KeyRange range = emptyRange.subRange(fromEntry, fromInclusive, toEntry, toInclusive);
return cursor(txn, range, adapter, config);
}
/**
* Put the given CrawlURI in at the appropriate place.
*
* @param curi
* @throws DatabaseException
*/
public void put(CrawlURI curi, boolean overwriteIfPresent) throws DatabaseException {
DatabaseEntry insertKey = (DatabaseEntry) curi.getHolderKey();
if (insertKey == null) {
insertKey = calculateInsertKey(curi);
curi.setHolderKey(insertKey);
}
DatabaseEntry value = new DatabaseEntry();
crawlUriBinding.objectToEntry(curi, value);
// Output tally on avg. size if level is FINE or greater.
if (LOGGER.isLoggable(Level.FINE)) {
tallyAverageEntrySize(curi, value);
}
OperationStatus status;
if (overwriteIfPresent) {
status = pendingUrisDB.put(null, insertKey, value);
} else {
status = pendingUrisDB.putNoOverwrite(null, insertKey, value);
}
if (status != OperationStatus.SUCCESS) {
LOGGER.log(
Level.SEVERE, "URI enqueueing failed; " + status + " " + curi, new RuntimeException());
}
}
@Override
public synchronized boolean trim(Long instance) {
if (instance == 0) {
return true;
} // fast track
Transaction t = null;
if (db.getConfig().getTransactional()) {
t = env.beginTransaction(null, null);
}
Cursor cursor = db.openCursor(t, null);
boolean dirty = false;
try {
while (cursor.getNext(key, data, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) {
Long i = keyBinding.entryToObject(key);
if (i < instance && cursor.delete() != OperationStatus.SUCCESS) {
logger.error("Error deleting instance " + i + " from DB!");
dirty = true;
}
}
} finally {
cursor.close();
if (!dirty) {
if (t != null) {
t.commit();
}
} else {
if (t != null) {
t.abort();
}
return false;
}
}
putDecision(-1L, new Decision(0, instance, 0, null));
logger.debug("DB deltete up to instance " + instance);
return true;
}
/** Insert or retrieve data */
public void run() throws DatabaseException {
/* Create a new, transactional database environment */
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setTransactional(true);
envConfig.setAllowCreate(true);
Environment exampleEnv = new Environment(envDir, envConfig);
/* Make a database within that environment */
Transaction txn = exampleEnv.beginTransaction(null, null);
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setTransactional(true);
dbConfig.setAllowCreate(true);
dbConfig.setSortedDuplicates(true);
Database exampleDb = exampleEnv.openDatabase(txn, "bindingsDb", dbConfig);
/*
* In our example, the database record is composed of an integer
* key and and instance of the MyData class as data.
*
* A class catalog database is needed for storing class descriptions
* for the serial binding used below. This avoids storing class
* descriptions redundantly in each record.
*/
DatabaseConfig catalogConfig = new DatabaseConfig();
catalogConfig.setTransactional(true);
catalogConfig.setAllowCreate(true);
Database catalogDb = exampleEnv.openDatabase(txn, "catalogDb", catalogConfig);
StoredClassCatalog catalog = new StoredClassCatalog(catalogDb);
/*
* Create a serial binding for MyData data objects. Serial bindings
* can be used to store any Serializable object.
*/
EntryBinding<MyData> dataBinding = new SerialBinding<MyData>(catalog, MyData.class);
txn.commit();
/*
* Further below we'll use a tuple binding (IntegerBinding
* specifically) for integer keys. Tuples, unlike serialized Java
* objects, have a well defined sort order.
*/
/* DatabaseEntry represents the key and data of each record */
DatabaseEntry keyEntry = new DatabaseEntry();
DatabaseEntry dataEntry = new DatabaseEntry();
if (doInsert) {
/* put some data in */
for (int i = offset; i < numRecords + offset; i++) {
StringBuilder stars = new StringBuilder();
for (int j = 0; j < i; j++) {
stars.append('*');
}
MyData data = new MyData(i, stars.toString());
IntegerBinding.intToEntry(i, keyEntry);
dataBinding.objectToEntry(data, dataEntry);
txn = exampleEnv.beginTransaction(null, null);
OperationStatus status = exampleDb.put(txn, keyEntry, dataEntry);
/*
* Note that put will throw a DatabaseException when
* error conditions are found such as deadlock.
* However, the status return conveys a variety of
* information. For example, the put might succeed,
* or it might not succeed if the record exists
* and duplicates were not
*/
if (status != OperationStatus.SUCCESS) {
throw new RuntimeException("Data insertion got status " + status);
}
txn.commit();
}
} else {
/* retrieve the data */
Cursor cursor = exampleDb.openCursor(null, null);
while (cursor.getNext(keyEntry, dataEntry, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
int key = IntegerBinding.entryToInt(keyEntry);
MyData data = dataBinding.entryToObject(dataEntry);
System.out.println("key=" + key + " data=" + data);
}
cursor.close();
}
catalogDb.close();
exampleDb.close();
exampleEnv.close();
}
/** Insert or retrieve data. */
public void run() throws DatabaseException {
/* Create a new, transactional database environment. */
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setTransactional(true);
envConfig.setAllowCreate(true);
Environment exampleEnv = new Environment(envDir, envConfig);
/*
* Make a database within that environment. Because this will be used
* as a primary database, it must not allow duplicates. The primary key
* of a primary database must be unique.
*/
Transaction txn = exampleEnv.beginTransaction(null, null);
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setTransactional(true);
dbConfig.setAllowCreate(true);
Database exampleDb = exampleEnv.openDatabase(txn, "bindingsDb", dbConfig);
/*
* In our example, the database record is composed of an integer key
* and and instance of the MyData class as data.
*
* A class catalog database is needed for storing class descriptions
* for the serial binding used below. This avoids storing class
* descriptions redundantly in each record.
*/
DatabaseConfig catalogConfig = new DatabaseConfig();
catalogConfig.setTransactional(true);
catalogConfig.setAllowCreate(true);
Database catalogDb = exampleEnv.openDatabase(txn, "catalogDb", catalogConfig);
StoredClassCatalog catalog = new StoredClassCatalog(catalogDb);
/*
* Create a serial binding for MyData data objects. Serial
* bindings can be used to store any Serializable object.
*/
EntryBinding<MyData> dataBinding = new SerialBinding<MyData>(catalog, MyData.class);
/*
* Further below we'll use a tuple binding (IntegerBinding
* specifically) for integer keys. Tuples, unlike serialized
* Java objects, have a well defined sort order.
*/
/*
* Define a String tuple binding for a secondary key. The
* secondary key is the msg field of the MyData object.
*/
EntryBinding<String> secKeyBinding = TupleBinding.getPrimitiveBinding(String.class);
/*
* Open a secondary database to allow accessing the primary
* database by the secondary key value.
*/
SecondaryConfig secConfig = new SecondaryConfig();
secConfig.setTransactional(true);
secConfig.setAllowCreate(true);
secConfig.setSortedDuplicates(true);
secConfig.setKeyCreator(new MyKeyCreator(secKeyBinding, dataBinding));
SecondaryDatabase exampleSecDb =
exampleEnv.openSecondaryDatabase(txn, "bindingsSecDb", exampleDb, secConfig);
txn.commit();
/* DatabaseEntry represents the key and data of each record. */
DatabaseEntry keyEntry = new DatabaseEntry();
DatabaseEntry dataEntry = new DatabaseEntry();
if (doInsert) {
/*
* Put some data in. Note that the primary database is always used
* to add data. Adding or changing data in the secondary database
* is not allowed; however, deleting through the secondary database
* is allowed.
*/
for (int i = offset; i < numRecords + offset; i++) {
txn = exampleEnv.beginTransaction(null, null);
StringBuffer stars = new StringBuffer();
for (int j = 0; j < i; j++) {
stars.append('*');
}
MyData data = new MyData(i, stars.toString());
IntegerBinding.intToEntry(i, keyEntry);
dataBinding.objectToEntry(data, dataEntry);
OperationStatus status = exampleDb.put(txn, keyEntry, dataEntry);
/*
* Note that put will throw a DatabaseException when error
* conditions are found such as deadlock. However, the status
* return conveys a variety of information. For example, the
* put might succeed, or it might not succeed if the record
* exists and duplicates were not
*/
if (status != OperationStatus.SUCCESS) {
throw new RuntimeException("Data insertion got status " + status);
}
txn.commit();
}
} else {
/*
* Retrieve the data by secondary key by opening a cursor on the
* secondary database. The key parameter for a secondary cursor is
* always the secondary key, but the data parameter is always the
* data of the primary database. You can cast the cursor to a
* SecondaryCursor and use additional method signatures for
* retrieving the primary key also. Or you can call
* openSecondaryCursor() to avoid casting.
*/
txn = exampleEnv.beginTransaction(null, null);
Cursor cursor = exampleSecDb.openCursor(txn, null);
while (cursor.getNext(keyEntry, dataEntry, LockMode.DEFAULT) == OperationStatus.SUCCESS) {
String key = secKeyBinding.entryToObject(keyEntry);
MyData data = dataBinding.entryToObject(dataEntry);
System.out.println("key=" + key + " data=" + data);
}
cursor.close();
txn.commit();
}
/*
* Always close secondary databases before closing their associated
* primary database.
*/
catalogDb.close();
exampleSecDb.close();
exampleDb.close();
exampleEnv.close();
}
public V entryToValue(DatabaseEntry key, DatabaseEntry pkey, DatabaseEntry data) {
return (V) keyBinding.entryToObject(pkey);
}
/**
* @param m marker or null to start with first entry
* @param maxMatches
* @return list of matches starting from marker position
* @throws DatabaseException
*/
public CompositeData getFrom(String m, int maxMatches, Pattern pattern, boolean verbose)
throws DatabaseException {
int matches = 0;
int tries = 0;
ArrayList<String> results = new ArrayList<String>(maxMatches);
DatabaseEntry key;
if (m == null) {
key = getFirstKey();
} else {
byte[] marker = m.getBytes(); // = FrontierJMXTypes.fromString(m);
key = new DatabaseEntry(marker);
}
DatabaseEntry value = new DatabaseEntry();
Cursor cursor = null;
OperationStatus result = null;
try {
cursor = pendingUrisDB.openCursor(null, null);
result = cursor.getSearchKey(key, value, null);
while (matches < maxMatches && result == OperationStatus.SUCCESS) {
if (value.getData().length > 0) {
CrawlURI curi = (CrawlURI) crawlUriBinding.entryToObject(value);
if (pattern.matcher(curi.toString()).matches()) {
if (verbose) {
results.add("[" + curi.getClassKey() + "] " + curi.shortReportLine());
} else {
results.add(curi.toString());
}
matches++;
}
tries++;
}
result = cursor.getNext(key, value, null);
}
} finally {
if (cursor != null) {
cursor.close();
}
}
if (result != OperationStatus.SUCCESS) {
// end of scan
m = null;
} else {
m = new String(key.getData()); // = FrontierJMXTypes.toString(key.getData());
}
String[] arr = results.toArray(new String[results.size()]);
CompositeData cd;
try {
cd =
new CompositeDataSupport(
/*FrontierJMXTypes.URI_LIST_DATA*/ null,
new String[] {"list", "marker"},
new Object[] {arr, m});
} catch (OpenDataException e) {
throw new IllegalStateException(e);
}
return cd;
}