protected long deleteMatchingFromQueue(final WorkQueueFrontier frontier, final String match)
throws IOException {
try {
final BdbMultipleWorkQueues queues = ((BdbFrontier) frontier).getWorkQueues();
return queues.deleteMatchingFromQueue(match, classKey, new DatabaseEntry(origin));
} catch (DatabaseException e) {
throw IoUtils.wrapAsIOException(e);
}
}
protected void deleteItem(final WorkQueueFrontier frontier, final CrawlURI peekItem)
throws IOException {
try {
final BdbMultipleWorkQueues queues = ((BdbFrontier) frontier).getWorkQueues();
queues.delete(peekItem);
} catch (DatabaseException e) {
e.printStackTrace();
throw IoUtils.wrapAsIOException(e);
}
}
protected CrawlURI peekItem(final WorkQueueFrontier frontier) throws IOException {
final BdbMultipleWorkQueues queues = ((BdbFrontier) frontier).getWorkQueues();
DatabaseEntry key = new DatabaseEntry(origin);
CrawlURI curi = null;
int tries = 1;
while (true) {
try {
curi = queues.get(key);
} catch (DatabaseException e) {
LOGGER.log(Level.SEVERE, "peekItem failure; retrying", e);
}
// ensure CrawlURI, if any, came from acceptable range:
if (!ArchiveUtils.startsWith(key.getData(), origin)) {
LOGGER.severe(
"inconsistency: "
+ classKey
+ "("
+ getPrefixClassKey(origin)
+ ") with "
+ getCount()
+ " items gave "
+ curi
+ "("
+ getPrefixClassKey(key.getData()));
// clear curi to allow retry
curi = null;
// reset key to original origin for retry
key.setData(origin);
}
if (curi != null) {
// success
break;
}
if (tries > 3) {
LOGGER.severe("no item where expected in queue " + classKey);
break;
}
tries++;
LOGGER.severe(
"Trying get #"
+ Integer.toString(tries)
+ " in queue "
+ classKey
+ " with "
+ getCount()
+ " items using key "
+ getPrefixClassKey(key.getData()));
}
return curi;
}
/**
* Create a virtual queue inside the given BdbMultipleWorkQueues
*
* @param classKey
*/
public BdbWorkQueue(String classKey, BdbFrontier frontier) {
super(classKey);
this.origin = BdbMultipleWorkQueues.calculateOriginKey(classKey);
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine(getPrefixClassKey(this.origin) + " " + classKey);
}
// add the queue-front 'cap' entry; see...
// http://sourceforge.net/tracker/index.php?func=detail&aid=1262665&group_id=73833&atid=539102
frontier.getWorkQueues().addCap(origin);
}
protected void insertItem(
final WorkQueueFrontier frontier, final CrawlURI curi, boolean overwriteIfPresent)
throws IOException {
try {
final BdbMultipleWorkQueues queues = ((BdbFrontier) frontier).getWorkQueues();
queues.put(curi, overwriteIfPresent);
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.fine(
"Inserted into "
+ getPrefixClassKey(this.origin)
+ " (count "
+ Long.toString(getCount())
+ "): "
+ curi.toString());
}
} catch (DatabaseException e) {
throw IoUtils.wrapAsIOException(e);
}
}