private void seriousError(Error err) {
// try to prevent timeslicing until we have a chance to deal with OOM
// Note that modern-day JVM priority indifference with native threads
// may make this priority-jumbling pointless
setPriority(DEFAULT_PRIORITY + 1);
if (controller != null) {
// hold all ToeThreads from proceeding to next processor
controller.freeReserveMemory();
controller.requestCrawlPause();
if (controller.getFrontier().getFrontierJournal() != null) {
controller.getFrontier().getFrontierJournal().seriousError(getName() + err.getMessage());
}
}
// OutOfMemory etc.
String extraInfo = DevUtils.extraInfo();
System.err.println("<<<");
System.err.println(ArchiveUtils.getLog17Date());
System.err.println(err);
System.err.println(extraInfo);
err.printStackTrace(System.err);
if (controller != null) {
PrintWriter pw = new PrintWriter(System.err);
controller.getToePool().compactReportTo(pw);
pw.flush();
}
System.err.println(">>>");
// DevUtils.sigquitSelf();
String context = "unknown";
if (currentCuri != null) {
// update fetch-status, saving original as annotation
currentCuri.getAnnotations().add("err=" + err.getClass().getName());
currentCuri.getAnnotations().add("os" + currentCuri.getFetchStatus());
currentCuri.setFetchStatus(S_SERIOUS_ERROR);
context = currentCuri.shortReportLine() + " in " + currentProcessorName;
}
String message = "Serious error occured trying " + "to process '" + context + "'\n" + extraInfo;
logger.log(Level.SEVERE, message.toString(), err);
setPriority(DEFAULT_PRIORITY);
}
/**
* @param m marker or null to start with first entry
* @param maxMatches
* @return list of matches starting from marker position
* @throws DatabaseException
*/
public CompositeData getFrom(String m, int maxMatches, Pattern pattern, boolean verbose)
throws DatabaseException {
int matches = 0;
int tries = 0;
ArrayList<String> results = new ArrayList<String>(maxMatches);
DatabaseEntry key;
if (m == null) {
key = getFirstKey();
} else {
byte[] marker = m.getBytes(); // = FrontierJMXTypes.fromString(m);
key = new DatabaseEntry(marker);
}
DatabaseEntry value = new DatabaseEntry();
Cursor cursor = null;
OperationStatus result = null;
try {
cursor = pendingUrisDB.openCursor(null, null);
result = cursor.getSearchKey(key, value, null);
while (matches < maxMatches && result == OperationStatus.SUCCESS) {
if (value.getData().length > 0) {
CrawlURI curi = (CrawlURI) crawlUriBinding.entryToObject(value);
if (pattern.matcher(curi.toString()).matches()) {
if (verbose) {
results.add("[" + curi.getClassKey() + "] " + curi.shortReportLine());
} else {
results.add(curi.toString());
}
matches++;
}
tries++;
}
result = cursor.getNext(key, value, null);
}
} finally {
if (cursor != null) {
cursor.close();
}
}
if (result != OperationStatus.SUCCESS) {
// end of scan
m = null;
} else {
m = new String(key.getData()); // = FrontierJMXTypes.toString(key.getData());
}
String[] arr = results.toArray(new String[results.size()]);
CompositeData cd;
try {
cd =
new CompositeDataSupport(
/*FrontierJMXTypes.URI_LIST_DATA*/ null,
new String[] {"list", "marker"},
new Object[] {arr, m});
} catch (OpenDataException e) {
throw new IllegalStateException(e);
}
return cd;
}
