public void run() {
LiveQueueItem currentPage = new LiveQueueItem(0, "");
LiveQueueItem lastPage = null;
while (keepRunning) {
try {
LiveQueueItem page = LiveQueue.take();
if (page.equals(lastPage)) {
logger.info(
"Ignoring duplicatre page {} ({}) with priority {}",
page.getItemName(),
page.getItemID(),
page.getPriority());
continue;
}
lastPage = page;
currentPage = page;
// If a mapping page set extractor to reload mappings and ontology
if (page.getPriority() == LiveQueuePriority.MappingPriority) {
LiveExtractionConfigLoader.reload(page.getStatQueueAdd());
}
if (page.isDeleted() == true) {
JSONCache.deleteCacheItem(page.getItemID(), LiveExtractionConfigLoader.policies());
logger.info(
"Deleted page with ID: " + page.getItemID() + " (" + page.getItemName() + ")");
} else processPage(page);
} catch (Exception exp) {
logger.error(
"Failed to process page " + currentPage.getItemID() + " reason: " + exp.getMessage(),
exp);
}
}
}
private void processPage(LiveQueueItem item) {
try {
Boolean extracted =
LiveExtractionConfigLoader.extractPage(
item, LiveOptions.options.get("localApiURL"), LiveOptions.language);
if (!extracted) JSONCache.setErrorOnCache(item.getItemID(), -1);
} catch (Exception exp) {
logger.error(
"Error in processing page number "
+ item.getItemID()
+ ", and the reason is "
+ exp.getMessage(),
exp);
JSONCache.setErrorOnCache(item.getItemID(), -2);
}
}
