private void runLoop() throws InterruptedException {
while (!stop) {
CrawlerTask task = crawlerResponseQueue.take();
if (task.isExitTask()) {
crawlerRequestQueue.add(CrawlerTask.createExitTask());
crawlerResponseQueue.add(task);
return;
}
HttpUriRequest req = task.getRequest();
activeRequest = req;
try {
if (task.getResponse() != null) {
task.getResponseProcessor()
.processResponse(crawler, req, task.getResponse(), task.getArgument());
}
} catch (Exception e) {
logger.log(
Level.WARNING, "Unexpected exception processing crawler request: " + req.getURI(), e);
} finally {
synchronized (requestLock) {
activeRequest = null;
}
final HttpEntity entity =
(task.getResponse() == null) ? (null) : task.getResponse().getRawResponse().getEntity();
if (entity != null)
try {
EntityUtils.consume(entity);
} catch (IOException e) {
logger.log(
Level.WARNING,
"I/O exception consuming request entity content for "
+ req.getURI()
+ " : "
+ e.getMessage());
}
}
synchronized (counter) {
counter.addCompletedTask();
crawler.updateProgress();
}
if (task.causedException()) {
crawler.notifyException(req, task.getException());
}
if (outstandingTasks.decrementAndGet() <= 0) {
crawlerRequestQueue.add(CrawlerTask.createExitTask());
crawlerResponseQueue.add(CrawlerTask.createExitTask());
return;
}
}
}
void stop() {
stop = true;
if (!crawlerResponseQueue.offer(CrawlerTask.createExitTask())) {
logger.warning("Failed to add STOP sentinel to crawler response queue");
}
synchronized (requestLock) {
if (activeRequest != null) activeRequest.abort();
}
}