private void startCrawlThread(ICrawler cw) {
RunnableCrawler rc = null;
synchronized (runnables) {
rc = runnables.get(cw);
}
Thread t = new Thread(rc, "crawler-" + cw.toString());
synchronized (runningCrawlers) {
runningCrawlers.put(cw.getType(), t);
}
t.start();
}
/**
* Queue a new data (re)fetch on a crawler & asynchronously index it.
*
* @param type
* @param id
*/
public void crawlData(String type, String id) {
if (stopped) {
return;
}
ICrawler c = null;
synchronized (crawlers) {
c = crawlers.get(type);
}
if (c != null) {
c.queueFetch(id);
runCrawler(c);
} else {
logger.warn("No crawler registered for fetching " + type + " / " + id);
}
}
private void runCrawler(ICrawler cw) {
if (stopped) {
return;
}
Thread t = null;
synchronized (runningCrawlers) {
t = runningCrawlers.get(cw.getType());
}
if (t == null || !t.isAlive()) {
startCrawlThread(cw);
}
}
