private void registerCrawler(AbstractCrawler cw) {
synchronized (crawlers) {
crawlers.put(cw.getType(), cw);
}
synchronized (runnables) {
runnables.put(cw, new RunnableCrawler(cw));
}
idxMgr.setupIndex(cw.getType());
cw.registerListener(this);
}
@Override
public void crawlComplete(String type, boolean commitNeeded) {
if (commitNeeded) {
idxMgr.getIndex(type).commit();
}
synchronized (runningCrawlers) {
runningCrawlers.remove(type);
}
logger.info("type " + type + " removed from runningCrawlers");
}
public void clearIndex(String type) {
idxMgr.getIndex(type).deleteByQuery("type:" + type);
}
public void queueDeletion(String type, String id) {
idxMgr.delete(type, id);
}
@Override
public List<Hit> findByType(String type, String query) {
return idxMgr.getIndex(type).doQuery(query);
}
@Override
public void dataFetched(String type, Map<String, String> fetchedData) {
idxMgr.index(type, fetchedData);
}