public void run() {
String url = pageInfo.getUrl();
try {
int distance = pageInfo.getDistance();
logger.info("Crawling " + url.toString());
download();
extractLinks(); // 抽取链接
crawler.reportPageFetched(pageInfo); // 保存需要store的
if (pageInfo.getCrawlFlag() == CrawlAction.FOLLOW) {
logger.debug("url:" + url + " ----------------follow");
crawler.reportLinks(pageInfo.getLinks(), distance, url);
} else if (pageInfo.getCrawlFlag() == CrawlAction.STORE) {
logger.debug("url:" + url + " ----------------store");
crawler.reportLinks(pageInfo.getLinks(), distance + 1, url);
}
} catch (WontFetchException e) {
logger.error("Won't fetch " + url);
e.printStackTrace();
} catch (Exception e) {
logger.error("Error crawling %s", e, url);
e.printStackTrace();
} finally {
crawler.reportJobDone();
}
}
