private void trySaveParseResults(List<ParseResult> results) {
CacheClient session = CommonUtils.getCacheClient();
int count = 0;
try {
for (int i = 0; i < results.size(); i++) {
ParseResult result = results.get(i);
try {
if (!CommonUtils.checkUrl(session, result.getPage().getUrl())) {
WebPage page = result.getPage();
String content = result.getContent();
// 保存文章到HBase文档服务器
try {
if (SystemProps.storeable()) {
CommonUtils.storage(true, page.getId(), content, true);
}
} catch (Exception e) {
LOGGER.error("保存至文档服务器失败！ ： " + e.getMessage());
continue;
}
// 保存页面到sql数据库
// WebPageRepository.save(page);
try {
WebPageManager.getInstance().savePage(page);
} catch (SQLException e) {
LOGGER.error(e.getMessage() + "保存至sql数据库失败", e);
continue;
} catch (Exception e) {
LOGGER.error(e.getMessage() + "保存至sql数据库失败", e);
continue;
}
// 保存contentUrl，即AJAX的JSON数据的URL
CommonUtils.storeUrl(session, page.getUrl());
count++;
}
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
}
}
} finally {
LOGGER.info("存放爬取结果" + count + "条");
try {
CommonUtils.recycleCacheClient(session);
} catch (Exception e) {
LOGGER.error(e.getMessage() + "释放cacheclient失败");
}
}
}
