private String parseFilename() {
if (StringUtils.isEmpty(this.webSite)
|| StringUtils.isEmpty(this.keyword)
|| StringUtils.isEmpty(this.uid)
|| StringUtils.isEmpty(type)) {
return "";
} else {
return this.webSite + "/" + this.keyword + "/" + this.uid + "." + this.type;
}
}
private void store(List<ParseResult> list, String name) {
CacheClient cache = CommonUtils.getCacheClient();
int count = 0;
try {
for (ParseResult pr : list) {
WebPage page = pr.getPage();
String url = page.getUrl();
// WebPageRepository.showPage(page);
if (!CommonUtils.checkUrl(cache, url)) {
String content = pr.getContent();
if (StringUtils.isEmpty(content)) {
continue;
}
try {
WebPageRepository.save(page);
} catch (Exception e) {
LOGGER.error(e.getMessage() + "保存至sql失败！ url：" + page.getUrl(), e);
continue;
}
try {
CommonUtils.storage(docUrl, true, page.getPageId(), content, true);
} catch (Exception e) {
LOGGER.error(e.getMessage() + "保存至文档服务器失败！", e);
continue;
}
CommonUtils.storeUrl(cache, url);
count++;
}
}
} finally {
CommonUtils.recycleCacheClient(cache);
}
LOGGER.info(name + "的facebook 成功存储 " + count + " 条, 总共下载 " + list.size() + " 条");
}
public Picture(String url, String webSite, String keyword) {
super();
this.url = url;
this.webSite = webSite;
this.keyword = keyword;
this.uid = StringUtils.MD5(url);
this.type = parseType(url);
this.filename = parseFilename();
}
/** {@inheritDoc} */
public void init() throws ServerInitException {
try {
LOGGER.info("正在初始化数据库连接池...");
Properties prop = loadJdbcProperties();
dataSource = new ComboPooledDataSource();
dataSource.setDriverClass(prop.getProperty(DRIVER_PROP));
dataSource.setJdbcUrl(prop.getProperty(URL_PROP));
dataSource.setUser(prop.getProperty(USERNAME_PROP));
dataSource.setPassword(prop.getProperty(PASSWORD_PROP));
dataSource.setCheckoutTimeout(
StringUtils.parseInt(prop.getProperty(MAXWAIT_PROP), DEFAULT_MAXWAIT_TIME));
dataSource.setMaxPoolSize(30);
dataSource.setInitialPoolSize(10);
testConnection(dataSource);
LOGGER.info("初始化数据库连接池成功！");
} catch (IOException e) {
throw new ServerInitException("初始化数据库连接池失败！", e);
} catch (PropertyVetoException e) {
throw new ServerInitException("初始化数据库连接池失败！", e);
}
}
