Example #1
0
 private String parseFilename() {
   if (StringUtils.isEmpty(this.webSite)
       || StringUtils.isEmpty(this.keyword)
       || StringUtils.isEmpty(this.uid)
       || StringUtils.isEmpty(type)) {
     return "";
   } else {
     return this.webSite + "/" + this.keyword + "/" + this.uid + "." + this.type;
   }
 }
 private void store(List<ParseResult> list, String name) {
   CacheClient cache = CommonUtils.getCacheClient();
   int count = 0;
   try {
     for (ParseResult pr : list) {
       WebPage page = pr.getPage();
       String url = page.getUrl();
       // WebPageRepository.showPage(page);
       if (!CommonUtils.checkUrl(cache, url)) {
         String content = pr.getContent();
         if (StringUtils.isEmpty(content)) {
           continue;
         }
         try {
           WebPageRepository.save(page);
         } catch (Exception e) {
           LOGGER.error(e.getMessage() + "保存至sql失败! url:" + page.getUrl(), e);
           continue;
         }
         try {
           CommonUtils.storage(docUrl, true, page.getPageId(), content, true);
         } catch (Exception e) {
           LOGGER.error(e.getMessage() + "保存至文档服务器失败!", e);
           continue;
         }
         CommonUtils.storeUrl(cache, url);
         count++;
       }
     }
   } finally {
     CommonUtils.recycleCacheClient(cache);
   }
   LOGGER.info(name + "的facebook 成功存储 " + count + " 条, 总共下载 " + list.size() + " 条");
 }
Example #3
0
 public Picture(String url, String webSite, String keyword) {
   super();
   this.url = url;
   this.webSite = webSite;
   this.keyword = keyword;
   this.uid = StringUtils.MD5(url);
   this.type = parseType(url);
   this.filename = parseFilename();
 }
  /** {@inheritDoc} */
  public void init() throws ServerInitException {
    try {
      LOGGER.info("正在初始化数据库连接池...");
      Properties prop = loadJdbcProperties();
      dataSource = new ComboPooledDataSource();
      dataSource.setDriverClass(prop.getProperty(DRIVER_PROP));
      dataSource.setJdbcUrl(prop.getProperty(URL_PROP));
      dataSource.setUser(prop.getProperty(USERNAME_PROP));
      dataSource.setPassword(prop.getProperty(PASSWORD_PROP));
      dataSource.setCheckoutTimeout(
          StringUtils.parseInt(prop.getProperty(MAXWAIT_PROP), DEFAULT_MAXWAIT_TIME));
      dataSource.setMaxPoolSize(30);
      dataSource.setInitialPoolSize(10);

      testConnection(dataSource);
      LOGGER.info("初始化数据库连接池成功!");
    } catch (IOException e) {
      throw new ServerInitException("初始化数据库连接池失败!", e);
    } catch (PropertyVetoException e) {
      throw new ServerInitException("初始化数据库连接池失败!", e);
    }
  }