Exemplo n.º 1
0
  protected List<Resource> getResourceByList(String sql) {
    List<Resource> resList = new ArrayList<Resource>();
    Resource res = null;
    ResultSet rs = null;
    Statement stmt = null;
    try {
      stmt = daoHelp.getConnection().createStatement();
      rs = stmt.executeQuery(sql);
      while (rs.next()) {
        int rid = rs.getInt("rid");
        res = this.createResourceFromRecord(rs);
        daoHelp.execSql("update spider_resource set crawl = 1 where rid = " + rid);
        resList.add(res);
      }
      daoHelp.safeClose(rs);
      daoHelp.safeClose(stmt);
    } catch (SQLException e) {
      log.error("SQLException -> ", e);
      daoHelp.safeClose(rs);
      daoHelp.safeClose(stmt);
    } finally {
      daoHelp.safeClose(rs);
      daoHelp.safeClose(stmt);
    }

    return resList;
  }
Exemplo n.º 2
0
 protected synchronized boolean isHavingUrl(String url) {
   boolean isHaving = false;
   // 判断该URL是否这次任务中被抓取
   if (URLCache.findUrlInCache(url)) // 存在为true
   return true;
   // 判断该URL是否曾已被抓取
   if (storage.getSinfoDAO().isCrawlUrl(url)) return true;
   Statement stmt = null;
   String sql = "select top 1 url from spider_resource where url = '" + url + "'";
   ResultSet rs = null;
   try {
     stmt = daoHelp.getConnection().createStatement();
     rs = stmt.executeQuery(sql);
     while (rs.next()) {
       isHaving = true;
     }
     daoHelp.safeClose(rs);
     daoHelp.safeClose(stmt);
   } catch (SQLException e) {
     log.error("SQLException -> ", e);
     daoHelp.safeClose(rs);
     daoHelp.safeClose(stmt);
   } finally {
     daoHelp.safeClose(rs);
     daoHelp.safeClose(stmt);
   }
   return isHaving;
 }
Exemplo n.º 3
0
  public List<CheckInfo> get() {
    List<CheckInfo> list = new ArrayList<CheckInfo>();
    Statement stmt = null;
    String sql = "select * from spider_check where cType <> 5 order by sid ";
    ResultSet rs = null;
    try {
      stmt = daoHelp.getConnection().createStatement();
      rs = stmt.executeQuery(sql);
      while (rs.next()) {
        CheckInfo checkInfo = new CheckInfo();
        checkInfo.setSid(rs.getString(1));
        checkInfo.setPid(rs.getString(2));
        checkInfo.setField(rs.getString(3));
        checkInfo.setUrl(rs.getString(4));
        checkInfo.setCType(rs.getInt(5));
        checkInfo.setSite(rs.getString(6));
        checkInfo.setSType(rs.getString(7));
        checkInfo.setContent(rs.getString(8));
        list.add(checkInfo);
      }
      daoHelp.safeClose(rs);
      daoHelp.safeClose(stmt);
    } catch (SQLException e) {
      log.error("SQLException -> ", e);
      daoHelp.safeClose(rs);
      daoHelp.safeClose(stmt);
    } finally {
      daoHelp.safeClose(rs);
      daoHelp.safeClose(stmt);
    }

    return list;
  }
Exemplo n.º 4
0
  private void updResourceInfo(String url, String site, String type, int city) {
    int rid = 0;
    Statement stmt = null;
    String sql = "select rid from spider_resource where url = '" + url + "'";
    ResultSet rs = null;
    try {
      stmt = daoHelp.getConnection().createStatement();
      rs = stmt.executeQuery(sql);
      while (rs.next()) {
        rid = rs.getInt("rid");
      }
      daoHelp.safeClose(rs);
      daoHelp.safeClose(stmt);
    } catch (SQLException e) {
      log.error("SQLException -> ", e);
      daoHelp.safeClose(rs);
      daoHelp.safeClose(stmt);
    } finally {
      daoHelp.safeClose(rs);
      daoHelp.safeClose(stmt);
    }
    String uSql = "update spider_resource set paid = " + rid + " where rid = " + rid;
    daoHelp.execSql(uSql);

    String infoSql =
        "insert into spider_resource_info(rid, site, iType, city ) "
            + " values ("
            + ""
            + rid
            + ","
            + daoHelp.format(site)
            + ", "
            + daoHelp.format(type)
            + ", "
            + daoHelp.format(city)
            + ")";
    daoHelp.execSql(infoSql);
  }
Exemplo n.º 5
0
 private boolean isAdd() {
   boolean isGet = false;
   String sql =
       "select top 1 rid rid from spider_resource where crawl = 1 and ( httpStatus = 0 or httpStatus is NULL)";
   Statement stmt = null;
   ResultSet rs = null;
   try {
     stmt = daoHelp.getConnection().createStatement();
     rs = stmt.executeQuery(sql);
     while (rs.next()) {
       isGet = true;
     }
     daoHelp.safeClose(rs);
     daoHelp.safeClose(stmt);
   } catch (SQLException e) {
     log.error("SQLException -> ", e);
     daoHelp.safeClose(rs);
     daoHelp.safeClose(stmt);
   } finally {
     daoHelp.safeClose(rs);
     daoHelp.safeClose(stmt);
   }
   return isGet;
 }
Exemplo n.º 6
0
 public Date[] getCrawlDate() {
   Date[] date = new Date[2];
   String sql = "select * from spider_info ";
   ResultSet rs = null;
   Statement stmt = null;
   try {
     stmt = daoHelp.getConnection().createStatement();
     rs = stmt.executeQuery(sql);
     while (rs.next()) {
       date[0] = rs.getDate(1);
       date[1] = rs.getDate(2);
     }
     daoHelp.safeClose(rs);
     daoHelp.safeClose(stmt);
   } catch (SQLException e) {
     log.error("SQLException -> ", e);
     daoHelp.safeClose(rs);
     daoHelp.safeClose(stmt);
   } finally {
     daoHelp.safeClose(rs);
     daoHelp.safeClose(stmt);
   }
   return date;
 }