예제 #1
0
  protected List<Resource> getResourceByList(String sql) {
    List<Resource> resList = new ArrayList<Resource>();
    Resource res = null;
    ResultSet rs = null;
    Statement stmt = null;
    try {
      stmt = daoHelp.getConnection().createStatement();
      rs = stmt.executeQuery(sql);
      while (rs.next()) {
        int rid = rs.getInt("rid");
        res = this.createResourceFromRecord(rs);
        daoHelp.execSql("update spider_resource set crawl = 1 where rid = " + rid);
        resList.add(res);
      }
      daoHelp.safeClose(rs);
      daoHelp.safeClose(stmt);
    } catch (SQLException e) {
      log.error("SQLException -> ", e);
      daoHelp.safeClose(rs);
      daoHelp.safeClose(stmt);
    } finally {
      daoHelp.safeClose(rs);
      daoHelp.safeClose(stmt);
    }

    return resList;
  }
예제 #2
0
  public void createSeed(String url, String site, int city, String type, String seedType) {
    String sql =
        "insert into spider_resource(url, type) values ('" + url + "', '" + seedType + "')";
    daoHelp.execSql(sql);

    this.updResourceInfo(url, site, type, city);
  }
예제 #3
0
 public void setSpidered(String url, URLSpideredProxyOkEvent event) {
   String sql =
       "update spider_resource set saved = 1, httpStatus = "
           + daoHelp.format(event.getHttpStatus())
           + ", "
           + " dTime = "
           + daoHelp.format(new Date().toLocaleString())
           + " where url = "
           + daoHelp.format(url)
           + "";
   daoHelp.execSql(sql);
 }
예제 #4
0
 public void setSpiderError(URLSpideredProxyErrorEvent event) {
   String sql =
       "update spider_resource set saved = 0, httpStatus = "
           + event.getHttpStatus()
           + ", "
           + " dTime = '"
           + new Date().toLocaleString()
           + "' where url = '"
           + event.getUrl()
           + "'";
   daoHelp.execSql(sql);
 }
예제 #5
0
 public void createSeed(SeedInfo seedInfo) {
   String sql =
       "insert into spider_resource(url, type, kid) values ('"
           + seedInfo.getUrl()
           + "', '"
           + seedInfo.getSeedType()
           + "', "
           + seedInfo.getKid()
           + ")";
   daoHelp.execSql(sql);
   this.updResourceInfo(
       seedInfo.getUrl(), seedInfo.getSite(), seedInfo.getType(), seedInfo.getCity());
 }
예제 #6
0
  private void updResourceInfo(String url, String site, String type, int city) {
    int rid = 0;
    Statement stmt = null;
    String sql = "select rid from spider_resource where url = '" + url + "'";
    ResultSet rs = null;
    try {
      stmt = daoHelp.getConnection().createStatement();
      rs = stmt.executeQuery(sql);
      while (rs.next()) {
        rid = rs.getInt("rid");
      }
      daoHelp.safeClose(rs);
      daoHelp.safeClose(stmt);
    } catch (SQLException e) {
      log.error("SQLException -> ", e);
      daoHelp.safeClose(rs);
      daoHelp.safeClose(stmt);
    } finally {
      daoHelp.safeClose(rs);
      daoHelp.safeClose(stmt);
    }
    String uSql = "update spider_resource set paid = " + rid + " where rid = " + rid;
    daoHelp.execSql(uSql);

    String infoSql =
        "insert into spider_resource_info(rid, site, iType, city ) "
            + " values ("
            + ""
            + rid
            + ","
            + daoHelp.format(site)
            + ", "
            + daoHelp.format(type)
            + ", "
            + daoHelp.format(city)
            + ")";
    daoHelp.execSql(infoSql);
  }
예제 #7
0
 public void create(CheckInfo info) {
   StringBuffer sql = new StringBuffer();
   sql.append("insert into spider_check (sid, pid, field, url, cType, site, sType, content");
   sql.append(" ) values ( ");
   sql.append(daoHelp.format(info.getSid()) + ",");
   sql.append(daoHelp.format(info.getPid()) + ",");
   sql.append(daoHelp.format(info.getField()) + ",");
   sql.append(daoHelp.format(info.getUrl()) + ",");
   sql.append(daoHelp.format(info.getCType()) + ",");
   sql.append(daoHelp.format(info.getSite()) + ",");
   sql.append(daoHelp.format(info.getSType()) + ",");
   sql.append(daoHelp.format(info.getContent()) + ")");
   daoHelp.execSql(sql.toString());
 }
예제 #8
0
 public boolean create(Resource resource) {
   if (this.isHavingUrl(resource.getUrl())) return false;
   StringBuffer sql = new StringBuffer();
   sql.append("insert into spider_resource( ");
   sql.append("paId,");
   sql.append("url,");
   sql.append("type,");
   sql.append("httpStatus,");
   sql.append("dTime");
   sql.append(" ) values ( ");
   sql.append(daoHelp.format(resource.getPaId()) + ",");
   sql.append(daoHelp.format(resource.getUrl()) + ",");
   sql.append(daoHelp.format(resource.getType()) + ",");
   sql.append("0,");
   sql.append("''");
   sql.append(" ) ");
   daoHelp.execSql(sql.toString());
   return true;
 }
예제 #9
0
 public List<Resource> setSpider() {
   String sql =
       "update spider_resource set crawl = 0 , saved = 0, httpStatus = NULL , dTime = 0 where saved = 0 and crawl = 1  and (HttpStatus is NULL or HttpStatus  < 100  ) ";
   daoHelp.execSql(sql);
   return this.getSomeSeedResource();
 }
예제 #10
0
 public void addRequsetCount(int rid, int reqCount) {
   String sql = "update spider_resource set reqCount = " + (++reqCount) + " where rid = " + rid;
   daoHelp.execSql(sql);
 }
예제 #11
0
 public void setSeedNotSpider() {
   String sql =
       "update spider_resource set saved = 0, crawl = 0, HttpStatus = 0, reqCount = 0 where (type = 'monitor' or type = 'seed') and httpStatus <= 301";
   daoHelp.execSql(sql);
 }
예제 #12
0
 public void setStartTime() {
   SimpleDateFormat dateformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
   String sDate = dateformat.format(new Date());
   String sql = "update spider_info set stime = '" + sDate + "'";
   daoHelp.execSql(sql);
 }
예제 #13
0
 public void deleteAll() {
   String sql = "delete from spider_resource";
   daoHelp.execSql(sql);
 }
예제 #14
0
 public void setParsered(Resource res) {
   String sql = "update spider_resource set parser = 1 where rid = " + res.getRId();
   daoHelp.execSql(sql);
 }
예제 #15
0
 public void setParsered(ResourceParsedOkEvent event) {
   String sql = "update spider_resource set parser = 1 where url = '" + event.getUrl() + "'";
   daoHelp.execSql(sql);
 }