protected List<Resource> getResourceByList(String sql) {
List<Resource> resList = new ArrayList<Resource>();
Resource res = null;
ResultSet rs = null;
Statement stmt = null;
try {
stmt = daoHelp.getConnection().createStatement();
rs = stmt.executeQuery(sql);
while (rs.next()) {
int rid = rs.getInt("rid");
res = this.createResourceFromRecord(rs);
daoHelp.execSql("update spider_resource set crawl = 1 where rid = " + rid);
resList.add(res);
}
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} catch (SQLException e) {
log.error("SQLException -> ", e);
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} finally {
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
}
return resList;
}
protected synchronized boolean isHavingUrl(String url) {
boolean isHaving = false;
// 判断该URL是否这次任务中被抓取
if (URLCache.findUrlInCache(url)) // 存在为true
return true;
// 判断该URL是否曾已被抓取
if (storage.getSinfoDAO().isCrawlUrl(url)) return true;
Statement stmt = null;
String sql = "select top 1 url from spider_resource where url = '" + url + "'";
ResultSet rs = null;
try {
stmt = daoHelp.getConnection().createStatement();
rs = stmt.executeQuery(sql);
while (rs.next()) {
isHaving = true;
}
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} catch (SQLException e) {
log.error("SQLException -> ", e);
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} finally {
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
}
return isHaving;
}
public List<CheckInfo> get() {
List<CheckInfo> list = new ArrayList<CheckInfo>();
Statement stmt = null;
String sql = "select * from spider_check where cType <> 5 order by sid ";
ResultSet rs = null;
try {
stmt = daoHelp.getConnection().createStatement();
rs = stmt.executeQuery(sql);
while (rs.next()) {
CheckInfo checkInfo = new CheckInfo();
checkInfo.setSid(rs.getString(1));
checkInfo.setPid(rs.getString(2));
checkInfo.setField(rs.getString(3));
checkInfo.setUrl(rs.getString(4));
checkInfo.setCType(rs.getInt(5));
checkInfo.setSite(rs.getString(6));
checkInfo.setSType(rs.getString(7));
checkInfo.setContent(rs.getString(8));
list.add(checkInfo);
}
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} catch (SQLException e) {
log.error("SQLException -> ", e);
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} finally {
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
}
return list;
}
private void updResourceInfo(String url, String site, String type, int city) {
int rid = 0;
Statement stmt = null;
String sql = "select rid from spider_resource where url = '" + url + "'";
ResultSet rs = null;
try {
stmt = daoHelp.getConnection().createStatement();
rs = stmt.executeQuery(sql);
while (rs.next()) {
rid = rs.getInt("rid");
}
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} catch (SQLException e) {
log.error("SQLException -> ", e);
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} finally {
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
}
String uSql = "update spider_resource set paid = " + rid + " where rid = " + rid;
daoHelp.execSql(uSql);
String infoSql =
"insert into spider_resource_info(rid, site, iType, city ) "
+ " values ("
+ ""
+ rid
+ ","
+ daoHelp.format(site)
+ ", "
+ daoHelp.format(type)
+ ", "
+ daoHelp.format(city)
+ ")";
daoHelp.execSql(infoSql);
}
private boolean isAdd() {
boolean isGet = false;
String sql =
"select top 1 rid rid from spider_resource where crawl = 1 and ( httpStatus = 0 or httpStatus is NULL)";
Statement stmt = null;
ResultSet rs = null;
try {
stmt = daoHelp.getConnection().createStatement();
rs = stmt.executeQuery(sql);
while (rs.next()) {
isGet = true;
}
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} catch (SQLException e) {
log.error("SQLException -> ", e);
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} finally {
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
}
return isGet;
}
public Date[] getCrawlDate() {
Date[] date = new Date[2];
String sql = "select * from spider_info ";
ResultSet rs = null;
Statement stmt = null;
try {
stmt = daoHelp.getConnection().createStatement();
rs = stmt.executeQuery(sql);
while (rs.next()) {
date[0] = rs.getDate(1);
date[1] = rs.getDate(2);
}
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} catch (SQLException e) {
log.error("SQLException -> ", e);
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} finally {
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
}
return date;
}