protected List<Resource> getResourceByList(String sql) {
List<Resource> resList = new ArrayList<Resource>();
Resource res = null;
ResultSet rs = null;
Statement stmt = null;
try {
stmt = daoHelp.getConnection().createStatement();
rs = stmt.executeQuery(sql);
while (rs.next()) {
int rid = rs.getInt("rid");
res = this.createResourceFromRecord(rs);
daoHelp.execSql("update spider_resource set crawl = 1 where rid = " + rid);
resList.add(res);
}
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} catch (SQLException e) {
log.error("SQLException -> ", e);
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} finally {
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
}
return resList;
}
public void createSeed(String url, String site, int city, String type, String seedType) {
String sql =
"insert into spider_resource(url, type) values ('" + url + "', '" + seedType + "')";
daoHelp.execSql(sql);
this.updResourceInfo(url, site, type, city);
}
public void setSpidered(String url, URLSpideredProxyOkEvent event) {
String sql =
"update spider_resource set saved = 1, httpStatus = "
+ daoHelp.format(event.getHttpStatus())
+ ", "
+ " dTime = "
+ daoHelp.format(new Date().toLocaleString())
+ " where url = "
+ daoHelp.format(url)
+ "";
daoHelp.execSql(sql);
}
public void setSpiderError(URLSpideredProxyErrorEvent event) {
String sql =
"update spider_resource set saved = 0, httpStatus = "
+ event.getHttpStatus()
+ ", "
+ " dTime = '"
+ new Date().toLocaleString()
+ "' where url = '"
+ event.getUrl()
+ "'";
daoHelp.execSql(sql);
}
public void createSeed(SeedInfo seedInfo) {
String sql =
"insert into spider_resource(url, type, kid) values ('"
+ seedInfo.getUrl()
+ "', '"
+ seedInfo.getSeedType()
+ "', "
+ seedInfo.getKid()
+ ")";
daoHelp.execSql(sql);
this.updResourceInfo(
seedInfo.getUrl(), seedInfo.getSite(), seedInfo.getType(), seedInfo.getCity());
}
private void updResourceInfo(String url, String site, String type, int city) {
int rid = 0;
Statement stmt = null;
String sql = "select rid from spider_resource where url = '" + url + "'";
ResultSet rs = null;
try {
stmt = daoHelp.getConnection().createStatement();
rs = stmt.executeQuery(sql);
while (rs.next()) {
rid = rs.getInt("rid");
}
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} catch (SQLException e) {
log.error("SQLException -> ", e);
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
} finally {
daoHelp.safeClose(rs);
daoHelp.safeClose(stmt);
}
String uSql = "update spider_resource set paid = " + rid + " where rid = " + rid;
daoHelp.execSql(uSql);
String infoSql =
"insert into spider_resource_info(rid, site, iType, city ) "
+ " values ("
+ ""
+ rid
+ ","
+ daoHelp.format(site)
+ ", "
+ daoHelp.format(type)
+ ", "
+ daoHelp.format(city)
+ ")";
daoHelp.execSql(infoSql);
}
public void create(CheckInfo info) {
StringBuffer sql = new StringBuffer();
sql.append("insert into spider_check (sid, pid, field, url, cType, site, sType, content");
sql.append(" ) values ( ");
sql.append(daoHelp.format(info.getSid()) + ",");
sql.append(daoHelp.format(info.getPid()) + ",");
sql.append(daoHelp.format(info.getField()) + ",");
sql.append(daoHelp.format(info.getUrl()) + ",");
sql.append(daoHelp.format(info.getCType()) + ",");
sql.append(daoHelp.format(info.getSite()) + ",");
sql.append(daoHelp.format(info.getSType()) + ",");
sql.append(daoHelp.format(info.getContent()) + ")");
daoHelp.execSql(sql.toString());
}
public boolean create(Resource resource) {
if (this.isHavingUrl(resource.getUrl())) return false;
StringBuffer sql = new StringBuffer();
sql.append("insert into spider_resource( ");
sql.append("paId,");
sql.append("url,");
sql.append("type,");
sql.append("httpStatus,");
sql.append("dTime");
sql.append(" ) values ( ");
sql.append(daoHelp.format(resource.getPaId()) + ",");
sql.append(daoHelp.format(resource.getUrl()) + ",");
sql.append(daoHelp.format(resource.getType()) + ",");
sql.append("0,");
sql.append("''");
sql.append(" ) ");
daoHelp.execSql(sql.toString());
return true;
}
public List<Resource> setSpider() {
String sql =
"update spider_resource set crawl = 0 , saved = 0, httpStatus = NULL , dTime = 0 where saved = 0 and crawl = 1 and (HttpStatus is NULL or HttpStatus < 100 ) ";
daoHelp.execSql(sql);
return this.getSomeSeedResource();
}
public void addRequsetCount(int rid, int reqCount) {
String sql = "update spider_resource set reqCount = " + (++reqCount) + " where rid = " + rid;
daoHelp.execSql(sql);
}
public void setSeedNotSpider() {
String sql =
"update spider_resource set saved = 0, crawl = 0, HttpStatus = 0, reqCount = 0 where (type = 'monitor' or type = 'seed') and httpStatus <= 301";
daoHelp.execSql(sql);
}
public void setStartTime() {
SimpleDateFormat dateformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String sDate = dateformat.format(new Date());
String sql = "update spider_info set stime = '" + sDate + "'";
daoHelp.execSql(sql);
}
public void deleteAll() {
String sql = "delete from spider_resource";
daoHelp.execSql(sql);
}
public void setParsered(Resource res) {
String sql = "update spider_resource set parser = 1 where rid = " + res.getRId();
daoHelp.execSql(sql);
}
public void setParsered(ResourceParsedOkEvent event) {
String sql = "update spider_resource set parser = 1 where url = '" + event.getUrl() + "'";
daoHelp.execSql(sql);
}