public static void start(List<Fetcher> fetchers) {
try {
for (Fetcher fetcher : fetchers) {
Fetch fetch = fetcher.getFetch();
if (fetch == null) {
continue;
}
Link link = new Link(fetch.getTitle(), fetch.getUrl());
link.setRoot(fetch.getUrl());
fetch(fetcher.createRequest(), fetcher.getRegexRule(), link, 10000);
EventInst.getInstance().getAsyncEventBus().post(fetcher);
}
} catch (Exception ex) {
Logger.getLogger(FetchUtils.class.getName()).log(Level.SEVERE, null, ex);
}
}
public static void fetch(HttpRequest request, RegexRule regexRule, Link link, int limit) {
try {
HttpResponse response = request.getResponse(link);
Raw raw = ParseUtils.getRaw(response, regexRule, limit);
Transaction ts = new Transaction();
LinkWriter linkWriter = new LinkWriter(ts);
LinkReader linkReader = new LinkReader();
for (Link childLink : raw.getLinks()) {
childLink.setDepth(link.getDepth() + 1);
childLink.setCreateDate(new Date(System.currentTimeMillis()));
childLink.setExpired(false);
childLink.setFetched(false);
childLink.setId(UuidUtils.base58Uuid());
childLink.setRoot(link.getRoot());
if (childLink.getDepth() < request.getDepth()
&& linkReader.countHash(childLink.getHash()) == 0) {
linkWriter.add(childLink);
}
}
// 保存内容啊。。
ContentWriter contentWriter = new ContentWriter(ts);
contentWriter.add(raw.getContent());
DescriptionWriter descriptionWriter = new DescriptionWriter(ts);
descriptionWriter.add(raw.getDescription());
ts.save();
} catch (Exception ex) {
Logger.getLogger(FetchUtils.class.getName()).log(Level.SEVERE, null, ex);
}
}
