Ejemplo n.º 1
0
  public static void start(List<Fetcher> fetchers) {

    try {

      for (Fetcher fetcher : fetchers) {
        Fetch fetch = fetcher.getFetch();
        if (fetch == null) {
          continue;
        }
        Link link = new Link(fetch.getTitle(), fetch.getUrl());
        link.setRoot(fetch.getUrl());
        fetch(fetcher.createRequest(), fetcher.getRegexRule(), link, 10000);
        EventInst.getInstance().getAsyncEventBus().post(fetcher);
      }

    } catch (Exception ex) {
      Logger.getLogger(FetchUtils.class.getName()).log(Level.SEVERE, null, ex);
    }
  }
Ejemplo n.º 2
0
  public static void fetch(HttpRequest request, RegexRule regexRule, Link link, int limit) {
    try {

      HttpResponse response = request.getResponse(link);
      Raw raw = ParseUtils.getRaw(response, regexRule, limit);
      Transaction ts = new Transaction();
      LinkWriter linkWriter = new LinkWriter(ts);
      LinkReader linkReader = new LinkReader();
      for (Link childLink : raw.getLinks()) {
        childLink.setDepth(link.getDepth() + 1);
        childLink.setCreateDate(new Date(System.currentTimeMillis()));
        childLink.setExpired(false);
        childLink.setFetched(false);
        childLink.setId(UuidUtils.base58Uuid());
        childLink.setRoot(link.getRoot());

        if (childLink.getDepth() < request.getDepth()
            && linkReader.countHash(childLink.getHash()) == 0) {
          linkWriter.add(childLink);
        }
      }
      // 保存内容啊。。
      ContentWriter contentWriter = new ContentWriter(ts);
      contentWriter.add(raw.getContent());
      DescriptionWriter descriptionWriter = new DescriptionWriter(ts);
      descriptionWriter.add(raw.getDescription());
      ts.save();

    } catch (Exception ex) {
      Logger.getLogger(FetchUtils.class.getName()).log(Level.SEVERE, null, ex);
    }
  }