コード例 #1
0
  private void requestNextLinks(LevelLink levelLink) throws InterruptedException {

    int currentLevel = levelLink.getLevel() + 1;

    String ch = "";
    for (int i = 0; i < currentLevel; i++) {
      ch = ch + "--";
    }

    if (currentLevel >= maxDeep) {
      return;
    }

    randomPause();

    Elements links = getLinks(levelLink.getLink());

    if (links == null) {
      return;
    }

    LOG.log(
        Level.DEBUG,
        " "
            + ch
            + " "
            + levelLink.getLink()
            + " ||| Thread: "
            + Thread.currentThread().getName()
            + " ||| level: "
            + currentLevel
            + " newLinks="
            + links.size());

    // get all ref on a page
    for (Element element : links) {
      URL url = null;
      try {
        url = new URL(element.attr("abs:href"));
      } catch (MalformedURLException e) {
        LOG.log(Level.INFO, "Exception: ", e);
      }
      String newURI = Utils.toFullForm(url, false);
      if (newURI == null) {
        continue;
      }
      putLink(new LevelLink(newURI, currentLevel));
    }

    LevelLink nextLevelLink = readNextLink();
    if (nextLevelLink != null) {
      requestNextLinks(nextLevelLink);
    }
  }
コード例 #2
0
  public synchronized void putLink(LevelLink levelLink) throws InterruptedException {

    if (levelLink == null) return;

    if (!fullLinks.contains(levelLink.getLink())) {
      queue.offer(levelLink, 10, TimeUnit.MILLISECONDS);
    }
  }
コード例 #3
0
 private synchronized void putResultLink(LevelLink resultLink) throws InterruptedException {
   if (!fullLinks.contains(resultLink.getLink())) {
     resultLinks.offer(resultLink.getLink(), 10, TimeUnit.MILLISECONDS);
     fullLinks.add(resultLink.getLink());
   }
 }