Example #1
0
  public DecideResult innerDecide(CrawlURI uri) {
    DecideRule decisiveRule = null;
    int decisiveRuleNumber = -1;
    DecideResult result = DecideResult.NONE;
    List<DecideRule> rules = getRules();
    int max = rules.size();

    for (int i = 0; i < max; i++) {
      DecideRule rule = rules.get(i);
      if (rule.onlyDecision(uri) != result) {
        DecideResult r = rule.decisionFor(uri);
        if (LOGGER.isLoggable(Level.FINEST)) {
          LOGGER.finest(
              "DecideRule #"
                  + i
                  + " "
                  + rule.getClass().getName()
                  + " returned "
                  + r
                  + " for url: "
                  + uri);
        }
        if (r != DecideResult.NONE) {
          result = r;
          decisiveRule = rule;
          decisiveRuleNumber = i;
        }
      }
    }

    decisionMade(uri, decisiveRule, decisiveRuleNumber, result);

    return result;
  }
Example #2
0
  protected void decisionMade(
      CrawlURI uri, DecideRule decisiveRule, int decisiveRuleNumber, DecideResult result) {
    if (fileLogger != null) {
      JSONObject extraInfo = null;
      if (logExtraInfo) {
        CrawlHost crawlHost = getServerCache().getHostFor(uri.getUURI());
        String host = "-";
        if (crawlHost != null) {
          host = crawlHost.fixUpName();
        }

        extraInfo = new JSONObject();
        extraInfo.put("hopPath", uri.getPathFromSeed());
        extraInfo.put("via", uri.getVia());
        extraInfo.put("seed", uri.getSourceTag());
        extraInfo.put("host", host);
      }

      fileLogger.info(
          decisiveRuleNumber
              + " "
              + decisiveRule.getClass().getSimpleName()
              + " "
              + result
              + " "
              + uri
              + (extraInfo != null ? " " + extraInfo : ""));
    }
  }