Java RobotDirective.isSpecial示例

示例#1

文件： SimpleRobotRulesParser.java 项目： alei76/crawler-commons

  static {
    for (RobotDirective directive : RobotDirective.values()) {
      if (!directive.isSpecial()) {
        String prefix = directive.name().toLowerCase().replaceAll("_", "-");
        DIRECTIVE_PREFIX.put(prefix, directive);
      }
    }

    DIRECTIVE_PREFIX.put("useragent", RobotDirective.USER_AGENT);
    DIRECTIVE_PREFIX.put("useg-agent", RobotDirective.USER_AGENT);
    DIRECTIVE_PREFIX.put("ser-agent", RobotDirective.USER_AGENT);

    DIRECTIVE_PREFIX.put("desallow", RobotDirective.DISALLOW);
    DIRECTIVE_PREFIX.put("dissalow", RobotDirective.DISALLOW);
    DIRECTIVE_PREFIX.put("dssalow", RobotDirective.DISALLOW);
    DIRECTIVE_PREFIX.put("dsallow", RobotDirective.DISALLOW);

    DIRECTIVE_PREFIX.put("crawl delay", RobotDirective.CRAWL_DELAY);
  }