コード例 #1
0
 protected Set<String> filesInArcs() throws IOException {
   List<ArchiveRecordHeader> headers = headersInArcs();
   HashSet<String> result = new HashSet<String>();
   for (ArchiveRecordHeader arh : headers) {
     // ignore 'filedesc:' record
     if (arh.getUrl().startsWith("filedesc:")) {
       continue;
     }
     UURI uuri = UURIFactory.getInstance(arh.getUrl());
     String path = uuri.getPath();
     if (path.startsWith("/")) {
       path = path.substring(1);
     }
     if (arh.getUrl().startsWith("http:")) {
       result.add(path);
     }
   }
   LOGGER.finest(result.toString());
   return result;
 }
コード例 #2
0
 public boolean isPrerequisite(final CrawlURI curi) {
   boolean result = false;
   String curiStr = curi.getUURI().toString();
   String loginUri = getPrerequisite(curi);
   if (loginUri != null) {
     try {
       UURI uuri = UURIFactory.getInstance(curi.getUURI(), loginUri);
       if (uuri != null && curiStr != null && uuri.toString().equals(curiStr)) {
         result = true;
         if (!curi.isPrerequisite()) {
           curi.setPrerequisite(true);
           logger.fine(curi + " is prereq.");
         }
       }
     } catch (URIException e) {
       logger.severe("Failed to uuri: " + curi + ", " + e.getMessage());
     }
   }
   return result;
 }
コード例 #3
0
ファイル: ParseHTTP.java プロジェクト: yangdayuan/spider-2
  protected void addHeaderLink(CrawlURI curi, Header loc) {
    if (loc == null) {
      // If null, return without adding anything.
      return;
    }
    // TODO: consider possibility of multiple headers
    try {
      /**
       * 302重定向使用自定义的方法存储link
       *
       * @modify: wuliufu
       * @since : 2012-05-11
       */
      curi.createAndAddLocationLink(
          curi.getVia(), loc.getValue(), loc.getName() + ":", Link.REFER_HOP);

      if (curi.getObject(URLInfo.ATTACH) != null) {
        UURI outUURI = UURIFactory.getInstance(curi.getUURI(), loc.getValue());
        logger.debug(
            "ParseHTTP: curi = "
                + curi.getUURI().toString()
                + "&& "
                + loc.getName()
                + "="
                + outUURI.toString());
        curi.putObject(outUURI.toString(), curi.getObject(URLInfo.ATTACH));
      }

      numberOfLinksExtracted++;
    } catch (URIException e) {
      // There may not be a controller (e.g. If we're being run
      // by the extractor tool).
      if (getController() != null) {
        getController().logUriError(e, curi.getUURI(), loc.getValue());
      } else {
        logger.info(curi + ", " + loc.getValue() + ": " + e.getMessage());
      }
    }
  }
コード例 #4
0
 /**
  * Add constant penalties for certain features of URI (and its 'via') that make it more
  * delayable/skippable.
  *
  * @param curi CrawlURI to be assigned a cost
  * @see org.archive.crawler.frontier.CostAssignmentPolicy#costOf(org.archive.modules.CrawlURI)
  */
 public int costOf(CrawlURI curi) {
   int cost = 1;
   UURI uuri = curi.getUURI();
   if (uuri.hasQuery()) {
     // has query string
     cost++;
     int qIndex = uuri.toString().indexOf('?');
     if (curi.flattenVia().startsWith(uuri.toString().substring(0, qIndex))) {
       // non-query-string portion of URI is same as previous
       cost++;
     }
     // TODO: other potential query-related cost penalties:
     //  - more than X query-string attributes
     //  - calendarish terms
     //  - query-string over certain size
   }
   // TODO: other potential path-based penalties
   //  - new path is simply extension of via path
   //  - many path segments
   // TODO: other potential hops-based penalties
   //  - more than X hops
   //  - each speculative hop
   return cost;
 }
コード例 #5
0
 /* (non-Javadoc)
  * @see org.archive.crawler.settings.refinements.Criteria#isWithinRefinementBounds(org.archive.crawler.datamodel.UURI, int)
  */
 public boolean isWithinRefinementBounds(UURI uri) {
   return (uri == null || uri == null) ? false : TextUtils.matches(regexp, uri.toString());
 }