예제 #1
0
 private void doCrawl(boolean force, boolean deep) {
   ArchivalUnit au = getAu();
   if (au == null) return;
   try {
     startCrawl(au, force, deep);
   } catch (CrawlManagerImpl.NotEligibleException.RateLimiter e) {
     errMsg = "AU has crawled recently (" + e.getMessage() + ").  Click again to override.";
     showForceCrawl = true;
     return;
   } catch (CrawlManagerImpl.NotEligibleException e) {
     errMsg = "Can't enqueue crawl: " + e.getMessage();
   }
 }
예제 #2
0
 private void crawlPluginRegistries() {
   StringBuilder sb = new StringBuilder();
   for (ArchivalUnit au : pluginMgr.getAllRegistryAus()) {
     sb.append(au.getName());
     sb.append(": ");
     try {
       startCrawl(au, true, false);
       sb.append("Queued.");
     } catch (CrawlManagerImpl.NotEligibleException e) {
       sb.append("Failed: ");
       sb.append(e.getMessage());
     }
     sb.append("\n");
   }
   statusMsg = sb.toString();
 }
예제 #3
0
  private boolean startCrawl(ArchivalUnit au, boolean force, boolean deep)
      throws CrawlManagerImpl.NotEligibleException {
    CrawlManagerImpl cmi = (CrawlManagerImpl) crawlMgr;
    if (force) {
      RateLimiter limit = cmi.getNewContentRateLimiter(au);
      if (!limit.isEventOk()) {
        limit.unevent();
      }
    }
    cmi.checkEligibleToQueueNewContentCrawl(au);
    String delayMsg = "";
    String deepMsg = "";
    try {
      cmi.checkEligibleForNewContentCrawl(au);
    } catch (CrawlManagerImpl.NotEligibleException e) {
      delayMsg = ", Start delayed due to: " + e.getMessage();
    }
    Configuration config = ConfigManager.getCurrentConfig();
    int pri = config.getInt(PARAM_CRAWL_PRIORITY, DEFAULT_CRAWL_PRIORITY);

    CrawlReq req;
    try {
      req = new CrawlReq(au);
      req.setPriority(pri);
      if (deep) {
        int d = Integer.parseInt(formDepth);
        if (d < 0) {
          errMsg = "Illegal refetch depth: " + d;
          return false;
        }
        req.setRefetchDepth(d);
        deepMsg = "Deep (" + req.getRefetchDepth() + ") ";
      }
    } catch (NumberFormatException e) {
      errMsg = "Illegal refetch depth: " + formDepth;
      return false;
    } catch (RuntimeException e) {
      log.error("Couldn't create CrawlReq: " + au, e);
      errMsg = "Couldn't create CrawlReq: " + e.toString();
      return false;
    }
    cmi.startNewContentCrawl(req, null);
    statusMsg = deepMsg + "Crawl requested for " + au.getName() + delayMsg;
    return true;
  }