Пример #1
0
  private void seriousError(Error err) {
    // try to prevent timeslicing until we have a chance to deal with OOM
    // Note that modern-day JVM priority indifference with native threads
    // may make this priority-jumbling pointless
    setPriority(DEFAULT_PRIORITY + 1);
    if (controller != null) {
      // hold all ToeThreads from proceeding to next processor
      controller.freeReserveMemory();
      controller.requestCrawlPause();
      if (controller.getFrontier().getFrontierJournal() != null) {
        controller.getFrontier().getFrontierJournal().seriousError(getName() + err.getMessage());
      }
    }

    // OutOfMemory etc.
    String extraInfo = DevUtils.extraInfo();
    System.err.println("<<<");
    System.err.println(ArchiveUtils.getLog17Date());
    System.err.println(err);
    System.err.println(extraInfo);
    err.printStackTrace(System.err);

    if (controller != null) {
      PrintWriter pw = new PrintWriter(System.err);
      controller.getToePool().compactReportTo(pw);
      pw.flush();
    }
    System.err.println(">>>");
    //        DevUtils.sigquitSelf();

    String context = "unknown";
    if (currentCuri != null) {
      // update fetch-status, saving original as annotation
      currentCuri.getAnnotations().add("err=" + err.getClass().getName());
      currentCuri.getAnnotations().add("os" + currentCuri.getFetchStatus());
      currentCuri.setFetchStatus(S_SERIOUS_ERROR);
      context = currentCuri.shortReportLine() + " in " + currentProcessorName;
    }
    String message = "Serious error occured trying " + "to process '" + context + "'\n" + extraInfo;
    logger.log(Level.SEVERE, message.toString(), err);
    setPriority(DEFAULT_PRIORITY);
  }
  /**
   * @param m marker or null to start with first entry
   * @param maxMatches
   * @return list of matches starting from marker position
   * @throws DatabaseException
   */
  public CompositeData getFrom(String m, int maxMatches, Pattern pattern, boolean verbose)
      throws DatabaseException {
    int matches = 0;
    int tries = 0;
    ArrayList<String> results = new ArrayList<String>(maxMatches);

    DatabaseEntry key;
    if (m == null) {
      key = getFirstKey();
    } else {
      byte[] marker = m.getBytes(); // = FrontierJMXTypes.fromString(m);
      key = new DatabaseEntry(marker);
    }

    DatabaseEntry value = new DatabaseEntry();

    Cursor cursor = null;
    OperationStatus result = null;
    try {
      cursor = pendingUrisDB.openCursor(null, null);
      result = cursor.getSearchKey(key, value, null);

      while (matches < maxMatches && result == OperationStatus.SUCCESS) {
        if (value.getData().length > 0) {
          CrawlURI curi = (CrawlURI) crawlUriBinding.entryToObject(value);
          if (pattern.matcher(curi.toString()).matches()) {
            if (verbose) {
              results.add("[" + curi.getClassKey() + "] " + curi.shortReportLine());
            } else {
              results.add(curi.toString());
            }
            matches++;
          }
          tries++;
        }
        result = cursor.getNext(key, value, null);
      }
    } finally {
      if (cursor != null) {
        cursor.close();
      }
    }

    if (result != OperationStatus.SUCCESS) {
      // end of scan
      m = null;
    } else {
      m = new String(key.getData()); // = FrontierJMXTypes.toString(key.getData());
    }

    String[] arr = results.toArray(new String[results.size()]);
    CompositeData cd;
    try {
      cd =
          new CompositeDataSupport(
              /*FrontierJMXTypes.URI_LIST_DATA*/ null,
              new String[] {"list", "marker"},
              new Object[] {arr, m});
    } catch (OpenDataException e) {
      throw new IllegalStateException(e);
    }
    return cd;
  }