コード例 #1
0
 protected long deleteMatchingFromQueue(final WorkQueueFrontier frontier, final String match)
     throws IOException {
   try {
     final BdbMultipleWorkQueues queues = ((BdbFrontier) frontier).getWorkQueues();
     return queues.deleteMatchingFromQueue(match, classKey, new DatabaseEntry(origin));
   } catch (DatabaseException e) {
     throw IoUtils.wrapAsIOException(e);
   }
 }
コード例 #2
0
 protected void deleteItem(final WorkQueueFrontier frontier, final CrawlURI peekItem)
     throws IOException {
   try {
     final BdbMultipleWorkQueues queues = ((BdbFrontier) frontier).getWorkQueues();
     queues.delete(peekItem);
   } catch (DatabaseException e) {
     e.printStackTrace();
     throw IoUtils.wrapAsIOException(e);
   }
 }
コード例 #3
0
  protected CrawlURI peekItem(final WorkQueueFrontier frontier) throws IOException {
    final BdbMultipleWorkQueues queues = ((BdbFrontier) frontier).getWorkQueues();
    DatabaseEntry key = new DatabaseEntry(origin);
    CrawlURI curi = null;
    int tries = 1;
    while (true) {
      try {
        curi = queues.get(key);
      } catch (DatabaseException e) {
        LOGGER.log(Level.SEVERE, "peekItem failure; retrying", e);
      }

      // ensure CrawlURI, if any,  came from acceptable range:
      if (!ArchiveUtils.startsWith(key.getData(), origin)) {
        LOGGER.severe(
            "inconsistency: "
                + classKey
                + "("
                + getPrefixClassKey(origin)
                + ") with "
                + getCount()
                + " items gave "
                + curi
                + "("
                + getPrefixClassKey(key.getData()));
        // clear curi to allow retry
        curi = null;
        // reset key to original origin for retry
        key.setData(origin);
      }

      if (curi != null) {
        // success
        break;
      }

      if (tries > 3) {
        LOGGER.severe("no item where expected in queue " + classKey);
        break;
      }
      tries++;
      LOGGER.severe(
          "Trying get #"
              + Integer.toString(tries)
              + " in queue "
              + classKey
              + " with "
              + getCount()
              + " items using key "
              + getPrefixClassKey(key.getData()));
    }

    return curi;
  }
コード例 #4
0
 /**
  * Create a virtual queue inside the given BdbMultipleWorkQueues
  *
  * @param classKey
  */
 public BdbWorkQueue(String classKey, BdbFrontier frontier) {
   super(classKey);
   this.origin = BdbMultipleWorkQueues.calculateOriginKey(classKey);
   if (LOGGER.isLoggable(Level.FINE)) {
     LOGGER.fine(getPrefixClassKey(this.origin) + " " + classKey);
   }
   // add the queue-front 'cap' entry; see...
   // http://sourceforge.net/tracker/index.php?func=detail&aid=1262665&group_id=73833&atid=539102
   frontier.getWorkQueues().addCap(origin);
 }
コード例 #5
0
 protected void insertItem(
     final WorkQueueFrontier frontier, final CrawlURI curi, boolean overwriteIfPresent)
     throws IOException {
   try {
     final BdbMultipleWorkQueues queues = ((BdbFrontier) frontier).getWorkQueues();
     queues.put(curi, overwriteIfPresent);
     if (LOGGER.isLoggable(Level.FINE)) {
       LOGGER.fine(
           "Inserted into "
               + getPrefixClassKey(this.origin)
               + " (count "
               + Long.toString(getCount())
               + "): "
               + curi.toString());
     }
   } catch (DatabaseException e) {
     throw IoUtils.wrapAsIOException(e);
   }
 }