/**
   * Expunge an entry from memMap while updating diskMap.
   *
   * @param entry a SoftEntry<V> obtained from refQueuePoll()
   */
  private synchronized void pageOutStaleEntry(SoftEntry<V> entry) {
    PhantomEntry<V> phantom = entry.phantom;

    // Still in memMap? if not, was paged-out by earlier direct access
    // before placed into reference-queue; just return
    if (memMap.get(phantom.key) != entry) { // NOTE: intentional identity compare
      return;
    }

    // recover hidden value
    V phantomValue = phantom.doctoredGet();

    // Expected value present? (should be; only clear is at end of
    // this method, after entry removal from memMap)
    if (phantomValue == null) {
      logger.log(Level.WARNING, "unexpected null phantomValue", new Exception());
      return; // nothing to do
    }

    // given instance entry still in memMap;
    // we have the key and phantom Value,
    // the diskMap can be updated.
    diskMap.put(phantom.key, phantomValue); // unchecked cast
    expungeStatsDiskPut.incrementAndGet();

    //  remove memMap entry
    boolean removed = memMap.remove(phantom.key, entry);
    if (!removed) {
      logger.log(Level.WARNING, "expunge memMap.remove() ineffective", new Exception());
    }
    phantom.clear(); // truly allows GC of unreferenced V object
  }
  /**
   * Copies entries from an existing environment db to a new one. If historyMap is not provided,
   * only logs the entries that would have been copied.
   *
   * @param sourceDir existing environment database directory
   * @param historyMap new environment db (or null for a dry run)
   * @return number of records
   * @throws DatabaseException
   */
  private static int copyPersistEnv(File sourceDir, StoredSortedMap<String, Map> historyMap)
      throws DatabaseException {
    int count = 0;

    // open the source env history DB, copying entries to target env
    EnhancedEnvironment sourceEnv = setupCopyEnvironment(sourceDir, true);
    StoredClassCatalog sourceClassCatalog = sourceEnv.getClassCatalog();
    DatabaseConfig historyDbConfig = HISTORY_DB_CONFIG.toDatabaseConfig();
    historyDbConfig.setReadOnly(true);
    Database sourceHistoryDB = sourceEnv.openDatabase(null, URI_HISTORY_DBNAME, historyDbConfig);
    StoredSortedMap<String, Map> sourceHistoryMap =
        new StoredSortedMap<String, Map>(
            sourceHistoryDB,
            new StringBinding(),
            new SerialBinding<Map>(sourceClassCatalog, Map.class),
            true);

    Iterator<Entry<String, Map>> iter = sourceHistoryMap.entrySet().iterator();
    while (iter.hasNext()) {
      Entry<String, Map> item = iter.next();
      if (logger.isLoggable(Level.FINE)) {
        logger.fine(item.getKey() + " " + new JSONObject(item.getValue()));
      }

      if (historyMap != null) {
        historyMap.put(item.getKey(), item.getValue());
      }
      count++;
    }
    StoredIterator.close(iter);
    sourceHistoryDB.close();
    sourceEnv.close();

    return count;
  }
  /**
   * Populates an environment db from a persist log. If historyMap is not provided, only logs the
   * entries that would have been populated.
   *
   * @param persistLogReader persist log
   * @param historyMap new environment db (or null for a dry run)
   * @return number of records
   * @throws UnsupportedEncodingException
   * @throws DatabaseException
   */
  private static int populatePersistEnvFromLog(
      BufferedReader persistLogReader, StoredSortedMap<String, Map> historyMap)
      throws UnsupportedEncodingException, DatabaseException {
    int count = 0;

    Iterator<String> iter = new LineReadingIterator(persistLogReader);
    while (iter.hasNext()) {
      String line = iter.next();
      if (line.length() == 0) {
        continue;
      }
      String[] splits = line.split(" ");
      if (splits.length != 2) {
        logger.severe("bad line has " + splits.length + " fields (should be 2): " + line);
        continue;
      }

      Map alist;
      try {
        alist =
            (Map) SerializationUtils.deserialize(Base64.decodeBase64(splits[1].getBytes("UTF-8")));
      } catch (Exception e) {
        logger.severe("caught exception " + e + " deserializing line: " + line);
        continue;
      }

      if (logger.isLoggable(Level.FINE)) {
        logger.fine(splits[0] + " " + ArchiveUtils.prettyString(alist));
      }

      if (historyMap != null)
        try {
          historyMap.put(splits[0], alist);
        } catch (Exception e) {
          logger.log(
              Level.SEVERE,
              "caught exception after loading "
                  + count
                  + " urls from the persist log (perhaps crawl was stopped by user?)",
              e);
          IOUtils.closeQuietly(persistLogReader);

          // seems to finish most cleanly when we return rather than throw something
          return count;
        }

      count++;
    }
    IOUtils.closeQuietly(persistLogReader);

    return count;
  }