Beispiel #1
0
    @Override
    protected void done() {
      try {
        super.get(); // block and get all exceptions thrown while doInBackground()
        // notify modules of completion
        if (!this.isCancelled()) {
          for (IngestModuleAbstractFile s : abstractFileModules) {
            s.complete();
            IngestManager.fireModuleEvent(IngestModuleEvent.COMPLETED.toString(), s.getName());
          }
        }

        logger.log(Level.INFO, PlatformUtil.getAllMemUsageInfo());
        logger.log(Level.INFO, "Freeing jvm heap resources post file pipeline run");
        System.gc();
        logger.log(Level.INFO, PlatformUtil.getAllMemUsageInfo());

      } catch (CancellationException e) {
        // task was cancelled
        handleInterruption();

      } catch (InterruptedException ex) {
        handleInterruption();
      } catch (ExecutionException ex) {
        handleInterruption();
        logger.log(Level.SEVERE, "Fatal error during ingest.", ex);

      } catch (Exception ex) {
        handleInterruption();
        logger.log(Level.SEVERE, "Fatal error during ingest.", ex);
      } finally {
        stats.end();
        progress.finish();

        if (!this.isCancelled()) {
          logger.log(Level.INFO, "Summary Report: " + stats.toString());
          logger.log(Level.INFO, "File module timings: " + stats.getFileModuleStats());
          if (ui != null) {
            logger.log(Level.INFO, "Ingest messages count: " + ui.getMessagesCount());
          }

          IngestManager.this.postMessage(
              IngestMessage.createManagerMessage("File Ingest Complete", stats.toHtmlString()));
        }
      }
    }
Beispiel #2
0
  // case change helper
  private static void doCaseChange(Case toChangeTo) {
    logger.log(Level.INFO, "Changing Case to: " + toChangeTo);
    if (toChangeTo != null) { // new case is open

      // clear the temp folder when the case is created / opened
      Case.clearTempFolder();
      checkSubFolders(toChangeTo);

      // enable these menus
      CallableSystemAction.get(AddImageAction.class).setEnabled(true);
      CallableSystemAction.get(CaseCloseAction.class).setEnabled(true);
      CallableSystemAction.get(CasePropertiesAction.class).setEnabled(true);
      CallableSystemAction.get(CaseDeleteAction.class).setEnabled(true); // Delete Case menu

      if (toChangeTo.getRootObjectsCount() > 0) {
        // open all top components
        CoreComponentControl.openCoreWindows();
      } else {
        // close all top components
        CoreComponentControl.closeCoreWindows();
      }
    } else { // case is closed
      // close all top components first
      CoreComponentControl.closeCoreWindows();

      // disable these menus
      CallableSystemAction.get(AddImageAction.class).setEnabled(false); // Add Image menu
      CallableSystemAction.get(CaseCloseAction.class).setEnabled(false); // Case Close menu
      CallableSystemAction.get(CasePropertiesAction.class)
          .setEnabled(false); // Case Properties menu
      CallableSystemAction.get(CaseDeleteAction.class).setEnabled(false); // Delete Case menu

      // clear pending notifications
      MessageNotifyUtil.Notify.clear();

      Frame f = WindowManager.getDefault().getMainWindow();
      f.setTitle(Case.getAppName()); // set the window name to just application name

      // try to force gc to happen
      System.gc();
      System.gc();
    }

    // log memory usage after case changed
    logger.log(Level.INFO, PlatformUtil.getAllMemUsageInfo());
  }
Beispiel #3
0
    @Override
    protected Object doInBackground() throws Exception {

      logger.log(Level.INFO, "Starting background ingest file processor");
      logger.log(Level.INFO, PlatformUtil.getAllMemUsageInfo());

      stats.start();

      // notify main thread modules started
      for (IngestModuleAbstractFile s : abstractFileModules) {
        IngestManager.fireModuleEvent(IngestModuleEvent.STARTED.toString(), s.getName());
      }

      final String displayName = "File Ingest";
      progress =
          ProgressHandleFactory.createHandle(
              displayName,
              new Cancellable() {
                @Override
                public boolean cancel() {
                  logger.log(Level.INFO, "Filed ingest cancelled by user.");
                  if (progress != null) {
                    progress.setDisplayName(displayName + " (Cancelling...)");
                  }
                  return IngestAbstractFileProcessor.this.cancel(true);
                }
              });

      final IngestScheduler.FileScheduler fileScheduler = scheduler.getFileScheduler();

      // initialize the progress bar
      progress.start();
      progress.switchToIndeterminate();
      // set initial totals and processed (to be updated as we process or new files are scheduled)
      int totalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
      progress.switchToDeterminate(totalEnqueuedFiles);
      int processedFiles = 0;
      // process AbstractFiles queue
      while (fileScheduler.hasNext()) {
        final ProcessTask fileTask = fileScheduler.next();
        final PipelineContext<IngestModuleAbstractFile> filepipelineContext = fileTask.context;
        final ScheduledTask<IngestModuleAbstractFile> fileIngestTask =
            filepipelineContext.getScheduledTask();
        final AbstractFile fileToProcess = fileTask.file;

        // clear return values from modules for last file
        synchronized (abstractFileModulesRetValues) {
          abstractFileModulesRetValues.clear();
        }

        // logger.log(Level.INFO, "IngestManager: Processing: {0}", fileToProcess.getName());

        for (IngestModuleAbstractFile module : fileIngestTask.getModules()) {
          // process the file with every file module
          if (isCancelled()) {
            logger.log(Level.INFO, "Terminating file ingest due to cancellation.");
            return null;
          }
          progress.progress(
              fileToProcess.getName() + " (" + module.getName() + ")", processedFiles);

          try {
            stats.logFileModuleStartProcess(module);
            IngestModuleAbstractFile.ProcessResult result =
                module.process(filepipelineContext, fileToProcess);
            stats.logFileModuleEndProcess(module);

            // store the result for subsequent modules for this file
            synchronized (abstractFileModulesRetValues) {
              abstractFileModulesRetValues.put(module.getName(), result);
            }

          } catch (Exception e) {
            logger.log(
                Level.SEVERE, "Error: unexpected exception from module: " + module.getName(), e);
            stats.addError(module);
          } catch (OutOfMemoryError e) {
            logger.log(Level.SEVERE, "Error: out of memory from module: " + module.getName(), e);
            stats.addError(module);
          }
        } // end for every module

        // free the internal file resource after done with every module
        fileToProcess.close();

        int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst();
        if (newTotalEnqueuedFiles > totalEnqueuedFiles) {
          // update if new enqueued
          totalEnqueuedFiles = newTotalEnqueuedFiles + 1; // + processedFiles + 1;
          // processedFiles = 0;
          // reset
          progress.switchToIndeterminate();
          progress.switchToDeterminate(totalEnqueuedFiles);
        }
        if (processedFiles
            < totalEnqueuedFiles) { // fix for now to handle the same datasource Content enqueued
                                    // twice
          ++processedFiles;
        }
        // --totalEnqueuedFiles;

      } // end of for every AbstractFile
      logger.log(Level.INFO, "IngestManager: Finished processing files");
      return null;
    }
Beispiel #4
0
/**
 * This class implements a singleton that manages the set of hash databases used to classify files
 * as unknown, known or known bad.
 */
public class HashDbManager implements PropertyChangeListener {

  private static final String ROOT_ELEMENT = "hash_sets"; // NON-NLS
  private static final String SET_ELEMENT = "hash_set"; // NON-NLS
  private static final String SET_NAME_ATTRIBUTE = "name"; // NON-NLS
  private static final String SET_TYPE_ATTRIBUTE = "type"; // NON-NLS
  private static final String SEARCH_DURING_INGEST_ATTRIBUTE = "use_for_ingest"; // NON-NLS
  private static final String SEND_INGEST_MESSAGES_ATTRIBUTE = "show_inbox_messages"; // NON-NLS
  private static final String PATH_ELEMENT = "hash_set_path"; // NON-NLS
  private static final String LEGACY_PATH_NUMBER_ATTRIBUTE = "number"; // NON-NLS
  private static final String CONFIG_FILE_NAME = "hashsets.xml"; // NON-NLS
  private static final String XSD_FILE_NAME = "HashsetsSchema.xsd"; // NON-NLS
  private static final String ENCODING = "UTF-8"; // NON-NLS
  private static final String HASH_DATABASE_FILE_EXTENSON = "kdb"; // NON-NLS
  private static HashDbManager instance = null;
  private final String configFilePath =
      PlatformUtil.getUserConfigDirectory() + File.separator + CONFIG_FILE_NAME;
  private List<HashDb> knownHashSets = new ArrayList<>();
  private List<HashDb> knownBadHashSets = new ArrayList<>();
  private Set<String> hashSetNames = new HashSet<>();
  private Set<String> hashSetPaths = new HashSet<>();
  PropertyChangeSupport changeSupport = new PropertyChangeSupport(HashDbManager.class);
  private static final Logger logger = Logger.getLogger(HashDbManager.class.getName());

  /**
   * Property change event support In events: For both of these enums, the old value should be null,
   * and the new value should be the hashset name string.
   */
  public enum SetEvt {
    DB_ADDED,
    DB_DELETED,
    DB_INDEXED
  };

  /** Gets the singleton instance of this class. */
  public static synchronized HashDbManager getInstance() {
    if (instance == null) {
      instance = new HashDbManager();
    }
    return instance;
  }

  public synchronized void addPropertyChangeListener(PropertyChangeListener listener) {
    changeSupport.addPropertyChangeListener(listener);
  }

  private HashDbManager() {
    if (hashSetsConfigurationFileExists()) {
      readHashSetsConfigurationFromDisk();
    }
  }

  /**
   * Gets the extension, without the dot separator, that the SleuthKit requires for the hash
   * database files that combine a database and an index and can therefore be updated.
   */
  static String getHashDatabaseFileExtension() {
    return HASH_DATABASE_FILE_EXTENSON;
  }

  public class HashDbManagerException extends Exception {

    private HashDbManagerException(String message) {
      super(message);
    }
  }

  /**
   * Adds an existing hash database to the set of hash databases used to classify files as known or
   * known bad and saves the configuration.
   *
   * @param hashSetName Name used to represent the hash database in user interface components.
   * @param path Full path to either a hash database file or a hash database index file.
   * @param searchDuringIngest A flag indicating whether or not the hash database should be searched
   *     during ingest.
   * @param sendIngestMessages A flag indicating whether hash set hit messages should be sent as
   *     ingest messages.
   * @param knownFilesType The classification to apply to files whose hashes are found in the hash
   *     database.
   * @return A HashDb representing the hash database.
   * @throws HashDbManagerException
   */
  public HashDb addExistingHashDatabase(
      String hashSetName,
      String path,
      boolean searchDuringIngest,
      boolean sendIngestMessages,
      HashDb.KnownFilesType knownFilesType)
      throws HashDbManagerException {
    HashDb hashDb = null;
    try {
      addExistingHashDatabaseInternal(
          hashSetName, path, searchDuringIngest, sendIngestMessages, knownFilesType);
    } catch (TskCoreException ex) {
      throw new HashDbManagerException(ex.getMessage());
    }

    // Save the configuration
    if (!save()) {
      throw new HashDbManagerException(
          NbBundle.getMessage(this.getClass(), "HashDbManager.saveErrorExceptionMsg"));
    }

    return hashDb;
  }

  /**
   * Adds an existing hash database to the set of hash databases used to classify files as known or
   * known bad. Does not save the configuration - the configuration is only saved on demand to
   * support cancellation of configuration panels.
   *
   * @param hashSetName Name used to represent the hash database in user interface components.
   * @param path Full path to either a hash database file or a hash database index file.
   * @param searchDuringIngest A flag indicating whether or not the hash database should be searched
   *     during ingest.
   * @param sendIngestMessages A flag indicating whether hash set hit messages should be sent as
   *     ingest messages.
   * @param knownFilesType The classification to apply to files whose hashes are found in the hash
   *     database.
   * @return A HashDb representing the hash database.
   * @throws HashDbManagerException, TskCoreException
   */
  synchronized HashDb addExistingHashDatabaseInternal(
      String hashSetName,
      String path,
      boolean searchDuringIngest,
      boolean sendIngestMessages,
      HashDb.KnownFilesType knownFilesType)
      throws HashDbManagerException, TskCoreException {
    if (!new File(path).exists()) {
      throw new HashDbManagerException(
          NbBundle.getMessage(
              HashDbManager.class, "HashDbManager.hashDbDoesNotExistExceptionMsg", path));
    }

    if (hashSetPaths.contains(path)) {
      throw new HashDbManagerException(
          NbBundle.getMessage(
              HashDbManager.class, "HashDbManager.hashDbAlreadyAddedExceptionMsg", path));
    }

    if (hashSetNames.contains(hashSetName)) {
      throw new HashDbManagerException(
          NbBundle.getMessage(
              HashDbManager.class, "HashDbManager.duplicateHashSetNameExceptionMsg", hashSetName));
    }

    return addHashDatabase(
        SleuthkitJNI.openHashDatabase(path),
        hashSetName,
        searchDuringIngest,
        sendIngestMessages,
        knownFilesType);
  }

  /**
   * Adds a new hash database to the set of hash databases used to classify files as known or known
   * bad and saves the configuration.
   *
   * @param hashSetName Hash set name used to represent the hash database in user interface
   *     components.
   * @param path Full path to the database file to be created.
   * @param searchDuringIngest A flag indicating whether or not the hash database should be searched
   *     during ingest.
   * @param sendIngestMessages A flag indicating whether hash set hit messages should be sent as
   *     ingest messages.
   * @param knownFilesType The classification to apply to files whose hashes are found in the hash
   *     database.
   * @return A HashDb representing the hash database.
   * @throws HashDbManagerException
   */
  public HashDb addNewHashDatabase(
      String hashSetName,
      String path,
      boolean searchDuringIngest,
      boolean sendIngestMessages,
      HashDb.KnownFilesType knownFilesType)
      throws HashDbManagerException {

    HashDb hashDb = null;
    try {
      hashDb =
          addNewHashDatabaseInternal(
              hashSetName, path, searchDuringIngest, sendIngestMessages, knownFilesType);
    } catch (TskCoreException ex) {
      throw new HashDbManagerException(ex.getMessage());
    }

    // Save the configuration
    if (!save()) {
      throw new HashDbManagerException(
          NbBundle.getMessage(this.getClass(), "HashDbManager.saveErrorExceptionMsg"));
    }

    return hashDb;
  }

  /**
   * Adds a new hash database to the set of hash databases used to classify files as known or known
   * bad. Does not save the configuration - the configuration is only saved on demand to support
   * cancellation of configuration panels.
   *
   * @param hashSetName Hash set name used to represent the hash database in user interface
   *     components.
   * @param path Full path to the database file to be created.
   * @param searchDuringIngest A flag indicating whether or not the hash database should be searched
   *     during ingest.
   * @param sendIngestMessages A flag indicating whether hash set hit messages should be sent as
   *     ingest messages.
   * @param knownFilesType The classification to apply to files whose hashes are found in the hash
   *     database.
   * @return A HashDb representing the hash database.
   * @throws HashDbManagerException, TskCoreException
   */
  synchronized HashDb addNewHashDatabaseInternal(
      String hashSetName,
      String path,
      boolean searchDuringIngest,
      boolean sendIngestMessages,
      HashDb.KnownFilesType knownFilesType)
      throws HashDbManagerException, TskCoreException {
    File file = new File(path);
    if (file.exists()) {
      throw new HashDbManagerException(
          NbBundle.getMessage(
              HashDbManager.class, "HashDbManager.hashDbFileExistsExceptionMsg", path));
    }
    if (!FilenameUtils.getExtension(file.getName()).equalsIgnoreCase(HASH_DATABASE_FILE_EXTENSON)) {
      throw new HashDbManagerException(
          NbBundle.getMessage(
              HashDbManager.class,
              "HashDbManager.illegalHashDbFileNameExtensionMsg",
              getHashDatabaseFileExtension()));
    }

    if (hashSetPaths.contains(path)) {
      throw new HashDbManagerException(
          NbBundle.getMessage(
              HashDbManager.class, "HashDbManager.hashDbAlreadyAddedExceptionMsg", path));
    }

    if (hashSetNames.contains(hashSetName)) {
      throw new HashDbManagerException(
          NbBundle.getMessage(
              HashDbManager.class, "HashDbManager.duplicateHashSetNameExceptionMsg", hashSetName));
    }

    return addHashDatabase(
        SleuthkitJNI.createHashDatabase(path),
        hashSetName,
        searchDuringIngest,
        sendIngestMessages,
        knownFilesType);
  }

  private HashDb addHashDatabase(
      int handle,
      String hashSetName,
      boolean searchDuringIngest,
      boolean sendIngestMessages,
      HashDb.KnownFilesType knownFilesType)
      throws TskCoreException {
    // Wrap an object around the handle.
    HashDb hashDb =
        new HashDb(handle, hashSetName, searchDuringIngest, sendIngestMessages, knownFilesType);

    // Get the indentity data before updating the collections since the
    // accessor methods may throw.
    String databasePath = hashDb.getDatabasePath();
    String indexPath = hashDb.getIndexPath();

    // Update the collections used to ensure that hash set names are unique
    // and the same database is not added to the configuration more than once.
    hashSetNames.add(hashDb.getHashSetName());
    if (!databasePath.equals("None")) { // NON-NLS
      hashSetPaths.add(databasePath);
    }
    if (!indexPath.equals("None")) { // NON-NLS
      hashSetPaths.add(indexPath);
    }

    // Add the hash database to the appropriate collection for its type.
    if (hashDb.getKnownFilesType() == HashDb.KnownFilesType.KNOWN) {
      knownHashSets.add(hashDb);
    } else {
      knownBadHashSets.add(hashDb);
    }

    // Let any external listeners know that there's a new set
    try {
      changeSupport.firePropertyChange(SetEvt.DB_ADDED.toString(), null, hashSetName);
    } catch (Exception e) {
      logger.log(Level.SEVERE, "HashDbManager listener threw exception", e); // NON-NLS
      MessageNotifyUtil.Notify.show(
          NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErr"),
          NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErrorListeningToUpdatesMsg"),
          MessageNotifyUtil.MessageType.ERROR);
    }
    return hashDb;
  }

  synchronized void indexHashDatabase(HashDb hashDb) {
    hashDb.addPropertyChangeListener(this);
    HashDbIndexer creator = new HashDbIndexer(hashDb);
    creator.execute();
  }

  @Override
  public void propertyChange(PropertyChangeEvent event) {
    if (event.getPropertyName().equals(HashDb.Event.INDEXING_DONE.name())) {
      HashDb hashDb = (HashDb) event.getNewValue();
      if (null != hashDb) {
        try {
          String indexPath = hashDb.getIndexPath();
          if (!indexPath.equals("None")) { // NON-NLS
            hashSetPaths.add(indexPath);
          }
        } catch (TskCoreException ex) {
          Logger.getLogger(HashDbManager.class.getName())
              .log(
                  Level.SEVERE,
                  "Error getting index path of "
                      + hashDb.getHashSetName()
                      + " hash database after indexing",
                  ex); // NON-NLS
        }
      }
    }
  }

  /**
   * Removes a hash database from the set of hash databases used to classify files as known or known
   * bad and saves the configuration.
   *
   * @param hashDb
   * @throws HashDbManagerException
   */
  public synchronized void removeHashDatabase(HashDb hashDb) throws HashDbManagerException {
    // Don't remove a database if ingest is running
    boolean ingestIsRunning = IngestManager.getInstance().isIngestRunning();
    if (ingestIsRunning) {
      throw new HashDbManagerException(
          NbBundle.getMessage(this.getClass(), "HashDbManager.ingestRunningExceptionMsg"));
    }
    removeHashDatabaseInternal(hashDb);
    if (!save()) {
      throw new HashDbManagerException(
          NbBundle.getMessage(this.getClass(), "HashDbManager.saveErrorExceptionMsg"));
    }
  }

  /**
   * Removes a hash database from the set of hash databases used to classify files as known or known
   * bad. Does not save the configuration - the configuration is only saved on demand to support
   * cancellation of configuration panels.
   *
   * @throws TskCoreException
   */
  synchronized void removeHashDatabaseInternal(HashDb hashDb) {
    // Remove the database from whichever hash set list it occupies,
    // and remove its hash set name from the hash set used to ensure unique
    // hash set names are used, before undertaking These operations will succeed and constitute
    // a mostly effective removal, even if the subsequent operations fail.
    String hashSetName = hashDb.getHashSetName();
    knownHashSets.remove(hashDb);
    knownBadHashSets.remove(hashDb);
    hashSetNames.remove(hashSetName);

    // Now undertake the operations that could throw.
    try {
      hashSetPaths.remove(hashDb.getIndexPath());
    } catch (TskCoreException ex) {
      Logger.getLogger(HashDbManager.class.getName())
          .log(
              Level.SEVERE,
              "Error getting index path of "
                  + hashDb.getHashSetName()
                  + " hash database when removing the database",
              ex); // NON-NLS
    }
    try {
      if (!hashDb.hasIndexOnly()) {
        hashSetPaths.remove(hashDb.getDatabasePath());
      }
    } catch (TskCoreException ex) {
      Logger.getLogger(HashDbManager.class.getName())
          .log(
              Level.SEVERE,
              "Error getting database path of "
                  + hashDb.getHashSetName()
                  + " hash database when removing the database",
              ex); // NON-NLS
    }
    try {
      hashDb.close();
    } catch (TskCoreException ex) {
      Logger.getLogger(HashDbManager.class.getName())
          .log(
              Level.SEVERE,
              "Error closing "
                  + hashDb.getHashSetName()
                  + " hash database when removing the database",
              ex); // NON-NLS
    }

    // Let any external listeners know that a set has been deleted
    try {
      changeSupport.firePropertyChange(SetEvt.DB_DELETED.toString(), null, hashSetName);
    } catch (Exception e) {
      logger.log(Level.SEVERE, "HashDbManager listener threw exception", e); // NON-NLS
      MessageNotifyUtil.Notify.show(
          NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErr"),
          NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErrorListeningToUpdatesMsg"),
          MessageNotifyUtil.MessageType.ERROR);
    }
  }

  /**
   * Gets all of the hash databases used to classify files as known or known bad.
   *
   * @return A list, possibly empty, of hash databases.
   */
  public synchronized List<HashDb> getAllHashSets() {
    List<HashDb> hashDbs = new ArrayList<>();
    hashDbs.addAll(knownHashSets);
    hashDbs.addAll(knownBadHashSets);
    return hashDbs;
  }

  /**
   * Gets all of the hash databases used to classify files as known.
   *
   * @return A list, possibly empty, of hash databases.
   */
  public synchronized List<HashDb> getKnownFileHashSets() {
    List<HashDb> hashDbs = new ArrayList<>();
    hashDbs.addAll(knownHashSets);
    return hashDbs;
  }

  /**
   * Gets all of the hash databases used to classify files as known bad.
   *
   * @return A list, possibly empty, of hash databases.
   */
  public synchronized List<HashDb> getKnownBadFileHashSets() {
    List<HashDb> hashDbs = new ArrayList<>();
    hashDbs.addAll(knownBadHashSets);
    return hashDbs;
  }

  /**
   * Gets all of the hash databases that accept updates.
   *
   * @return A list, possibly empty, of hash databases.
   */
  public synchronized List<HashDb> getUpdateableHashSets() {
    List<HashDb> updateableDbs = getUpdateableHashSets(knownHashSets);
    updateableDbs.addAll(getUpdateableHashSets(knownBadHashSets));
    return updateableDbs;
  }

  private List<HashDb> getUpdateableHashSets(List<HashDb> hashDbs) {
    ArrayList<HashDb> updateableDbs = new ArrayList<>();
    for (HashDb db : hashDbs) {
      try {
        if (db.isUpdateable()) {
          updateableDbs.add(db);
        }
      } catch (TskCoreException ex) {
        Logger.getLogger(HashDbManager.class.getName())
            .log(
                Level.SEVERE,
                "Error checking updateable status of " + db.getHashSetName() + " hash database",
                ex); // NON-NLS
      }
    }
    return updateableDbs;
  }

  /**
   * Saves the hash sets configuration. Note that the configuration is only saved on demand to
   * support cancellation of configuration panels.
   *
   * @return True on success, false otherwise.
   */
  synchronized boolean save() {
    return writeHashSetConfigurationToDisk();
  }

  /**
   * Restores the last saved hash sets configuration. This supports cancellation of configuration
   * panels.
   */
  public synchronized void loadLastSavedConfiguration() {
    closeHashDatabases(knownHashSets);
    closeHashDatabases(knownBadHashSets);
    hashSetNames.clear();
    hashSetPaths.clear();

    if (hashSetsConfigurationFileExists()) {
      readHashSetsConfigurationFromDisk();
    }
  }

  private void closeHashDatabases(List<HashDb> hashDatabases) {
    for (HashDb database : hashDatabases) {
      try {
        database.close();
      } catch (TskCoreException ex) {
        Logger.getLogger(HashDbManager.class.getName())
            .log(
                Level.SEVERE,
                "Error closing " + database.getHashSetName() + " hash database",
                ex); // NON-NLS
      }
    }
    hashDatabases.clear();
  }

  private boolean writeHashSetConfigurationToDisk() {
    boolean success = false;
    DocumentBuilderFactory dbfac = DocumentBuilderFactory.newInstance();
    try {
      DocumentBuilder docBuilder = dbfac.newDocumentBuilder();
      Document doc = docBuilder.newDocument();
      Element rootEl = doc.createElement(ROOT_ELEMENT);
      doc.appendChild(rootEl);

      writeHashDbsToDisk(doc, rootEl, knownHashSets);
      writeHashDbsToDisk(doc, rootEl, knownBadHashSets);

      success = XMLUtil.saveDoc(HashDbManager.class, configFilePath, ENCODING, doc);
    } catch (ParserConfigurationException ex) {
      Logger.getLogger(HashDbManager.class.getName())
          .log(Level.SEVERE, "Error saving hash databases", ex); // NON-NLS
    }
    return success;
  }

  private static void writeHashDbsToDisk(Document doc, Element rootEl, List<HashDb> hashDbs) {
    for (HashDb db : hashDbs) {
      // Get the path for the hash database before writing anything, in
      // case an exception is thrown.
      String path;
      try {
        if (db.hasIndexOnly()) {
          path = db.getIndexPath();
        } else {
          path = db.getDatabasePath();
        }
      } catch (TskCoreException ex) {
        Logger.getLogger(HashDbManager.class.getName())
            .log(
                Level.SEVERE,
                "Error getting path of hash database "
                    + db.getHashSetName()
                    + ", discarding from hash database configuration",
                ex); // NON-NLS
        continue;
      }

      Element setElement = doc.createElement(SET_ELEMENT);
      setElement.setAttribute(SET_NAME_ATTRIBUTE, db.getHashSetName());
      setElement.setAttribute(SET_TYPE_ATTRIBUTE, db.getKnownFilesType().toString());
      setElement.setAttribute(
          SEARCH_DURING_INGEST_ATTRIBUTE, Boolean.toString(db.getSearchDuringIngest()));
      setElement.setAttribute(
          SEND_INGEST_MESSAGES_ATTRIBUTE, Boolean.toString(db.getSendIngestMessages()));
      Element pathElement = doc.createElement(PATH_ELEMENT);
      pathElement.setTextContent(path);
      setElement.appendChild(pathElement);
      rootEl.appendChild(setElement);
    }
  }

  private boolean hashSetsConfigurationFileExists() {
    File f = new File(configFilePath);
    return f.exists() && f.canRead() && f.canWrite();
  }

  private boolean readHashSetsConfigurationFromDisk() {
    boolean updatedSchema = false;

    // Open the XML document that implements the configuration file.
    final Document doc = XMLUtil.loadDoc(HashDbManager.class, configFilePath);
    if (doc == null) {
      return false;
    }

    // Get the root element.
    Element root = doc.getDocumentElement();
    if (root == null) {
      Logger.getLogger(HashDbManager.class.getName())
          .log(Level.SEVERE, "Error loading hash sets: invalid file format."); // NON-NLS
      return false;
    }

    // Get the hash set elements.
    NodeList setsNList = root.getElementsByTagName(SET_ELEMENT);
    int numSets = setsNList.getLength();
    if (numSets == 0) {
      Logger.getLogger(HashDbManager.class.getName())
          .log(Level.WARNING, "No element hash_set exists."); // NON-NLS
    }

    // Create HashDb objects for each hash set element. Skip to the next hash database if the
    // definition of
    // a particular hash database is not well-formed.
    String attributeErrorMessage =
        " attribute was not set for hash_set at index {0}, cannot make instance of HashDb class"; // NON-NLS
    String elementErrorMessage =
        " element was not set for hash_set at index {0}, cannot make instance of HashDb class"; // NON-NLS
    for (int i = 0; i < numSets; ++i) {
      Element setEl = (Element) setsNList.item(i);

      String hashSetName = setEl.getAttribute(SET_NAME_ATTRIBUTE);
      if (hashSetName.isEmpty()) {
        Logger.getLogger(HashDbManager.class.getName())
            .log(Level.SEVERE, SET_NAME_ATTRIBUTE + attributeErrorMessage, i);
        continue;
      }

      // Handle configurations saved before duplicate hash set names were not permitted.
      if (hashSetNames.contains(hashSetName)) {
        int suffix = 0;
        String newHashSetName;
        do {
          ++suffix;
          newHashSetName = hashSetName + suffix;
        } while (hashSetNames.contains(newHashSetName));
        JOptionPane.showMessageDialog(
            null,
            NbBundle.getMessage(
                this.getClass(),
                "HashDbManager.replacingDuplicateHashsetNameMsg",
                hashSetName,
                newHashSetName),
            NbBundle.getMessage(this.getClass(), "HashDbManager.openHashDbErr"),
            JOptionPane.ERROR_MESSAGE);
        hashSetName = newHashSetName;
      }

      String knownFilesType = setEl.getAttribute(SET_TYPE_ATTRIBUTE);
      if (knownFilesType.isEmpty()) {
        Logger.getLogger(HashDbManager.class.getName())
            .log(Level.SEVERE, SET_TYPE_ATTRIBUTE + attributeErrorMessage, i);
        continue;
      }

      // Handle legacy known files types.
      if (knownFilesType.equals("NSRL")) { // NON-NLS
        knownFilesType = HashDb.KnownFilesType.KNOWN.toString();
        updatedSchema = true;
      }

      final String searchDuringIngest = setEl.getAttribute(SEARCH_DURING_INGEST_ATTRIBUTE);
      if (searchDuringIngest.isEmpty()) {
        Logger.getLogger(HashDbManager.class.getName())
            .log(Level.SEVERE, SEARCH_DURING_INGEST_ATTRIBUTE + attributeErrorMessage, i);
        continue;
      }
      Boolean seearchDuringIngestFlag = Boolean.parseBoolean(searchDuringIngest);

      final String sendIngestMessages = setEl.getAttribute(SEND_INGEST_MESSAGES_ATTRIBUTE);
      if (searchDuringIngest.isEmpty()) {
        Logger.getLogger(HashDbManager.class.getName())
            .log(Level.SEVERE, SEND_INGEST_MESSAGES_ATTRIBUTE + attributeErrorMessage, i);
        continue;
      }
      Boolean sendIngestMessagesFlag = Boolean.parseBoolean(sendIngestMessages);

      String dbPath;
      NodeList pathsNList = setEl.getElementsByTagName(PATH_ELEMENT);
      if (pathsNList.getLength() > 0) {
        Element pathEl = (Element) pathsNList.item(0); // Shouldn't be more than one.

        // Check for legacy path number attribute.
        String legacyPathNumber = pathEl.getAttribute(LEGACY_PATH_NUMBER_ATTRIBUTE);
        if (null != legacyPathNumber && !legacyPathNumber.isEmpty()) {
          updatedSchema = true;
        }

        dbPath = pathEl.getTextContent();
        if (dbPath.isEmpty()) {
          Logger.getLogger(HashDbManager.class.getName())
              .log(Level.SEVERE, PATH_ELEMENT + elementErrorMessage, i);
          continue;
        }
      } else {
        Logger.getLogger(HashDbManager.class.getName())
            .log(Level.SEVERE, PATH_ELEMENT + elementErrorMessage, i);
        continue;
      }
      dbPath = getValidFilePath(hashSetName, dbPath);

      if (null != dbPath) {
        try {
          addExistingHashDatabaseInternal(
              hashSetName,
              dbPath,
              seearchDuringIngestFlag,
              sendIngestMessagesFlag,
              HashDb.KnownFilesType.valueOf(knownFilesType));
        } catch (HashDbManagerException | TskCoreException ex) {
          Logger.getLogger(HashDbManager.class.getName())
              .log(Level.SEVERE, "Error opening hash database", ex); // NON-NLS
          JOptionPane.showMessageDialog(
              null,
              NbBundle.getMessage(this.getClass(), "HashDbManager.unableToOpenHashDbMsg", dbPath),
              NbBundle.getMessage(this.getClass(), "HashDbManager.openHashDbErr"),
              JOptionPane.ERROR_MESSAGE);
        }
      } else {
        Logger.getLogger(HashDbManager.class.getName())
            .log(
                Level.WARNING,
                "No valid path for hash_set at index {0}, cannot make instance of HashDb class",
                i); // NON-NLS
      }
    }

    if (updatedSchema) {
      String backupFilePath = configFilePath + ".v1_backup"; // NON-NLS
      String messageBoxTitle =
          NbBundle.getMessage(this.getClass(), "HashDbManager.msgBoxTitle.confFileFmtChanged");
      String baseMessage =
          NbBundle.getMessage(
              this.getClass(), "HashDbManager.baseMessage.updatedFormatHashDbConfig");
      try {
        FileUtils.copyFile(new File(configFilePath), new File(backupFilePath));
        JOptionPane.showMessageDialog(
            null,
            NbBundle.getMessage(
                this.getClass(),
                "HashDbManager.savedBackupOfOldConfigMsg",
                baseMessage,
                backupFilePath),
            messageBoxTitle,
            JOptionPane.INFORMATION_MESSAGE);
      } catch (IOException ex) {
        Logger.getLogger(HashDbManager.class.getName())
            .log(
                Level.WARNING,
                "Failed to save backup of old format configuration file to " + backupFilePath,
                ex); // NON-NLS
        JOptionPane.showMessageDialog(
            null, baseMessage, messageBoxTitle, JOptionPane.INFORMATION_MESSAGE);
      }

      writeHashSetConfigurationToDisk();
    }

    return true;
  }

  private String getValidFilePath(String hashSetName, String configuredPath) {
    // Check the configured path.
    File database = new File(configuredPath);
    if (database.exists()) {
      return configuredPath;
    }

    // Give the user an opportunity to find the desired file.
    String newPath = null;
    if (JOptionPane.showConfirmDialog(
            null,
            NbBundle.getMessage(
                this.getClass(),
                "HashDbManager.dlgMsg.dbNotFoundAtLoc",
                hashSetName,
                configuredPath),
            NbBundle.getMessage(this.getClass(), "HashDbManager.dlgTitle.MissingDb"),
            JOptionPane.YES_NO_OPTION)
        == JOptionPane.YES_OPTION) {
      newPath = searchForFile();
      if (null != newPath && !newPath.isEmpty()) {
        database = new File(newPath);
        if (!database.exists()) {
          newPath = null;
        }
      }
    }
    return newPath;
  }

  private String searchForFile() {
    String filePath = null;
    JFileChooser fc = new JFileChooser();
    fc.setDragEnabled(false);
    fc.setFileSelectionMode(JFileChooser.FILES_ONLY);
    String[] EXTENSION = new String[] {"txt", "idx", "hash", "Hash", "kdb"}; // NON-NLS
    FileNameExtensionFilter filter =
        new FileNameExtensionFilter(
            NbBundle.getMessage(this.getClass(), "HashDbManager.fileNameExtensionFilter.title"),
            EXTENSION);
    fc.setFileFilter(filter);
    fc.setMultiSelectionEnabled(false);
    if (fc.showOpenDialog(null) == JFileChooser.APPROVE_OPTION) {
      File f = fc.getSelectedFile();
      try {
        filePath = f.getCanonicalPath();
      } catch (IOException ex) {
        Logger.getLogger(HashDbManager.class.getName())
            .log(Level.WARNING, "Couldn't get selected file path", ex); // NON-NLS
      }
    }
    return filePath;
  }

  /**
   * Instances of this class represent hash databases used to classify files as known or know bad.
   */
  public static class HashDb {

    /**
     * Indicates how files with hashes stored in a particular hash database object should be
     * classified.
     */
    public enum KnownFilesType {
      KNOWN(NbBundle.getMessage(HashDbManager.class, "HashDbManager.known.text")),
      KNOWN_BAD(NbBundle.getMessage(HashDbManager.class, "HashDbManager.knownBad.text"));
      private String displayName;

      private KnownFilesType(String displayName) {
        this.displayName = displayName;
      }

      public String getDisplayName() {
        return this.displayName;
      }
    }

    /** Property change events published by hash database objects. */
    public enum Event {
      INDEXING_DONE
    }

    private int handle;
    private String hashSetName;
    private boolean searchDuringIngest;
    private boolean sendIngestMessages;
    private KnownFilesType knownFilesType;
    private boolean indexing;
    private final PropertyChangeSupport propertyChangeSupport = new PropertyChangeSupport(this);

    private HashDb(
        int handle,
        String hashSetName,
        boolean useForIngest,
        boolean sendHitMessages,
        KnownFilesType knownFilesType) {
      this.handle = handle;
      this.hashSetName = hashSetName;
      this.searchDuringIngest = useForIngest;
      this.sendIngestMessages = sendHitMessages;
      this.knownFilesType = knownFilesType;
      this.indexing = false;
    }

    /** Adds a listener for the events defined in HashDb.Event. */
    public void addPropertyChangeListener(PropertyChangeListener pcl) {
      propertyChangeSupport.addPropertyChangeListener(pcl);
    }

    /** Removes a listener for the events defined in HashDb.Event. */
    public void removePropertyChangeListener(PropertyChangeListener pcl) {
      propertyChangeSupport.removePropertyChangeListener(pcl);
    }

    public String getHashSetName() {
      return hashSetName;
    }

    public String getDatabasePath() throws TskCoreException {
      return SleuthkitJNI.getHashDatabasePath(handle);
    }

    public String getIndexPath() throws TskCoreException {
      return SleuthkitJNI.getHashDatabaseIndexPath(handle);
    }

    public KnownFilesType getKnownFilesType() {
      return knownFilesType;
    }

    public boolean getSearchDuringIngest() {
      return searchDuringIngest;
    }

    void setSearchDuringIngest(boolean useForIngest) {
      this.searchDuringIngest = useForIngest;
    }

    public boolean getSendIngestMessages() {
      return sendIngestMessages;
    }

    void setSendIngestMessages(boolean showInboxMessages) {
      this.sendIngestMessages = showInboxMessages;
    }

    /**
     * Indicates whether the hash database accepts updates.
     *
     * @return True if the database accepts updates, false otherwise.
     */
    public boolean isUpdateable() throws TskCoreException {
      return SleuthkitJNI.isUpdateableHashDatabase(this.handle);
    }

    /**
     * Adds hashes of content (if calculated) to the hash database.
     *
     * @param content The content for which the calculated hashes, if any, are to be added to the
     *     hash database.
     * @throws TskCoreException
     */
    public void addHashes(Content content) throws TskCoreException {
      addHashes(content, null);
    }

    /**
     * Adds hashes of content (if calculated) to the hash database.
     *
     * @param content The content for which the calculated hashes, if any, are to be added to the
     *     hash database.
     * @param comment A comment to associate with the hashes, e.g., the name of the case in which
     *     the content was encountered.
     * @throws TskCoreException
     */
    public void addHashes(Content content, String comment) throws TskCoreException {
      // This only works for AbstractFiles and MD5 hashes at present.
      assert content instanceof AbstractFile;
      if (content instanceof AbstractFile) {
        AbstractFile file = (AbstractFile) content;
        if (null != file.getMd5Hash()) {
          SleuthkitJNI.addToHashDatabase(null, file.getMd5Hash(), null, null, comment, handle);
        }
      }
    }

    /**
     * Adds a list of hashes to the hash database at once
     *
     * @param hashes List of hashes
     * @throws TskCoreException
     */
    public void addHashes(List<HashEntry> hashes) throws TskCoreException {
      SleuthkitJNI.addToHashDatabase(hashes, handle);
    }

    /**
     * Perform a basic boolean lookup of the file's hash.
     *
     * @param content
     * @return True if file's MD5 is in the hash database
     * @throws TskCoreException
     */
    public boolean lookupMD5Quick(Content content) throws TskCoreException {
      boolean result = false;
      assert content instanceof AbstractFile;
      if (content instanceof AbstractFile) {
        AbstractFile file = (AbstractFile) content;
        if (null != file.getMd5Hash()) {
          result = SleuthkitJNI.lookupInHashDatabase(file.getMd5Hash(), handle);
        }
      }
      return result;
    }

    /**
     * Lookup hash value in DB and provide details on file.
     *
     * @param content
     * @return null if file is not in database.
     * @throws TskCoreException
     */
    public HashHitInfo lookupMD5(Content content) throws TskCoreException {
      HashHitInfo result = null;
      // This only works for AbstractFiles and MD5 hashes at present.
      assert content instanceof AbstractFile;
      if (content instanceof AbstractFile) {
        AbstractFile file = (AbstractFile) content;
        if (null != file.getMd5Hash()) {
          result = SleuthkitJNI.lookupInHashDatabaseVerbose(file.getMd5Hash(), handle);
        }
      }
      return result;
    }

    boolean hasIndex() throws TskCoreException {
      return SleuthkitJNI.hashDatabaseHasLookupIndex(handle);
    }

    boolean hasIndexOnly() throws TskCoreException {
      return SleuthkitJNI.hashDatabaseIsIndexOnly(handle);
    }

    boolean canBeReIndexed() throws TskCoreException {
      return SleuthkitJNI.hashDatabaseCanBeReindexed(handle);
    }

    boolean isIndexing() {
      return indexing;
    }

    private void close() throws TskCoreException {
      SleuthkitJNI.closeHashDatabase(handle);
    }
  }

  /** Worker thread to make an index of a database */
  private class HashDbIndexer extends SwingWorker<Object, Void> {

    private ProgressHandle progress = null;
    private HashDb hashDb = null;

    HashDbIndexer(HashDb hashDb) {
      this.hashDb = hashDb;
    };

    @Override
    protected Object doInBackground() {
      hashDb.indexing = true;
      progress =
          ProgressHandleFactory.createHandle(
              NbBundle.getMessage(
                  this.getClass(), "HashDbManager.progress.indexingHashSet", hashDb.hashSetName));
      progress.start();
      progress.switchToIndeterminate();
      try {
        SleuthkitJNI.createLookupIndexForHashDatabase(hashDb.handle);
      } catch (TskCoreException ex) {
        Logger.getLogger(HashDb.class.getName())
            .log(Level.SEVERE, "Error indexing hash database", ex); // NON-NLS
        JOptionPane.showMessageDialog(
            null,
            NbBundle.getMessage(
                this.getClass(),
                "HashDbManager.dlgMsg.errorIndexingHashSet",
                hashDb.getHashSetName()),
            NbBundle.getMessage(this.getClass(), "HashDbManager.hashDbIndexingErr"),
            JOptionPane.ERROR_MESSAGE);
      }
      return null;
    }

    @Override
    protected void done() {
      hashDb.indexing = false;
      progress.finish();

      // see if we got any errors
      try {
        get();
      } catch (InterruptedException | ExecutionException ex) {
        logger.log(Level.SEVERE, "Error creating index", ex); // NON-NLS
        MessageNotifyUtil.Notify.show(
            NbBundle.getMessage(this.getClass(), "HashDbManager.errCreatingIndex.title"),
            NbBundle.getMessage(
                this.getClass(), "HashDbManager.errCreatingIndex.msg", ex.getMessage()),
            MessageNotifyUtil.MessageType.ERROR);
      } // catch and ignore if we were cancelled
      catch (java.util.concurrent.CancellationException ex) {
      }

      try {
        hashDb.propertyChangeSupport.firePropertyChange(
            HashDb.Event.INDEXING_DONE.toString(), null, hashDb);
        hashDb.propertyChangeSupport.firePropertyChange(
            HashDbManager.SetEvt.DB_INDEXED.toString(), null, hashDb.getHashSetName());
      } catch (Exception e) {
        logger.log(Level.SEVERE, "HashDbManager listener threw exception", e); // NON-NLS
        MessageNotifyUtil.Notify.show(
            NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErr"),
            NbBundle.getMessage(this.getClass(), "HashDbManager.moduleErrorListeningToUpdatesMsg"),
            MessageNotifyUtil.MessageType.ERROR);
      }
    }
  }
}