@Override protected Object doInBackground() throws Exception { logger.log(Level.INFO, "Starting background ingest file processor"); logger.log(Level.INFO, PlatformUtil.getAllMemUsageInfo()); stats.start(); // notify main thread modules started for (IngestModuleAbstractFile s : abstractFileModules) { IngestManager.fireModuleEvent(IngestModuleEvent.STARTED.toString(), s.getName()); } final String displayName = "File Ingest"; progress = ProgressHandleFactory.createHandle( displayName, new Cancellable() { @Override public boolean cancel() { logger.log(Level.INFO, "Filed ingest cancelled by user."); if (progress != null) { progress.setDisplayName(displayName + " (Cancelling...)"); } return IngestAbstractFileProcessor.this.cancel(true); } }); final IngestScheduler.FileScheduler fileScheduler = scheduler.getFileScheduler(); // initialize the progress bar progress.start(); progress.switchToIndeterminate(); // set initial totals and processed (to be updated as we process or new files are scheduled) int totalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst(); progress.switchToDeterminate(totalEnqueuedFiles); int processedFiles = 0; // process AbstractFiles queue while (fileScheduler.hasNext()) { final ProcessTask fileTask = fileScheduler.next(); final PipelineContext<IngestModuleAbstractFile> filepipelineContext = fileTask.context; final ScheduledTask<IngestModuleAbstractFile> fileIngestTask = filepipelineContext.getScheduledTask(); final AbstractFile fileToProcess = fileTask.file; // clear return values from modules for last file synchronized (abstractFileModulesRetValues) { abstractFileModulesRetValues.clear(); } // logger.log(Level.INFO, "IngestManager: Processing: {0}", fileToProcess.getName()); for (IngestModuleAbstractFile module : fileIngestTask.getModules()) { // process the file with every file module if (isCancelled()) { logger.log(Level.INFO, "Terminating file ingest due to cancellation."); return null; } progress.progress( fileToProcess.getName() + " (" + module.getName() + ")", processedFiles); try { stats.logFileModuleStartProcess(module); IngestModuleAbstractFile.ProcessResult result = module.process(filepipelineContext, fileToProcess); stats.logFileModuleEndProcess(module); // store the result for subsequent modules for this file synchronized (abstractFileModulesRetValues) { abstractFileModulesRetValues.put(module.getName(), result); } } catch (Exception e) { logger.log( Level.SEVERE, "Error: unexpected exception from module: " + module.getName(), e); stats.addError(module); } catch (OutOfMemoryError e) { logger.log(Level.SEVERE, "Error: out of memory from module: " + module.getName(), e); stats.addError(module); } } // end for every module // free the internal file resource after done with every module fileToProcess.close(); int newTotalEnqueuedFiles = fileScheduler.getFilesEnqueuedEst(); if (newTotalEnqueuedFiles > totalEnqueuedFiles) { // update if new enqueued totalEnqueuedFiles = newTotalEnqueuedFiles + 1; // + processedFiles + 1; // processedFiles = 0; // reset progress.switchToIndeterminate(); progress.switchToDeterminate(totalEnqueuedFiles); } if (processedFiles < totalEnqueuedFiles) { // fix for now to handle the same datasource Content enqueued // twice ++processedFiles; } // --totalEnqueuedFiles; } // end of for every AbstractFile logger.log(Level.INFO, "IngestManager: Finished processing files"); return null; }
@Override protected Object doInBackground() throws Exception { logger.log(Level.INFO, "Pending start of new searcher"); final String displayName = "Keyword Search" + (finalRun ? " - Finalizing" : ""); progress = ProgressHandleFactory.createHandle( displayName + (" (Pending)"), new Cancellable() { @Override public boolean cancel() { logger.log(Level.INFO, "Cancelling the searcher by user."); if (progress != null) { progress.setDisplayName(displayName + " (Cancelling...)"); } return Searcher.this.cancel(true); } }); progress.start(); progress.switchToIndeterminate(); // block to ensure previous searcher is completely done with doInBackground() // even after previous searcher cancellation, we need to check this searcherLock.lock(); try { logger.log(Level.INFO, "Started a new searcher"); progress.setDisplayName(displayName); // make sure other searchers are not spawned searcherDone = false; runSearcher = false; if (searchTimer.isRunning()) { searchTimer.stop(); } int numSearched = 0; updateKeywords(); progress.switchToDeterminate(keywords.size()); for (Keyword keywordQuery : keywords) { if (this.isCancelled()) { logger.log( Level.INFO, "Cancel detected, bailing before new keyword processed: " + keywordQuery.getQuery()); return null; } final String queryStr = keywordQuery.getQuery(); final KeywordSearchList list = keywordToList.get(queryStr); final String listName = list.getName(); // DEBUG // logger.log(Level.INFO, "Searching: " + queryStr); progress.progress(queryStr, numSearched); KeywordSearchQuery del = null; boolean isRegex = !keywordQuery.isLiteral(); if (!isRegex) { del = new LuceneQuery(keywordQuery); del.escape(); } else { del = new TermComponentQuery(keywordQuery); } Map<String, List<ContentHit>> queryResult = null; try { queryResult = del.performQuery(); } catch (NoOpenCoreException ex) { logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), ex); // no reason to continue with next query if recovery failed // or wait for recovery to kick in and run again later // likely case has closed and threads are being interrupted return null; } catch (CancellationException e) { logger.log( Level.INFO, "Cancel detected, bailing during keyword query: " + keywordQuery.getQuery()); return null; } catch (Exception e) { logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), e); continue; } // calculate new results but substracting results already obtained in this run Map<Keyword, List<ContentHit>> newResults = new HashMap<Keyword, List<ContentHit>>(); for (String termResult : queryResult.keySet()) { List<ContentHit> queryTermResults = queryResult.get(termResult); Keyword termResultK = new Keyword(termResult, !isRegex); List<ContentHit> curTermResults = currentResults.get(termResultK); if (curTermResults == null) { currentResults.put(termResultK, queryTermResults); newResults.put(termResultK, queryTermResults); } else { // some AbstractFile hits already exist for this keyword for (ContentHit res : queryTermResults) { if (!previouslyHit(curTermResults, res)) { // add to new results List<ContentHit> newResultsFs = newResults.get(termResultK); if (newResultsFs == null) { newResultsFs = new ArrayList<ContentHit>(); newResults.put(termResultK, newResultsFs); } newResultsFs.add(res); curTermResults.add(res); } } } } if (!newResults.isEmpty()) { // write results to BB // new artifacts created, to report to listeners Collection<BlackboardArtifact> newArtifacts = new ArrayList<BlackboardArtifact>(); for (final Keyword hitTerm : newResults.keySet()) { List<ContentHit> contentHitsAll = newResults.get(hitTerm); Map<AbstractFile, Integer> contentHitsFlattened = ContentHit.flattenResults(contentHitsAll); for (final AbstractFile hitFile : contentHitsFlattened.keySet()) { String snippet = null; final String snippetQuery = KeywordSearchUtil.escapeLuceneQuery(hitTerm.getQuery(), true, false); int chunkId = contentHitsFlattened.get(hitFile); try { snippet = LuceneQuery.querySnippet( snippetQuery, hitFile.getId(), chunkId, isRegex, true); } catch (NoOpenCoreException e) { logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e); // no reason to continue return null; } catch (Exception e) { logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e); continue; } KeywordWriteResult written = del.writeToBlackBoard(hitTerm.getQuery(), hitFile, snippet, listName); if (written == null) { logger.log( Level.WARNING, "BB artifact for keyword hit not written, file: " + hitFile + ", hit: " + hitTerm.toString()); continue; } newArtifacts.add(written.getArtifact()); // generate a data message for each artifact StringBuilder subjectSb = new StringBuilder(); StringBuilder detailsSb = new StringBuilder(); // final int hitFiles = newResults.size(); if (!keywordQuery.isLiteral()) { subjectSb.append("RegExp hit: "); } else { subjectSb.append("Keyword hit: "); } // subjectSb.append("<"); String uniqueKey = null; BlackboardAttribute attr = written.getAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID()); if (attr != null) { final String keyword = attr.getValueString(); subjectSb.append(keyword); uniqueKey = keyword.toLowerCase(); } // subjectSb.append(">"); // String uniqueKey = queryStr; // details detailsSb.append("<table border='0' cellpadding='4' width='280'>"); // hit detailsSb.append("<tr>"); detailsSb.append("<th>Keyword hit</th>"); detailsSb .append("<td>") .append(StringEscapeUtils.escapeHtml(attr.getValueString())) .append("</td>"); detailsSb.append("</tr>"); // preview attr = written.getAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID()); if (attr != null) { detailsSb.append("<tr>"); detailsSb.append("<th>Preview</th>"); detailsSb .append("<td>") .append(StringEscapeUtils.escapeHtml(attr.getValueString())) .append("</td>"); detailsSb.append("</tr>"); } // file detailsSb.append("<tr>"); detailsSb.append("<th>File</th>"); if (hitFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) { detailsSb .append("<td>") .append(((FsContent) hitFile).getParentPath()) .append(hitFile.getName()) .append("</td>"); } else { detailsSb.append("<td>").append(hitFile.getName()).append("</td>"); } detailsSb.append("</tr>"); // list attr = written.getAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID()); detailsSb.append("<tr>"); detailsSb.append("<th>List</th>"); detailsSb.append("<td>").append(attr.getValueString()).append("</td>"); detailsSb.append("</tr>"); // regex if (!keywordQuery.isLiteral()) { attr = written.getAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID()); if (attr != null) { detailsSb.append("<tr>"); detailsSb.append("<th>RegEx</th>"); detailsSb.append("<td>").append(attr.getValueString()).append("</td>"); detailsSb.append("</tr>"); } } detailsSb.append("</table>"); // check if should send messages on hits on this list if (list.getIngestMessages()) // post ingest inbox msg { managerProxy.postMessage( IngestMessage.createDataMessage( ++messageID, instance, subjectSb.toString(), detailsSb.toString(), uniqueKey, written.getArtifact())); } } // for each term hit } // for each file hit // update artifact browser if (!newArtifacts.isEmpty()) { IngestManager.fireServiceDataEvent( new ServiceDataEvent(MODULE_NAME, ARTIFACT_TYPE.TSK_KEYWORD_HIT, newArtifacts)); } } progress.progress(queryStr, ++numSearched); } } // end try block catch (Exception ex) { logger.log(Level.WARNING, "searcher exception occurred", ex); } finally { finalizeSearcher(); searcherLock.unlock(); } return null; }