@Override protected Object doInBackground() throws Exception { registerWriter( this); // register (synchronized on class) outside of writerLock to prevent deadlock final String queryStr = query.getQueryString(); final String queryDisp = queryStr.length() > QUERY_DISPLAY_LEN ? queryStr.substring(0, QUERY_DISPLAY_LEN - 1) + " ..." : queryStr; // block until previous writer is done // writerLock.lock(); try { progress = ProgressHandleFactory.createHandle( NbBundle.getMessage( this.getClass(), "KeywordSearchResultFactory.progress.saving", queryDisp), new Cancellable() { @Override public boolean cancel() { return BlackboardResultWriter.this.cancel(true); } }); // Create blackboard artifacts newArtifacts = hits.writeAllHitsToBlackBoard(progress, null, this, false); } finally { finalizeWorker(); } return null; }
/** * Return the string used to later have SOLR highlight the document with. * * @param query * @param literal_query * @param queryResults * @param file * @return */ private String getHighlightQuery( KeywordSearchQuery query, boolean literal_query, QueryResults queryResults, Content content) { String highlightQueryEscaped; if (literal_query) { // literal, treat as non-regex, non-term component query highlightQueryEscaped = query.getQueryString(); } else { // construct a Solr query using aggregated terms to get highlighting // the query is executed later on demand StringBuilder highlightQuery = new StringBuilder(); if (queryResults.getKeywords().size() == 1) { // simple case, no need to process subqueries and do special escaping Keyword term = queryResults.getKeywords().iterator().next(); highlightQuery.append(term.toString()); } else { // find terms for this content hit List<String> hitTerms = new ArrayList<>(); for (Keyword keyword : queryResults.getKeywords()) { for (KeywordHit hit : queryResults.getResults(keyword)) { if (hit.getContent().equals(content)) { hitTerms.add(keyword.toString()); break; // go to next term } } } final int lastTerm = hitTerms.size() - 1; int curTerm = 0; for (String term : hitTerms) { // escape subqueries, they shouldn't be escaped again later final String termS = KeywordSearchUtil.escapeLuceneQuery(term); highlightQuery.append("\""); highlightQuery.append(termS); highlightQuery.append("\""); if (lastTerm != curTerm) { highlightQuery.append(" "); // acts as OR || // force HIGHLIGHT_FIELD_REGEX index and stored content // in each term after first. First term taken care by HighlightedMatchesSource highlightQuery.append(LuceneQuery.HIGHLIGHT_FIELD_REGEX).append(":"); } ++curTerm; } } // String highlightQueryEscaped = // KeywordSearchUtil.escapeLuceneQuery(highlightQuery.toString()); highlightQueryEscaped = highlightQuery.toString(); } return highlightQueryEscaped; }
@Override protected Object doInBackground() throws Exception { logger.log(Level.INFO, "Pending start of new searcher"); final String displayName = "Keyword Search" + (finalRun ? " - Finalizing" : ""); progress = ProgressHandleFactory.createHandle( displayName + (" (Pending)"), new Cancellable() { @Override public boolean cancel() { logger.log(Level.INFO, "Cancelling the searcher by user."); if (progress != null) { progress.setDisplayName(displayName + " (Cancelling...)"); } return Searcher.this.cancel(true); } }); progress.start(); progress.switchToIndeterminate(); // block to ensure previous searcher is completely done with doInBackground() // even after previous searcher cancellation, we need to check this searcherLock.lock(); try { logger.log(Level.INFO, "Started a new searcher"); progress.setDisplayName(displayName); // make sure other searchers are not spawned searcherDone = false; runSearcher = false; if (searchTimer.isRunning()) { searchTimer.stop(); } int numSearched = 0; updateKeywords(); progress.switchToDeterminate(keywords.size()); for (Keyword keywordQuery : keywords) { if (this.isCancelled()) { logger.log( Level.INFO, "Cancel detected, bailing before new keyword processed: " + keywordQuery.getQuery()); return null; } final String queryStr = keywordQuery.getQuery(); final KeywordSearchList list = keywordToList.get(queryStr); final String listName = list.getName(); // DEBUG // logger.log(Level.INFO, "Searching: " + queryStr); progress.progress(queryStr, numSearched); KeywordSearchQuery del = null; boolean isRegex = !keywordQuery.isLiteral(); if (!isRegex) { del = new LuceneQuery(keywordQuery); del.escape(); } else { del = new TermComponentQuery(keywordQuery); } Map<String, List<ContentHit>> queryResult = null; try { queryResult = del.performQuery(); } catch (NoOpenCoreException ex) { logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), ex); // no reason to continue with next query if recovery failed // or wait for recovery to kick in and run again later // likely case has closed and threads are being interrupted return null; } catch (CancellationException e) { logger.log( Level.INFO, "Cancel detected, bailing during keyword query: " + keywordQuery.getQuery()); return null; } catch (Exception e) { logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), e); continue; } // calculate new results but substracting results already obtained in this run Map<Keyword, List<ContentHit>> newResults = new HashMap<Keyword, List<ContentHit>>(); for (String termResult : queryResult.keySet()) { List<ContentHit> queryTermResults = queryResult.get(termResult); Keyword termResultK = new Keyword(termResult, !isRegex); List<ContentHit> curTermResults = currentResults.get(termResultK); if (curTermResults == null) { currentResults.put(termResultK, queryTermResults); newResults.put(termResultK, queryTermResults); } else { // some AbstractFile hits already exist for this keyword for (ContentHit res : queryTermResults) { if (!previouslyHit(curTermResults, res)) { // add to new results List<ContentHit> newResultsFs = newResults.get(termResultK); if (newResultsFs == null) { newResultsFs = new ArrayList<ContentHit>(); newResults.put(termResultK, newResultsFs); } newResultsFs.add(res); curTermResults.add(res); } } } } if (!newResults.isEmpty()) { // write results to BB // new artifacts created, to report to listeners Collection<BlackboardArtifact> newArtifacts = new ArrayList<BlackboardArtifact>(); for (final Keyword hitTerm : newResults.keySet()) { List<ContentHit> contentHitsAll = newResults.get(hitTerm); Map<AbstractFile, Integer> contentHitsFlattened = ContentHit.flattenResults(contentHitsAll); for (final AbstractFile hitFile : contentHitsFlattened.keySet()) { String snippet = null; final String snippetQuery = KeywordSearchUtil.escapeLuceneQuery(hitTerm.getQuery(), true, false); int chunkId = contentHitsFlattened.get(hitFile); try { snippet = LuceneQuery.querySnippet( snippetQuery, hitFile.getId(), chunkId, isRegex, true); } catch (NoOpenCoreException e) { logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e); // no reason to continue return null; } catch (Exception e) { logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e); continue; } KeywordWriteResult written = del.writeToBlackBoard(hitTerm.getQuery(), hitFile, snippet, listName); if (written == null) { logger.log( Level.WARNING, "BB artifact for keyword hit not written, file: " + hitFile + ", hit: " + hitTerm.toString()); continue; } newArtifacts.add(written.getArtifact()); // generate a data message for each artifact StringBuilder subjectSb = new StringBuilder(); StringBuilder detailsSb = new StringBuilder(); // final int hitFiles = newResults.size(); if (!keywordQuery.isLiteral()) { subjectSb.append("RegExp hit: "); } else { subjectSb.append("Keyword hit: "); } // subjectSb.append("<"); String uniqueKey = null; BlackboardAttribute attr = written.getAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID()); if (attr != null) { final String keyword = attr.getValueString(); subjectSb.append(keyword); uniqueKey = keyword.toLowerCase(); } // subjectSb.append(">"); // String uniqueKey = queryStr; // details detailsSb.append("<table border='0' cellpadding='4' width='280'>"); // hit detailsSb.append("<tr>"); detailsSb.append("<th>Keyword hit</th>"); detailsSb .append("<td>") .append(StringEscapeUtils.escapeHtml(attr.getValueString())) .append("</td>"); detailsSb.append("</tr>"); // preview attr = written.getAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID()); if (attr != null) { detailsSb.append("<tr>"); detailsSb.append("<th>Preview</th>"); detailsSb .append("<td>") .append(StringEscapeUtils.escapeHtml(attr.getValueString())) .append("</td>"); detailsSb.append("</tr>"); } // file detailsSb.append("<tr>"); detailsSb.append("<th>File</th>"); if (hitFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) { detailsSb .append("<td>") .append(((FsContent) hitFile).getParentPath()) .append(hitFile.getName()) .append("</td>"); } else { detailsSb.append("<td>").append(hitFile.getName()).append("</td>"); } detailsSb.append("</tr>"); // list attr = written.getAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID()); detailsSb.append("<tr>"); detailsSb.append("<th>List</th>"); detailsSb.append("<td>").append(attr.getValueString()).append("</td>"); detailsSb.append("</tr>"); // regex if (!keywordQuery.isLiteral()) { attr = written.getAttribute( BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID()); if (attr != null) { detailsSb.append("<tr>"); detailsSb.append("<th>RegEx</th>"); detailsSb.append("<td>").append(attr.getValueString()).append("</td>"); detailsSb.append("</tr>"); } } detailsSb.append("</table>"); // check if should send messages on hits on this list if (list.getIngestMessages()) // post ingest inbox msg { managerProxy.postMessage( IngestMessage.createDataMessage( ++messageID, instance, subjectSb.toString(), detailsSb.toString(), uniqueKey, written.getArtifact())); } } // for each term hit } // for each file hit // update artifact browser if (!newArtifacts.isEmpty()) { IngestManager.fireServiceDataEvent( new ServiceDataEvent(MODULE_NAME, ARTIFACT_TYPE.TSK_KEYWORD_HIT, newArtifacts)); } } progress.progress(queryStr, ++numSearched); } } // end try block catch (Exception ex) { logger.log(Level.WARNING, "searcher exception occurred", ex); } finally { finalizeSearcher(); searcherLock.unlock(); } return null; }
/** * @param queryRequest * @param toPopulate * @return */ private boolean createFlatKeys(QueryRequest queryRequest, List<KeyValueQueryContent> toPopulate) { /** Check the validity of the requested query. */ final KeywordSearchQuery keywordSearchQuery = queryRequest.getQuery(); if (!keywordSearchQuery.validate()) { // TODO mark the particular query node RED return false; } /** Execute the requested query. */ QueryResults queryResults; try { queryResults = keywordSearchQuery.performQuery(); } catch (NoOpenCoreException ex) { logger.log( Level.SEVERE, "Could not perform the query " + keywordSearchQuery.getQueryString(), ex); // NON-NLS return false; } int id = 0; List<KeyValueQueryContent> tempList = new ArrayList<>(); for (KeywordHit hit : getOneHitPerObject(queryResults)) { /** Get file properties. */ Map<String, Object> properties = new LinkedHashMap<>(); Content content = hit.getContent(); if (content instanceof AbstractFile) { AbstractFsContentNode.fillPropertyMap(properties, (AbstractFile) content); } else { properties.put( AbstractAbstractFileNode.AbstractFilePropertyType.LOCATION.toString(), content.getName()); } /** Add a snippet property, if available. */ if (hit.hasSnippet()) { setCommonProperty(properties, CommonPropertyTypes.CONTEXT, hit.getSnippet()); } // @@@ USE ConentHit in UniqueFileMap instead of the below search // get unique match result files // BC: @@@ THis is really ineffecient. We should keep track of this when // we flattened the list of files to the unique files. final String highlightQueryEscaped = getHighlightQuery( keywordSearchQuery, keywordSearchQuery.isLiteral(), queryResults, content); String name = content.getName(); if (hit.isArtifactHit()) name = hit.getArtifact().getDisplayName() + " Artifact"; // NON-NLS tempList.add( new KeyValueQueryContent( name, properties, ++id, hit.getSolrObjectId(), content, highlightQueryEscaped, keywordSearchQuery, queryResults)); } // Add all the nodes to toPopulate at once. Minimizes node creation // EDT threads, which can slow and/or hang the UI on large queries. toPopulate.addAll(tempList); // write to bb // cannot reuse snippet in BlackboardResultWriter // because for regex searches in UI we compress results by showing a content per regex once // (even if multiple term hits) // whereas in bb we write every hit per content separately new BlackboardResultWriter(queryResults, queryRequest.getQuery().getKeywordList().getName()) .execute(); return true; }