@Override
protected Object doInBackground() throws Exception {
registerWriter(
this); // register (synchronized on class) outside of writerLock to prevent deadlock
final String queryStr = query.getQueryString();
final String queryDisp =
queryStr.length() > QUERY_DISPLAY_LEN
? queryStr.substring(0, QUERY_DISPLAY_LEN - 1) + " ..."
: queryStr;
// block until previous writer is done
// writerLock.lock();
try {
progress =
ProgressHandleFactory.createHandle(
NbBundle.getMessage(
this.getClass(), "KeywordSearchResultFactory.progress.saving", queryDisp),
new Cancellable() {
@Override
public boolean cancel() {
return BlackboardResultWriter.this.cancel(true);
}
});
// Create blackboard artifacts
newArtifacts = hits.writeAllHitsToBlackBoard(progress, null, this, false);
} finally {
finalizeWorker();
}
return null;
}
/**
* Return the string used to later have SOLR highlight the document with.
*
* @param query
* @param literal_query
* @param queryResults
* @param file
* @return
*/
private String getHighlightQuery(
KeywordSearchQuery query, boolean literal_query, QueryResults queryResults, Content content) {
String highlightQueryEscaped;
if (literal_query) {
// literal, treat as non-regex, non-term component query
highlightQueryEscaped = query.getQueryString();
} else {
// construct a Solr query using aggregated terms to get highlighting
// the query is executed later on demand
StringBuilder highlightQuery = new StringBuilder();
if (queryResults.getKeywords().size() == 1) {
// simple case, no need to process subqueries and do special escaping
Keyword term = queryResults.getKeywords().iterator().next();
highlightQuery.append(term.toString());
} else {
// find terms for this content hit
List<String> hitTerms = new ArrayList<>();
for (Keyword keyword : queryResults.getKeywords()) {
for (KeywordHit hit : queryResults.getResults(keyword)) {
if (hit.getContent().equals(content)) {
hitTerms.add(keyword.toString());
break; // go to next term
}
}
}
final int lastTerm = hitTerms.size() - 1;
int curTerm = 0;
for (String term : hitTerms) {
// escape subqueries, they shouldn't be escaped again later
final String termS = KeywordSearchUtil.escapeLuceneQuery(term);
highlightQuery.append("\"");
highlightQuery.append(termS);
highlightQuery.append("\"");
if (lastTerm != curTerm) {
highlightQuery.append(" "); // acts as OR ||
// force HIGHLIGHT_FIELD_REGEX index and stored content
// in each term after first. First term taken care by HighlightedMatchesSource
highlightQuery.append(LuceneQuery.HIGHLIGHT_FIELD_REGEX).append(":");
}
++curTerm;
}
}
// String highlightQueryEscaped =
// KeywordSearchUtil.escapeLuceneQuery(highlightQuery.toString());
highlightQueryEscaped = highlightQuery.toString();
}
return highlightQueryEscaped;
}
@Override
protected Object doInBackground() throws Exception {
logger.log(Level.INFO, "Pending start of new searcher");
final String displayName = "Keyword Search" + (finalRun ? " - Finalizing" : "");
progress =
ProgressHandleFactory.createHandle(
displayName + (" (Pending)"),
new Cancellable() {
@Override
public boolean cancel() {
logger.log(Level.INFO, "Cancelling the searcher by user.");
if (progress != null) {
progress.setDisplayName(displayName + " (Cancelling...)");
}
return Searcher.this.cancel(true);
}
});
progress.start();
progress.switchToIndeterminate();
// block to ensure previous searcher is completely done with doInBackground()
// even after previous searcher cancellation, we need to check this
searcherLock.lock();
try {
logger.log(Level.INFO, "Started a new searcher");
progress.setDisplayName(displayName);
// make sure other searchers are not spawned
searcherDone = false;
runSearcher = false;
if (searchTimer.isRunning()) {
searchTimer.stop();
}
int numSearched = 0;
updateKeywords();
progress.switchToDeterminate(keywords.size());
for (Keyword keywordQuery : keywords) {
if (this.isCancelled()) {
logger.log(
Level.INFO,
"Cancel detected, bailing before new keyword processed: "
+ keywordQuery.getQuery());
return null;
}
final String queryStr = keywordQuery.getQuery();
final KeywordSearchList list = keywordToList.get(queryStr);
final String listName = list.getName();
// DEBUG
// logger.log(Level.INFO, "Searching: " + queryStr);
progress.progress(queryStr, numSearched);
KeywordSearchQuery del = null;
boolean isRegex = !keywordQuery.isLiteral();
if (!isRegex) {
del = new LuceneQuery(keywordQuery);
del.escape();
} else {
del = new TermComponentQuery(keywordQuery);
}
Map<String, List<ContentHit>> queryResult = null;
try {
queryResult = del.performQuery();
} catch (NoOpenCoreException ex) {
logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), ex);
// no reason to continue with next query if recovery failed
// or wait for recovery to kick in and run again later
// likely case has closed and threads are being interrupted
return null;
} catch (CancellationException e) {
logger.log(
Level.INFO,
"Cancel detected, bailing during keyword query: " + keywordQuery.getQuery());
return null;
} catch (Exception e) {
logger.log(Level.WARNING, "Error performing query: " + keywordQuery.getQuery(), e);
continue;
}
// calculate new results but substracting results already obtained in this run
Map<Keyword, List<ContentHit>> newResults = new HashMap<Keyword, List<ContentHit>>();
for (String termResult : queryResult.keySet()) {
List<ContentHit> queryTermResults = queryResult.get(termResult);
Keyword termResultK = new Keyword(termResult, !isRegex);
List<ContentHit> curTermResults = currentResults.get(termResultK);
if (curTermResults == null) {
currentResults.put(termResultK, queryTermResults);
newResults.put(termResultK, queryTermResults);
} else {
// some AbstractFile hits already exist for this keyword
for (ContentHit res : queryTermResults) {
if (!previouslyHit(curTermResults, res)) {
// add to new results
List<ContentHit> newResultsFs = newResults.get(termResultK);
if (newResultsFs == null) {
newResultsFs = new ArrayList<ContentHit>();
newResults.put(termResultK, newResultsFs);
}
newResultsFs.add(res);
curTermResults.add(res);
}
}
}
}
if (!newResults.isEmpty()) {
// write results to BB
// new artifacts created, to report to listeners
Collection<BlackboardArtifact> newArtifacts = new ArrayList<BlackboardArtifact>();
for (final Keyword hitTerm : newResults.keySet()) {
List<ContentHit> contentHitsAll = newResults.get(hitTerm);
Map<AbstractFile, Integer> contentHitsFlattened =
ContentHit.flattenResults(contentHitsAll);
for (final AbstractFile hitFile : contentHitsFlattened.keySet()) {
String snippet = null;
final String snippetQuery =
KeywordSearchUtil.escapeLuceneQuery(hitTerm.getQuery(), true, false);
int chunkId = contentHitsFlattened.get(hitFile);
try {
snippet =
LuceneQuery.querySnippet(
snippetQuery, hitFile.getId(), chunkId, isRegex, true);
} catch (NoOpenCoreException e) {
logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e);
// no reason to continue
return null;
} catch (Exception e) {
logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e);
continue;
}
KeywordWriteResult written =
del.writeToBlackBoard(hitTerm.getQuery(), hitFile, snippet, listName);
if (written == null) {
logger.log(
Level.WARNING,
"BB artifact for keyword hit not written, file: "
+ hitFile
+ ", hit: "
+ hitTerm.toString());
continue;
}
newArtifacts.add(written.getArtifact());
// generate a data message for each artifact
StringBuilder subjectSb = new StringBuilder();
StringBuilder detailsSb = new StringBuilder();
// final int hitFiles = newResults.size();
if (!keywordQuery.isLiteral()) {
subjectSb.append("RegExp hit: ");
} else {
subjectSb.append("Keyword hit: ");
}
// subjectSb.append("<");
String uniqueKey = null;
BlackboardAttribute attr =
written.getAttribute(
BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID());
if (attr != null) {
final String keyword = attr.getValueString();
subjectSb.append(keyword);
uniqueKey = keyword.toLowerCase();
}
// subjectSb.append(">");
// String uniqueKey = queryStr;
// details
detailsSb.append("<table border='0' cellpadding='4' width='280'>");
// hit
detailsSb.append("<tr>");
detailsSb.append("<th>Keyword hit</th>");
detailsSb
.append("<td>")
.append(StringEscapeUtils.escapeHtml(attr.getValueString()))
.append("</td>");
detailsSb.append("</tr>");
// preview
attr =
written.getAttribute(
BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID());
if (attr != null) {
detailsSb.append("<tr>");
detailsSb.append("<th>Preview</th>");
detailsSb
.append("<td>")
.append(StringEscapeUtils.escapeHtml(attr.getValueString()))
.append("</td>");
detailsSb.append("</tr>");
}
// file
detailsSb.append("<tr>");
detailsSb.append("<th>File</th>");
if (hitFile.getType().equals(TskData.TSK_DB_FILES_TYPE_ENUM.FS)) {
detailsSb
.append("<td>")
.append(((FsContent) hitFile).getParentPath())
.append(hitFile.getName())
.append("</td>");
} else {
detailsSb.append("<td>").append(hitFile.getName()).append("</td>");
}
detailsSb.append("</tr>");
// list
attr =
written.getAttribute(
BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID());
detailsSb.append("<tr>");
detailsSb.append("<th>List</th>");
detailsSb.append("<td>").append(attr.getValueString()).append("</td>");
detailsSb.append("</tr>");
// regex
if (!keywordQuery.isLiteral()) {
attr =
written.getAttribute(
BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID());
if (attr != null) {
detailsSb.append("<tr>");
detailsSb.append("<th>RegEx</th>");
detailsSb.append("<td>").append(attr.getValueString()).append("</td>");
detailsSb.append("</tr>");
}
}
detailsSb.append("</table>");
// check if should send messages on hits on this list
if (list.getIngestMessages()) // post ingest inbox msg
{
managerProxy.postMessage(
IngestMessage.createDataMessage(
++messageID,
instance,
subjectSb.toString(),
detailsSb.toString(),
uniqueKey,
written.getArtifact()));
}
} // for each term hit
} // for each file hit
// update artifact browser
if (!newArtifacts.isEmpty()) {
IngestManager.fireServiceDataEvent(
new ServiceDataEvent(MODULE_NAME, ARTIFACT_TYPE.TSK_KEYWORD_HIT, newArtifacts));
}
}
progress.progress(queryStr, ++numSearched);
}
} // end try block
catch (Exception ex) {
logger.log(Level.WARNING, "searcher exception occurred", ex);
} finally {
finalizeSearcher();
searcherLock.unlock();
}
return null;
}
/**
* @param queryRequest
* @param toPopulate
* @return
*/
private boolean createFlatKeys(QueryRequest queryRequest, List<KeyValueQueryContent> toPopulate) {
/** Check the validity of the requested query. */
final KeywordSearchQuery keywordSearchQuery = queryRequest.getQuery();
if (!keywordSearchQuery.validate()) {
// TODO mark the particular query node RED
return false;
}
/** Execute the requested query. */
QueryResults queryResults;
try {
queryResults = keywordSearchQuery.performQuery();
} catch (NoOpenCoreException ex) {
logger.log(
Level.SEVERE,
"Could not perform the query " + keywordSearchQuery.getQueryString(),
ex); // NON-NLS
return false;
}
int id = 0;
List<KeyValueQueryContent> tempList = new ArrayList<>();
for (KeywordHit hit : getOneHitPerObject(queryResults)) {
/** Get file properties. */
Map<String, Object> properties = new LinkedHashMap<>();
Content content = hit.getContent();
if (content instanceof AbstractFile) {
AbstractFsContentNode.fillPropertyMap(properties, (AbstractFile) content);
} else {
properties.put(
AbstractAbstractFileNode.AbstractFilePropertyType.LOCATION.toString(),
content.getName());
}
/** Add a snippet property, if available. */
if (hit.hasSnippet()) {
setCommonProperty(properties, CommonPropertyTypes.CONTEXT, hit.getSnippet());
}
// @@@ USE ConentHit in UniqueFileMap instead of the below search
// get unique match result files
// BC: @@@ THis is really ineffecient. We should keep track of this when
// we flattened the list of files to the unique files.
final String highlightQueryEscaped =
getHighlightQuery(
keywordSearchQuery, keywordSearchQuery.isLiteral(), queryResults, content);
String name = content.getName();
if (hit.isArtifactHit()) name = hit.getArtifact().getDisplayName() + " Artifact"; // NON-NLS
tempList.add(
new KeyValueQueryContent(
name,
properties,
++id,
hit.getSolrObjectId(),
content,
highlightQueryEscaped,
keywordSearchQuery,
queryResults));
}
// Add all the nodes to toPopulate at once. Minimizes node creation
// EDT threads, which can slow and/or hang the UI on large queries.
toPopulate.addAll(tempList);
// write to bb
// cannot reuse snippet in BlackboardResultWriter
// because for regex searches in UI we compress results by showing a content per regex once
// (even if multiple term hits)
// whereas in bb we write every hit per content separately
new BlackboardResultWriter(queryResults, queryRequest.getQuery().getKeywordList().getName())
.execute();
return true;
}