@Test
public void shouldReturnCursorThatAccessesTuples() {
results = new QueryResults(columns, statistics, tuples, context.getProblems(), null);
Cursor cursor = results.getCursor();
Iterator<Object[]> expectedIter = tuples.iterator();
int rowNumber = 0;
while (cursor.hasNext() && expectedIter.hasNext()) {
cursor.next();
Object[] tuple = expectedIter.next();
// Check the column values by column name and index ...
for (Column column : results.getColumns().getColumns()) {
String columnName = column.getColumnName();
int columnIndex = columns.getColumnIndexForName(columnName);
assertThat(cursor.getValue(columnName), is(tuple[columnIndex]));
assertThat(cursor.getValue(columnIndex), is(tuple[columnIndex]));
// Get the location for this column ...
int locationIndex = columns.getLocationIndex(column.selectorName().name());
Location location = (Location) tuple[locationIndex];
assertThat(cursor.getLocation(columnIndex), is(location));
}
// Check the locations by selector name and index ...
for (String selectorName : results.getColumns().getSelectorNames()) {
int locationIndex = columns.getLocationIndex(selectorName);
Location location = (Location) tuple[locationIndex];
assertThat(cursor.getLocation(selectorName), is(location));
assertThat(location.getPath(), is(notNullValue()));
}
// Check the row index ...
assertThat(cursor.getRowIndex(), is(rowNumber++));
}
assertThat(cursor.hasNext(), is(false));
assertThat(expectedIter.hasNext(), is(false));
}
@Test
public void shouldHaveNoTuplesIfConstructedWithEmptyTuplesList() {
tuples.clear();
results = new QueryResults(columns, statistics, tuples, context.getProblems(), null);
assertThat(results.getTuples().isEmpty(), is(true));
assertThat(results.getCursor().hasNext(), is(false));
}
@Test
public void shouldReturnMutableTuplesList() {
results = new QueryResults(columns, statistics, tuples, context.getProblems(), null);
assertThat(results.getTuples().isEmpty(), is(false));
results.getTuples().clear();
assertThat(results.getTuples().isEmpty(), is(true));
assertThat(tuples.isEmpty(), is(true));
}
/**
* Return the string used to later have SOLR highlight the document with.
*
* @param query
* @param literal_query
* @param queryResults
* @param file
* @return
*/
private String getHighlightQuery(
KeywordSearchQuery query, boolean literal_query, QueryResults queryResults, Content content) {
String highlightQueryEscaped;
if (literal_query) {
// literal, treat as non-regex, non-term component query
highlightQueryEscaped = query.getQueryString();
} else {
// construct a Solr query using aggregated terms to get highlighting
// the query is executed later on demand
StringBuilder highlightQuery = new StringBuilder();
if (queryResults.getKeywords().size() == 1) {
// simple case, no need to process subqueries and do special escaping
Keyword term = queryResults.getKeywords().iterator().next();
highlightQuery.append(term.toString());
} else {
// find terms for this content hit
List<String> hitTerms = new ArrayList<>();
for (Keyword keyword : queryResults.getKeywords()) {
for (KeywordHit hit : queryResults.getResults(keyword)) {
if (hit.getContent().equals(content)) {
hitTerms.add(keyword.toString());
break; // go to next term
}
}
}
final int lastTerm = hitTerms.size() - 1;
int curTerm = 0;
for (String term : hitTerms) {
// escape subqueries, they shouldn't be escaped again later
final String termS = KeywordSearchUtil.escapeLuceneQuery(term);
highlightQuery.append("\"");
highlightQuery.append(termS);
highlightQuery.append("\"");
if (lastTerm != curTerm) {
highlightQuery.append(" "); // acts as OR ||
// force HIGHLIGHT_FIELD_REGEX index and stored content
// in each term after first. First term taken care by HighlightedMatchesSource
highlightQuery.append(LuceneQuery.HIGHLIGHT_FIELD_REGEX).append(":");
}
++curTerm;
}
}
// String highlightQueryEscaped =
// KeywordSearchUtil.escapeLuceneQuery(highlightQuery.toString());
highlightQueryEscaped = highlightQuery.toString();
}
return highlightQueryEscaped;
}
private ExecutionResult markSurplusRows(final QueryResults queryResults) {
List<Integer> unmatchedRows = queryResults.getUnmatchedRows();
ExecutionResult result = ExecutionResult.PASS;
for (int unmatchedRow : unmatchedRows) {
List<String> surplusRow = queryResults.getList(fieldNames, unmatchedRow);
int newTableRow = table.addRow(surplusRow);
TestResult testResult = TestResult.fail(surplusRow.get(0), null, "surplus");
table.updateContent(0, newTableRow, testResult);
getTestContext().increment(result);
markMissingFields(surplusRow, newTableRow);
result = ExecutionResult.FAIL;
}
return result;
}
@Override
protected Object doInBackground() throws Exception {
registerWriter(
this); // register (synchronized on class) outside of writerLock to prevent deadlock
final String queryStr = query.getQueryString();
final String queryDisp =
queryStr.length() > QUERY_DISPLAY_LEN
? queryStr.substring(0, QUERY_DISPLAY_LEN - 1) + " ..."
: queryStr;
// block until previous writer is done
// writerLock.lock();
try {
progress =
ProgressHandleFactory.createHandle(
NbBundle.getMessage(
this.getClass(), "KeywordSearchResultFactory.progress.saving", queryDisp),
new Cancellable() {
@Override
public boolean cancel() {
return BlackboardResultWriter.this.cancel(true);
}
});
// Create blackboard artifacts
newArtifacts = hits.writeAllHitsToBlackBoard(progress, null, this, false);
} finally {
finalizeWorker();
}
return null;
}
@Test(expected = IllegalStateException.class)
public void shouldRequireNextOnCursorToBeCalledBeforeGettingValueUsingColumnName() {
results = new QueryResults(columns, statistics, tuples, context.getProblems(), null);
Cursor cursor = results.getCursor();
assertThat(cursor.hasNext(), is(true));
cursor.getValue("colA");
}
protected TestResult markField(int tableRow, int matchedRow, int col, QueryResults queryResults) {
if (col >= fieldNames.size()) return null; // ignore strange table geometry.
String fieldName = fieldNames.get(col);
String actualValue = queryResults.getCell(fieldName, matchedRow);
String expectedValue = table.getCellContents(col, tableRow);
TestResult testResult;
if (actualValue == null)
testResult = TestResult.fail(String.format("field %s not present", fieldName), expectedValue);
else if (expectedValue == null || expectedValue.length() == 0)
testResult = TestResult.ignore(actualValue);
else {
testResult = matchMessage(actualValue, expectedValue);
// if (testResult != null)
// table.substitute(col, tableRow, replaceSymbolsWithFullExpansion(message));
// else
// table.substitute(col, tableRow, replaceSymbolsWithFullExpansion(expectedValue));
// else
if (testResult == null)
testResult = TestResult.fail(actualValue, replaceSymbolsWithFullExpansion(expectedValue));
else if (testResult.getExecutionResult() == ExecutionResult.PASS)
testResult = markMatch(tableRow, matchedRow, col, testResult.getMessage());
}
table.updateContent(col, tableRow, testResult);
getTestContext().increment(testResult.getExecutionResult());
return testResult;
}
@Test
public void shouldPrintToStringAllResults() {
results = new QueryResults(columns, statistics, tuples, context.getProblems(), null);
List<String> lines = StringUtil.splitLines(results.toString());
assertThat(
lines.size(), is(tuples.size() + 4)); // = delim + header + delim + (...lines...) + delim
}
/**
* This method returns a collection of KeywordHits with lowest SolrObjectID- Chunk-ID combination.
* The output generated is consistent across multiple runs.
*
* @param queryResults QueryResult object
* @return A consistent collection of keyword hits
*/
Collection<KeywordHit> getOneHitPerObject(QueryResults queryResults) {
HashMap<Long, KeywordHit> hits = new HashMap<Long, KeywordHit>();
for (Keyword keyWord : queryResults.getKeywords()) {
for (KeywordHit hit : queryResults.getResults(keyWord)) {
// add hit with lowest SolrObjectID-Chunk-ID combination.
if (!hits.containsKey(hit.getSolrObjectId())) {
hits.put(hit.getSolrObjectId(), hit);
} else {
if (hit.getChunkId() < hits.get(hit.getSolrObjectId()).getChunkId()) {
hits.put(hit.getSolrObjectId(), hit);
}
}
}
}
return hits.values();
}
@Test
public void shouldPrintToStringBuilderOnlyFirstLinesOfResults() {
results = new QueryResults(columns, statistics, tuples, context.getProblems(), null);
StringBuilder sb = new StringBuilder();
results.toString(typeSystem, sb, 1);
List<String> lines = StringUtil.splitLines(sb.toString());
assertThat(lines.size(), is(1 + 4)); // = delim + header + delim + (...lines...) + delim
}
@Test
public void shouldPrintToStringBuilderAllResultsWhenMaxRowParameterIsLargerThanNumberOfTuples() {
tuples.clear();
results = new QueryResults(columns, statistics, tuples, context.getProblems(), null);
StringBuilder sb = new StringBuilder();
results.toString(typeSystem, sb, 3);
List<String> lines = StringUtil.splitLines(sb.toString());
assertThat(
lines.size(), is(tuples.size() + 4)); // = delim + header + delim + (...lines...) + delim
}
protected void scanRowForMatch(int tableRow, QueryResults queryResults) {
int matchedRow = queryResults.findBestMatch(tableRow);
if (matchedRow == -1) {
replaceAllvariablesInRow(tableRow);
TestResult testResult = TestResult.fail(null, table.getCellContents(0, tableRow), "missing");
table.updateContent(0, tableRow, testResult);
getTestContext().increment(testResult.getExecutionResult());
} else {
markFieldsInMatchedRow(tableRow, matchedRow, queryResults);
}
}
/*
* @see org.eclipse.birt.data.engine.api.IResultIterator#getSecondaryIterator(java.lang.String,
* org.mozilla.javascript.Scriptable)
*/
public IResultIterator getSecondaryIterator(String subQueryName, Scriptable scope)
throws DataException {
String queryResultsID = null;
String baseQueryResultsID = null;
int rootIdIdex = queryResultID.indexOf("/");
if (rootIdIdex > -1) baseQueryResultsID = queryResultID.substring(0, rootIdIdex);
else baseQueryResultsID = queryResultID;
if (this.subQueryName == null) {
queryResultsID = queryResultID;
} else {
queryResultsID = queryResultID + "/" + this.subQueryName + "/" + this.subQueryIndex;
}
QueryResults queryResults = null;
try {
queryResults =
new QueryResults(
tempDir,
context,
baseQueryResultsID,
queryResultsID,
this.getResultMetaData(),
subQueryName,
this.exprResultSet.getCurrentIndex(),
this.queryResults,
null);
} catch (Exception e) {
throw new DataException(ResourceConstants.RD_LOAD_ERROR, e, "Subquery");
}
try {
ResultIterator ri = (ResultIterator) queryResults.getResultIterator();
ri.setSubQueryName(subQueryName);
return ri;
} catch (BirtException e) {
throw new DataException(ResourceConstants.RD_LOAD_ERROR, e, "Subquery");
}
}
@Override
public QueryResults performQuery() throws NoOpenCoreException {
/*
* Execute the regex query to get a list of terms that match the regex.
* Note that the field that is being searched is tokenized based on
* whitespace.
*/
// create the query
final SolrQuery q = new SolrQuery();
q.setRequestHandler(TERMS_HANDLER);
q.setTerms(true);
q.setTermsRegexFlag(CASE_INSENSITIVE);
q.setTermsRegex(escapedQuery);
q.addTermsField(TERMS_SEARCH_FIELD);
q.setTimeAllowed(TERMS_TIMEOUT);
q.setShowDebugInfo(DEBUG);
q.setTermsLimit(MAX_TERMS_RESULTS);
LOGGER.log(Level.INFO, "Query: {0}", q.toString()); // NON-NLS
// execute the query
List<Term> terms = null;
try {
terms = KeywordSearch.getServer().queryTerms(q).getTerms(TERMS_SEARCH_FIELD);
} catch (KeywordSearchModuleException ex) {
LOGGER.log(
Level.SEVERE,
"Error executing the regex terms query: " + keyword.getQuery(),
ex); // NON-NLS
// TODO: this is almost certainly wrong and guaranteed to throw a NPE at some point!!!!
}
/*
* For each term that matched the regex, query for full set of document
* hits for that term.
*/
QueryResults results = new QueryResults(this, keywordList);
int resultSize = 0;
for (Term term : terms) {
final String termStr = KeywordSearchUtil.escapeLuceneQuery(term.getTerm());
if (keyword.getType() == ATTRIBUTE_TYPE.TSK_CARD_NUMBER) {
// If the keyword is a credit card number, pass it through luhn validator
Matcher matcher = CCN_PATTERN.matcher(term.getTerm());
matcher.find();
final String ccn = CharMatcher.anyOf(" -").removeFrom(matcher.group("ccn"));
if (false == LUHN_CHECK.isValid(ccn)) {
continue; // if the hit does not pass the luhn check, skip it.
}
}
/*
* Note: we can't set filter query on terms query but setting filter
* query on fileResults query will yield the same result
*/
LuceneQuery filesQuery = new LuceneQuery(keywordList, new Keyword(termStr, true));
filters.forEach(filesQuery::addFilter);
try {
QueryResults fileQueryResults = filesQuery.performQuery();
Set<KeywordHit> filesResults = new HashSet<>();
for (Keyword key : fileQueryResults.getKeywords()) { // flatten results into a single list
List<KeywordHit> keyRes = fileQueryResults.getResults(key);
resultSize += keyRes.size();
filesResults.addAll(keyRes);
}
results.addResult(new Keyword(term.getTerm(), false), new ArrayList<>(filesResults));
} catch (NoOpenCoreException | RuntimeException e) {
LOGGER.log(Level.WARNING, "Error executing Solr query,", e); // NON-NLS
throw e;
}
}
// TODO limit how many results we store, not to hit memory limits
LOGGER.log(Level.INFO, "Regex # results: {0}", resultSize); // NON-NLS
return results;
}
BlackboardResultWriter(QueryResults hits, String listName) {
this.hits = hits;
this.query = hits.getQuery();
this.listName = listName;
}
@Test
public void shouldReturnSameProblemsObjectAsInQueryContext() {
assertThat(results.getProblems(), is(sameInstance(context.getProblems())));
}
@Test
public void shouldReturnSameStatisticsPassedIntoConstructor() {
assertThat(results.getStatistics(), is(sameInstance(statistics)));
}
@Test
public void shouldReturnSameColumnsPassedIntoConstructor() {
assertThat(results.getColumns(), is(sameInstance(columns)));
}
@Test
public void shouldReturnSameTuplesListPassedIntoConstructor() {
results = new QueryResults(columns, statistics, tuples, context.getProblems(), null);
assertThat(results.getTuples(), is(sameInstance(tuples)));
}
@Test
public void shouldHaveNoTuplesIfConstructedWithNoTuples() {
assertThat(results.getTuples().isEmpty(), is(true));
assertThat(results.getCursor().hasNext(), is(false));
}
