@Test public void shouldReturnCursorThatAccessesTuples() { results = new QueryResults(columns, statistics, tuples, context.getProblems(), null); Cursor cursor = results.getCursor(); Iterator<Object[]> expectedIter = tuples.iterator(); int rowNumber = 0; while (cursor.hasNext() && expectedIter.hasNext()) { cursor.next(); Object[] tuple = expectedIter.next(); // Check the column values by column name and index ... for (Column column : results.getColumns().getColumns()) { String columnName = column.getColumnName(); int columnIndex = columns.getColumnIndexForName(columnName); assertThat(cursor.getValue(columnName), is(tuple[columnIndex])); assertThat(cursor.getValue(columnIndex), is(tuple[columnIndex])); // Get the location for this column ... int locationIndex = columns.getLocationIndex(column.selectorName().name()); Location location = (Location) tuple[locationIndex]; assertThat(cursor.getLocation(columnIndex), is(location)); } // Check the locations by selector name and index ... for (String selectorName : results.getColumns().getSelectorNames()) { int locationIndex = columns.getLocationIndex(selectorName); Location location = (Location) tuple[locationIndex]; assertThat(cursor.getLocation(selectorName), is(location)); assertThat(location.getPath(), is(notNullValue())); } // Check the row index ... assertThat(cursor.getRowIndex(), is(rowNumber++)); } assertThat(cursor.hasNext(), is(false)); assertThat(expectedIter.hasNext(), is(false)); }
@Test public void shouldHaveNoTuplesIfConstructedWithEmptyTuplesList() { tuples.clear(); results = new QueryResults(columns, statistics, tuples, context.getProblems(), null); assertThat(results.getTuples().isEmpty(), is(true)); assertThat(results.getCursor().hasNext(), is(false)); }
@Test public void shouldReturnMutableTuplesList() { results = new QueryResults(columns, statistics, tuples, context.getProblems(), null); assertThat(results.getTuples().isEmpty(), is(false)); results.getTuples().clear(); assertThat(results.getTuples().isEmpty(), is(true)); assertThat(tuples.isEmpty(), is(true)); }
/** * Return the string used to later have SOLR highlight the document with. * * @param query * @param literal_query * @param queryResults * @param file * @return */ private String getHighlightQuery( KeywordSearchQuery query, boolean literal_query, QueryResults queryResults, Content content) { String highlightQueryEscaped; if (literal_query) { // literal, treat as non-regex, non-term component query highlightQueryEscaped = query.getQueryString(); } else { // construct a Solr query using aggregated terms to get highlighting // the query is executed later on demand StringBuilder highlightQuery = new StringBuilder(); if (queryResults.getKeywords().size() == 1) { // simple case, no need to process subqueries and do special escaping Keyword term = queryResults.getKeywords().iterator().next(); highlightQuery.append(term.toString()); } else { // find terms for this content hit List<String> hitTerms = new ArrayList<>(); for (Keyword keyword : queryResults.getKeywords()) { for (KeywordHit hit : queryResults.getResults(keyword)) { if (hit.getContent().equals(content)) { hitTerms.add(keyword.toString()); break; // go to next term } } } final int lastTerm = hitTerms.size() - 1; int curTerm = 0; for (String term : hitTerms) { // escape subqueries, they shouldn't be escaped again later final String termS = KeywordSearchUtil.escapeLuceneQuery(term); highlightQuery.append("\""); highlightQuery.append(termS); highlightQuery.append("\""); if (lastTerm != curTerm) { highlightQuery.append(" "); // acts as OR || // force HIGHLIGHT_FIELD_REGEX index and stored content // in each term after first. First term taken care by HighlightedMatchesSource highlightQuery.append(LuceneQuery.HIGHLIGHT_FIELD_REGEX).append(":"); } ++curTerm; } } // String highlightQueryEscaped = // KeywordSearchUtil.escapeLuceneQuery(highlightQuery.toString()); highlightQueryEscaped = highlightQuery.toString(); } return highlightQueryEscaped; }
private ExecutionResult markSurplusRows(final QueryResults queryResults) { List<Integer> unmatchedRows = queryResults.getUnmatchedRows(); ExecutionResult result = ExecutionResult.PASS; for (int unmatchedRow : unmatchedRows) { List<String> surplusRow = queryResults.getList(fieldNames, unmatchedRow); int newTableRow = table.addRow(surplusRow); TestResult testResult = TestResult.fail(surplusRow.get(0), null, "surplus"); table.updateContent(0, newTableRow, testResult); getTestContext().increment(result); markMissingFields(surplusRow, newTableRow); result = ExecutionResult.FAIL; } return result; }
@Override protected Object doInBackground() throws Exception { registerWriter( this); // register (synchronized on class) outside of writerLock to prevent deadlock final String queryStr = query.getQueryString(); final String queryDisp = queryStr.length() > QUERY_DISPLAY_LEN ? queryStr.substring(0, QUERY_DISPLAY_LEN - 1) + " ..." : queryStr; // block until previous writer is done // writerLock.lock(); try { progress = ProgressHandleFactory.createHandle( NbBundle.getMessage( this.getClass(), "KeywordSearchResultFactory.progress.saving", queryDisp), new Cancellable() { @Override public boolean cancel() { return BlackboardResultWriter.this.cancel(true); } }); // Create blackboard artifacts newArtifacts = hits.writeAllHitsToBlackBoard(progress, null, this, false); } finally { finalizeWorker(); } return null; }
@Test(expected = IllegalStateException.class) public void shouldRequireNextOnCursorToBeCalledBeforeGettingValueUsingColumnName() { results = new QueryResults(columns, statistics, tuples, context.getProblems(), null); Cursor cursor = results.getCursor(); assertThat(cursor.hasNext(), is(true)); cursor.getValue("colA"); }
protected TestResult markField(int tableRow, int matchedRow, int col, QueryResults queryResults) { if (col >= fieldNames.size()) return null; // ignore strange table geometry. String fieldName = fieldNames.get(col); String actualValue = queryResults.getCell(fieldName, matchedRow); String expectedValue = table.getCellContents(col, tableRow); TestResult testResult; if (actualValue == null) testResult = TestResult.fail(String.format("field %s not present", fieldName), expectedValue); else if (expectedValue == null || expectedValue.length() == 0) testResult = TestResult.ignore(actualValue); else { testResult = matchMessage(actualValue, expectedValue); // if (testResult != null) // table.substitute(col, tableRow, replaceSymbolsWithFullExpansion(message)); // else // table.substitute(col, tableRow, replaceSymbolsWithFullExpansion(expectedValue)); // else if (testResult == null) testResult = TestResult.fail(actualValue, replaceSymbolsWithFullExpansion(expectedValue)); else if (testResult.getExecutionResult() == ExecutionResult.PASS) testResult = markMatch(tableRow, matchedRow, col, testResult.getMessage()); } table.updateContent(col, tableRow, testResult); getTestContext().increment(testResult.getExecutionResult()); return testResult; }
@Test public void shouldPrintToStringAllResults() { results = new QueryResults(columns, statistics, tuples, context.getProblems(), null); List<String> lines = StringUtil.splitLines(results.toString()); assertThat( lines.size(), is(tuples.size() + 4)); // = delim + header + delim + (...lines...) + delim }
/** * This method returns a collection of KeywordHits with lowest SolrObjectID- Chunk-ID combination. * The output generated is consistent across multiple runs. * * @param queryResults QueryResult object * @return A consistent collection of keyword hits */ Collection<KeywordHit> getOneHitPerObject(QueryResults queryResults) { HashMap<Long, KeywordHit> hits = new HashMap<Long, KeywordHit>(); for (Keyword keyWord : queryResults.getKeywords()) { for (KeywordHit hit : queryResults.getResults(keyWord)) { // add hit with lowest SolrObjectID-Chunk-ID combination. if (!hits.containsKey(hit.getSolrObjectId())) { hits.put(hit.getSolrObjectId(), hit); } else { if (hit.getChunkId() < hits.get(hit.getSolrObjectId()).getChunkId()) { hits.put(hit.getSolrObjectId(), hit); } } } } return hits.values(); }
@Test public void shouldPrintToStringBuilderOnlyFirstLinesOfResults() { results = new QueryResults(columns, statistics, tuples, context.getProblems(), null); StringBuilder sb = new StringBuilder(); results.toString(typeSystem, sb, 1); List<String> lines = StringUtil.splitLines(sb.toString()); assertThat(lines.size(), is(1 + 4)); // = delim + header + delim + (...lines...) + delim }
@Test public void shouldPrintToStringBuilderAllResultsWhenMaxRowParameterIsLargerThanNumberOfTuples() { tuples.clear(); results = new QueryResults(columns, statistics, tuples, context.getProblems(), null); StringBuilder sb = new StringBuilder(); results.toString(typeSystem, sb, 3); List<String> lines = StringUtil.splitLines(sb.toString()); assertThat( lines.size(), is(tuples.size() + 4)); // = delim + header + delim + (...lines...) + delim }
protected void scanRowForMatch(int tableRow, QueryResults queryResults) { int matchedRow = queryResults.findBestMatch(tableRow); if (matchedRow == -1) { replaceAllvariablesInRow(tableRow); TestResult testResult = TestResult.fail(null, table.getCellContents(0, tableRow), "missing"); table.updateContent(0, tableRow, testResult); getTestContext().increment(testResult.getExecutionResult()); } else { markFieldsInMatchedRow(tableRow, matchedRow, queryResults); } }
/* * @see org.eclipse.birt.data.engine.api.IResultIterator#getSecondaryIterator(java.lang.String, * org.mozilla.javascript.Scriptable) */ public IResultIterator getSecondaryIterator(String subQueryName, Scriptable scope) throws DataException { String queryResultsID = null; String baseQueryResultsID = null; int rootIdIdex = queryResultID.indexOf("/"); if (rootIdIdex > -1) baseQueryResultsID = queryResultID.substring(0, rootIdIdex); else baseQueryResultsID = queryResultID; if (this.subQueryName == null) { queryResultsID = queryResultID; } else { queryResultsID = queryResultID + "/" + this.subQueryName + "/" + this.subQueryIndex; } QueryResults queryResults = null; try { queryResults = new QueryResults( tempDir, context, baseQueryResultsID, queryResultsID, this.getResultMetaData(), subQueryName, this.exprResultSet.getCurrentIndex(), this.queryResults, null); } catch (Exception e) { throw new DataException(ResourceConstants.RD_LOAD_ERROR, e, "Subquery"); } try { ResultIterator ri = (ResultIterator) queryResults.getResultIterator(); ri.setSubQueryName(subQueryName); return ri; } catch (BirtException e) { throw new DataException(ResourceConstants.RD_LOAD_ERROR, e, "Subquery"); } }
@Override public QueryResults performQuery() throws NoOpenCoreException { /* * Execute the regex query to get a list of terms that match the regex. * Note that the field that is being searched is tokenized based on * whitespace. */ // create the query final SolrQuery q = new SolrQuery(); q.setRequestHandler(TERMS_HANDLER); q.setTerms(true); q.setTermsRegexFlag(CASE_INSENSITIVE); q.setTermsRegex(escapedQuery); q.addTermsField(TERMS_SEARCH_FIELD); q.setTimeAllowed(TERMS_TIMEOUT); q.setShowDebugInfo(DEBUG); q.setTermsLimit(MAX_TERMS_RESULTS); LOGGER.log(Level.INFO, "Query: {0}", q.toString()); // NON-NLS // execute the query List<Term> terms = null; try { terms = KeywordSearch.getServer().queryTerms(q).getTerms(TERMS_SEARCH_FIELD); } catch (KeywordSearchModuleException ex) { LOGGER.log( Level.SEVERE, "Error executing the regex terms query: " + keyword.getQuery(), ex); // NON-NLS // TODO: this is almost certainly wrong and guaranteed to throw a NPE at some point!!!! } /* * For each term that matched the regex, query for full set of document * hits for that term. */ QueryResults results = new QueryResults(this, keywordList); int resultSize = 0; for (Term term : terms) { final String termStr = KeywordSearchUtil.escapeLuceneQuery(term.getTerm()); if (keyword.getType() == ATTRIBUTE_TYPE.TSK_CARD_NUMBER) { // If the keyword is a credit card number, pass it through luhn validator Matcher matcher = CCN_PATTERN.matcher(term.getTerm()); matcher.find(); final String ccn = CharMatcher.anyOf(" -").removeFrom(matcher.group("ccn")); if (false == LUHN_CHECK.isValid(ccn)) { continue; // if the hit does not pass the luhn check, skip it. } } /* * Note: we can't set filter query on terms query but setting filter * query on fileResults query will yield the same result */ LuceneQuery filesQuery = new LuceneQuery(keywordList, new Keyword(termStr, true)); filters.forEach(filesQuery::addFilter); try { QueryResults fileQueryResults = filesQuery.performQuery(); Set<KeywordHit> filesResults = new HashSet<>(); for (Keyword key : fileQueryResults.getKeywords()) { // flatten results into a single list List<KeywordHit> keyRes = fileQueryResults.getResults(key); resultSize += keyRes.size(); filesResults.addAll(keyRes); } results.addResult(new Keyword(term.getTerm(), false), new ArrayList<>(filesResults)); } catch (NoOpenCoreException | RuntimeException e) { LOGGER.log(Level.WARNING, "Error executing Solr query,", e); // NON-NLS throw e; } } // TODO limit how many results we store, not to hit memory limits LOGGER.log(Level.INFO, "Regex # results: {0}", resultSize); // NON-NLS return results; }
BlackboardResultWriter(QueryResults hits, String listName) { this.hits = hits; this.query = hits.getQuery(); this.listName = listName; }
@Test public void shouldReturnSameProblemsObjectAsInQueryContext() { assertThat(results.getProblems(), is(sameInstance(context.getProblems()))); }
@Test public void shouldReturnSameStatisticsPassedIntoConstructor() { assertThat(results.getStatistics(), is(sameInstance(statistics))); }
@Test public void shouldReturnSameColumnsPassedIntoConstructor() { assertThat(results.getColumns(), is(sameInstance(columns))); }
@Test public void shouldReturnSameTuplesListPassedIntoConstructor() { results = new QueryResults(columns, statistics, tuples, context.getProblems(), null); assertThat(results.getTuples(), is(sameInstance(tuples))); }
@Test public void shouldHaveNoTuplesIfConstructedWithNoTuples() { assertThat(results.getTuples().isEmpty(), is(true)); assertThat(results.getCursor().hasNext(), is(false)); }