/** * This is best effort only: the PhraseQuery may contain multiple terms at the same position * (think synonyms) or gaps (think stopwords) and it's in this case impossible to translate it * into a correct ElasticsearchQuery. */ private static JsonObject convertPhraseQuery(PhraseQuery query) { Term[] terms = query.getTerms(); if (terms.length == 0) { throw LOG.cannotQueryOnEmptyPhraseQuery(); } String field = terms[0].field(); // phrase queries are only supporting one field StringBuilder phrase = new StringBuilder(); for (Term term : terms) { phrase.append(" ").append(term.text()); } JsonObject phraseQuery = JsonBuilder.object() .add( "match_phrase", JsonBuilder.object() .add( field, JsonBuilder.object() .addProperty("query", phrase.toString().trim()) .addProperty("slop", query.getSlop()) .addProperty("boost", query.getBoost()))) .build(); return wrapQueryForNestedIfRequired(field, phraseQuery); }
public void testPositionIncrement() throws Exception { boolean dflt = StopFilter.getEnablePositionIncrementsDefault(); StopFilter.setEnablePositionIncrementsDefault(true); try { QueryParserWrapper qp = new QueryParserWrapper("a", new StopAnalyzer(new String[] {"the", "in", "are", "this"})); qp.setEnablePositionIncrements(true); String qtxt = "\"the words in poisitions pos02578 are stopped in this phrasequery\""; // 0 2 5 7 8 int expectedPositions[] = {1, 3, 4, 6, 9}; PhraseQuery pq = (PhraseQuery) qp.parse(qtxt); // System.out.println("Query text: "+qtxt); // System.out.println("Result: "+pq); Term t[] = pq.getTerms(); int pos[] = pq.getPositions(); for (int i = 0; i < t.length; i++) { // System.out.println(i+". "+t[i]+" pos: "+pos[i]); assertEquals( "term " + i + " = " + t[i] + " has wrong term-position!", expectedPositions[i], pos[i]); } } finally { StopFilter.setEnablePositionIncrementsDefault(dflt); } }
public Query createPhrasePrefixQuery( String field, String queryText, int phraseSlop, int maxExpansions) { final Query query = createFieldQuery(getAnalyzer(), Occur.MUST, field, queryText, true, phraseSlop); final MultiPhrasePrefixQuery prefixQuery = new MultiPhrasePrefixQuery(); prefixQuery.setMaxExpansions(maxExpansions); prefixQuery.setSlop(phraseSlop); if (query instanceof PhraseQuery) { PhraseQuery pq = (PhraseQuery) query; Term[] terms = pq.getTerms(); int[] positions = pq.getPositions(); for (int i = 0; i < terms.length; i++) { prefixQuery.add(new Term[] {terms[i]}, positions[i]); } return prefixQuery; } else if (query instanceof MultiPhraseQuery) { MultiPhraseQuery pq = (MultiPhraseQuery) query; Term[][] terms = pq.getTermArrays(); int[] positions = pq.getPositions(); for (int i = 0; i < terms.length; i++) { prefixQuery.add(terms[i], positions[i]); } return prefixQuery; } else if (query instanceof TermQuery) { prefixQuery.add(((TermQuery) query).getTerm()); return prefixQuery; } return query; }
/** * Construct a span phrase query. TODO While we have code set up to check for wildcard characters * in the query, they have already been stripped so this will never happen. This is one thing that * would need to be addressed. * * @param scope * @param q * @return */ private SpanQuery buildPhraseQuery(String scope, Query q) { PhraseQuery pq = (PhraseQuery) q; Term[] terms = pq.getTerms(); ArrayList<SpanQuery> spans = new ArrayList<SpanQuery>(); spans.add( new SpanTermQuery(new Term(scope, getTag(scope, terms[0].field(), TAG_TYPE.BEG_TAG)))); spans.add( new SpanTermQuery(new Term(scope, getTag(scope, terms[0].field(), TAG_TYPE.END_TAG)))); ArrayList<SpanQuery> phraseSpan = new ArrayList<SpanQuery>(); for (int i = 0; i < terms.length; i++) { String term = terms[i].text(); if (term.endsWith("*")) { WildcardQuery wildcard = new WildcardQuery(new Term(scope, term)); phraseSpan.add(new SpanMultiTermQueryWrapper<WildcardQuery>(wildcard)); } else if (term.contains(singleCs) || term.contains(multiCs)) { WildcardQuery wildcard = new WildcardQuery(new Term(scope, term)); phraseSpan.add(new SpanMultiTermQueryWrapper<WildcardQuery>(wildcard)); } else { phraseSpan.add(new SpanTermQuery(new Term(scope, term))); } } spans.add( new SpanNearQuery( phraseSpan.toArray(new SpanQuery[phraseSpan.size()]), phraseSpan.size(), true)); return new SpanBetweenQuery(spans.toArray(new SpanQuery[spans.size()])); }
protected PhraseQuery toPhraseQuery(List<BytesRef> bytesRefs, String field) { PhraseQuery phraseQuery = new PhraseQuery(); for (BytesRef bytesRef : bytesRefs) { phraseQuery.add(new Term(field, bytesRef)); } return phraseQuery; }
public void testSpanNearQuery() throws Exception { SpanQuery[] quick_brown_dog = new SpanQuery[] {quick, brown, dog}; SpanNearQuery snq = new SpanNearQuery(quick_brown_dog, 0, true); // #1 assertNoMatches(snq); dumpSpans(snq); snq = new SpanNearQuery(quick_brown_dog, 4, true); // #2 assertNoMatches(snq); dumpSpans(snq); snq = new SpanNearQuery(quick_brown_dog, 5, true); // #3 assertOnlyBrownFox(snq); dumpSpans(snq); // interesting - even a sloppy phrase query would require // more slop to match snq = new SpanNearQuery(new SpanQuery[] {lazy, fox}, 3, false); // #4 assertOnlyBrownFox(snq); dumpSpans(snq); PhraseQuery pq = new PhraseQuery(); // #5 pq.add(new Term("f", "lazy")); // #5 pq.add(new Term("f", "fox")); // #5 pq.setSlop(4); // #5 assertNoMatches(pq); pq.setSlop(5); // #6 assertOnlyBrownFox(pq); // #6 }
protected void smokeTestSearcher(IndexSearcher s) throws Exception { runQuery(s, new TermQuery(new Term("body", "united"))); runQuery(s, new TermQuery(new Term("titleTokenized", "states"))); PhraseQuery pq = new PhraseQuery(); pq.add(new Term("body", "united")); pq.add(new Term("body", "states")); runQuery(s, pq); }
private static PhraseQuery makePhraseQuery(String terms) { PhraseQuery query = new PhraseQuery(); String[] t = terms.split(" +"); for (int i = 0; i < t.length; i++) { query.add(new Term("f", t[i])); } return query; }
protected Query pq(float boost, int slop, String field, String... texts) { PhraseQuery query = new PhraseQuery(); for (String text : texts) { query.add(new Term(field, text)); } query.setBoost(boost); query.setSlop(slop); return query; }
public void testCJKPhrase() throws Exception { // individual CJK chars as terms StandardAnalyzer analyzer = new StandardAnalyzer(TEST_VERSION_CURRENT); PhraseQuery expected = new PhraseQuery(); expected.add(new Term("field", "中")); expected.add(new Term("field", "国")); assertEquals(expected, getQuery("\"中国\"", analyzer)); }
protected Query eq(String field, String[] terms, boolean ignoreCase) { if (terms.length > 1) { PhraseQuery pq = new PhraseQuery(); for (String s : terms) { pq.add(new Term(field, s)); } return pq; } return new TermQuery(new Term(field, terms[0])); }
public void testAutoGeneratePhraseQueriesOn() throws Exception { // individual CJK chars as terms StandardAnalyzer analyzer = new StandardAnalyzer(TEST_VERSION_CURRENT); PhraseQuery expected = new PhraseQuery(); expected.add(new Term("field", "中")); expected.add(new Term("field", "国")); QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, "field", analyzer); parser.setAutoGeneratePhraseQueries(true); assertEquals(expected, parser.parse("中国")); }
@Override public Query construct(IndexEnvironment env, Map<String, String[]> querySource) throws ParseException { if ("1".equals(StringTools.arrayToString(querySource.get("queryversion"), ""))) { // preserving old stuff: // 1. all lucene special chars to be quoted // 2. if "wholewords" is "on" or "true" -> don't add *_*, otherwise add *_* BooleanQuery result = new BooleanQuery(); String wholeWords = StringTools.arrayToString(querySource.get("wholewords"), ""); boolean useWildcards = !(null != wholeWords && StringTools.stringToBoolean(wholeWords)); for (Map.Entry<String, String[]> queryItem : querySource.entrySet()) { String field = queryItem.getKey(); if (env.fields.containsKey(field) && queryItem.getValue().length > 0) { for (String value : queryItem.getValue()) { if (null != value) { value = value.trim().toLowerCase(); if (0 != value.length()) { if ("keywords".equals(field) && ACCESSION_REGEX.test(value)) { result.add(new TermQuery(new Term("accession", value)), BooleanClause.Occur.MUST); } else if ("keywords".equals(field) && '"' == value.charAt(0) && '"' == value.charAt(value.length() - 1)) { value = value.substring(1, value.length() - 1); PhraseQuery q = new PhraseQuery(); String[] tokens = value.split("\\s+"); for (String token : tokens) { q.add(new Term(field, token)); } result.add(q, BooleanClause.Occur.MUST); } else { String[] tokens = value.split("\\s+"); for (String token : tokens) { // we use wildcards for keywords depending on "wholewords" switch, // *ALWAYS* for other fields, *NEVER* for user id and accession or boolean // fields Query q = !"boolean".equals(env.fields.get(field).type) && !" userid accession ".contains(" " + field + " ") && (useWildcards || (!" keywords ".contains(" " + field + " "))) ? new WildcardQuery(new Term(field, "*" + token + "*")) : new TermQuery(new Term(field, token)); result.add(q, BooleanClause.Occur.MUST); } } } } } } } return result; } else { return super.construct(env, querySource); } }
@Test public void testLotsOfPhrases() throws IOException { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter( dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer( random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET))); FieldType type = new FieldType(TextField.TYPE_STORED); type.setStoreTermVectorOffsets(true); type.setStoreTermVectorPositions(true); type.setStoreTermVectors(true); type.freeze(); String[] terms = {"org", "apache", "lucene"}; int iters = atLeast(1000); StringBuilder builder = new StringBuilder(); for (int i = 0; i < iters; i++) { builder.append(terms[random().nextInt(terms.length)]).append(" "); if (random().nextInt(6) == 3) { builder.append("elasticsearch").append(" "); } } Document doc = new Document(); Field field = new Field("field", builder.toString(), type); doc.add(field); writer.addDocument(doc); PhraseQuery query = new PhraseQuery(); query.add(new Term("field", "org")); query.add(new Term("field", "apache")); query.add(new Term("field", "lucene")); XFastVectorHighlighter highlighter = new XFastVectorHighlighter(); IndexReader reader = DirectoryReader.open(writer, true); IndexSearcher searcher = newSearcher(reader); TopDocs hits = searcher.search(query, 10); assertEquals(1, hits.totalHits); XFieldQuery fieldQuery = highlighter.getFieldQuery(query, reader); String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[0].doc, "field", 1000, 1); for (int i = 0; i < bestFragments.length; i++) { String result = bestFragments[i].replaceAll("<b>org apache lucene</b>", "FOOBAR"); assertFalse(result.contains("org apache lucene")); } reader.close(); writer.close(); dir.close(); }
BooleanClause partialMatch(SearchTerm s, int slop) { String[] phrase = getPhrase(s.getValue().toLowerCase().trim()); PhraseQuery query = new PhraseQuery(); BooleanClause partialMatchClause = null; query.setSlop(slop); for (int i = 0; i < phrase.length; i++) { query.add(new Term(s.getFieldName(), phrase[i].toLowerCase().trim())); } if (s.getOperator().equalsIgnoreCase("=")) { partialMatchClause = new BooleanClause(query, BooleanClause.Occur.MUST); } else if (s.getOperator().equalsIgnoreCase("-")) { partialMatchClause = new BooleanClause(query, BooleanClause.Occur.MUST_NOT); } return partialMatchClause; }
private float checkPhraseQuery(Document doc, PhraseQuery query, int slop, int expectedNumResults) throws Exception { query.setSlop(slop); Directory ramDir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random, ramDir, new MockAnalyzer(MockTokenizer.WHITESPACE, false)); writer.addDocument(doc); IndexReader reader = writer.getReader(); IndexSearcher searcher = newSearcher(reader); TopDocs td = searcher.search(query, null, 10); // System.out.println("slop: "+slop+" query: "+query+" doc: "+doc+" Expecting number of hits: // "+expectedNumResults+" maxScore="+td.getMaxScore()); assertEquals( "slop: " + slop + " query: " + query + " doc: " + doc + " Wrong number of hits", expectedNumResults, td.totalHits); // QueryUtils.check(query,searcher); writer.close(); searcher.close(); reader.close(); ramDir.close(); return td.getMaxScore(); }
public void testToQueryPhraseQueryBoostAndSlop() throws IOException { assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0); QueryStringQueryBuilder queryStringQueryBuilder = new QueryStringQueryBuilder("\"test phrase\"~2").field(STRING_FIELD_NAME, 5f); Query query = queryStringQueryBuilder.toQuery(createShardContext()); assertThat(query, instanceOf(DisjunctionMaxQuery.class)); DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query; assertThat(disjunctionMaxQuery.getDisjuncts().size(), equalTo(1)); assertThat(disjunctionMaxQuery.getDisjuncts().get(0), instanceOf(BoostQuery.class)); BoostQuery boostQuery = (BoostQuery) disjunctionMaxQuery.getDisjuncts().get(0); assertThat(boostQuery.getBoost(), equalTo(5f)); assertThat(boostQuery.getQuery(), instanceOf(PhraseQuery.class)); PhraseQuery phraseQuery = (PhraseQuery) boostQuery.getQuery(); assertThat(phraseQuery.getSlop(), Matchers.equalTo(2)); assertThat(phraseQuery.getTerms().length, equalTo(2)); }
public void testToQueryPhraseQuery() throws IOException { assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0); Query query = queryStringQuery("\"term1 term2\"") .defaultField(STRING_FIELD_NAME) .phraseSlop(3) .toQuery(createShardContext()); assertThat(query, instanceOf(DisjunctionMaxQuery.class)); DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query; assertThat(disjunctionMaxQuery.getDisjuncts().size(), equalTo(1)); assertThat(disjunctionMaxQuery.getDisjuncts().get(0), instanceOf(PhraseQuery.class)); PhraseQuery phraseQuery = (PhraseQuery) disjunctionMaxQuery.getDisjuncts().get(0); assertThat(phraseQuery.getTerms().length, equalTo(2)); assertThat(phraseQuery.getTerms()[0], equalTo(new Term(STRING_FIELD_NAME, "term1"))); assertThat(phraseQuery.getTerms()[1], equalTo(new Term(STRING_FIELD_NAME, "term2"))); assertThat(phraseQuery.getSlop(), equalTo(3)); }
protected Taxon[] findTaxon(String fieldName1, String fieldValue) throws IOException { Taxon[] terms = new TaxonImpl[0]; if (StringUtils.isNotBlank(fieldValue) && indexSearcher != null) { PhraseQuery query = new PhraseQuery(); query.add(new Term(fieldName1, fieldValue)); int maxHits = 3; TopDocs docs = indexSearcher.search(query, maxHits); if (docs.totalHits > 0) { terms = new TaxonImpl[docs.totalHits]; for (int i = 0; i < docs.totalHits && i < maxHits; i++) { ScoreDoc scoreDoc = docs.scoreDocs[i]; Document foundDoc = indexSearcher.doc(scoreDoc.doc); Taxon term = new TaxonImpl(); Fieldable idField = foundDoc.getFieldable(FIELD_ID); if (idField != null) { term.setExternalId(idField.stringValue()); } Fieldable rankPathField = foundDoc.getFieldable(FIELD_RANK_PATH); if (rankPathField != null) { term.setPath(rankPathField.stringValue()); } Fieldable rankPathIdsField = foundDoc.getFieldable(FIELD_RANK_PATH_IDS); if (rankPathIdsField != null) { term.setPathIds(rankPathIdsField.stringValue()); } Fieldable rankPathNamesField = foundDoc.getFieldable(FIELD_RANK_PATH_NAMES); if (rankPathNamesField != null) { term.setPathNames(rankPathNamesField.stringValue()); } Fieldable commonNamesFields = foundDoc.getFieldable(FIELD_COMMON_NAMES); if (commonNamesFields != null) { term.setCommonNames(commonNamesFields.stringValue()); } Fieldable fieldName = foundDoc.getFieldable(FIELD_RECOMMENDED_NAME); if (fieldName != null) { term.setName(fieldName.stringValue()); } terms[i] = term; } } } return terms; }
@Test public void testPhraseQuery() { Map<String, ColumnMapper> map = new HashMap<>(); map.put("name", new ColumnMapperBoolean()); Schema mappers = new Schema(map, null, EnglishAnalyzer.class.getName()); List<String> values = new ArrayList<>(); values.add("hola"); values.add("adios"); PhraseCondition phraseCondition = new PhraseCondition(0.5f, "name", values, 2); Query query = phraseCondition.query(mappers); Assert.assertNotNull(query); Assert.assertEquals(org.apache.lucene.search.PhraseQuery.class, query.getClass()); org.apache.lucene.search.PhraseQuery luceneQuery = (org.apache.lucene.search.PhraseQuery) query; Assert.assertEquals(values.size(), luceneQuery.getTerms().length); Assert.assertEquals(2, luceneQuery.getSlop()); Assert.assertEquals(0.5f, query.getBoost(), 0); }
public void testWithPendingDeletes3() throws IOException { // main directory Directory dir = newDirectory(); // auxiliary directory Directory aux = newDirectory(); setUpDirs(dir, aux); IndexWriter writer = newWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) .setOpenMode(OpenMode.APPEND)); // Adds 10 docs, then replaces them with another 10 // docs, so 10 pending deletes: for (int i = 0; i < 20; i++) { Document doc = new Document(); doc.add(newStringField("id", "" + (i % 10), Field.Store.NO)); doc.add(newTextField("content", "bbb " + i, Field.Store.NO)); writer.updateDocument(new Term("id", "" + (i % 10)), doc); } // Deletes one of the 10 added docs, leaving 9: PhraseQuery q = new PhraseQuery(); q.add(new Term("content", "bbb")); q.add(new Term("content", "14")); writer.deleteDocuments(q); writer.addIndexes(aux); writer.forceMerge(1); writer.commit(); verifyNumDocs(dir, 1039); verifyTermDocs(dir, new Term("content", "aaa"), 1030); verifyTermDocs(dir, new Term("content", "bbb"), 9); writer.close(); dir.close(); aux.close(); }
@Override public void execute(String indexDir) { ConsoleUtilities.printHeader(); try { Directory indexDirectory = FSDirectory.open(new File(indexDir)); IndexReader indexReader = DirectoryReader.open(indexDirectory); IndexSearcher searcher = new IndexSearcher(indexReader); System.out.println("[BooleanQueryOperation.execute]"); // TODO: Implement and execute BooleanQuery PhraseQuery phraseQuery1 = new PhraseQuery(); phraseQuery1.add(new Term("description", "legendary")); phraseQuery1.add(new Term("description", "sacred")); phraseQuery1.add(new Term("description", "realm")); // cursed land PhraseQuery phraseQuery2 = new PhraseQuery(); phraseQuery2.add(new Term("description", "cursed")); phraseQuery2.add(new Term("description", "land")); BooleanQuery booleanQuery = new BooleanQuery(); booleanQuery.add(new BooleanClause(phraseQuery1, Occur.SHOULD)); booleanQuery.add(new BooleanClause(phraseQuery2, Occur.SHOULD)); int topHitNum = 30; TopDocs topDocs = searcher.search(booleanQuery, topHitNum); for (ScoreDoc scoreDoc : topDocs.scoreDocs) { Document doc = searcher.doc(scoreDoc.doc); String path = doc.get("path"); Game game = gameBuilder.build(new File(path)); System.out.println(ResultFormatter.format(game, scoreDoc.score)); } // BooleanClause.Occur.SHOULD means that the clause is optional, // whereas BooleanClause.Occur.Must means that the clause is required. // However, if a boolean query only has optional clauses, at least one // clause must match for a document to appear in the results. // For better control over what documents match a BooleanQuery, there is // also a minimumShouldMatch parameter which lets you tell Lucene that at // least minimumShouldMatch BooleanClause.Occur.SHOULD clauses must match // for a document to appear in the results. } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
public void testPositionIncrement() throws Exception { AqpQueryParser qp = getParser(); qp.setAnalyzer( new StopAnalyzer( TEST_VERSION_CURRENT, StopFilter.makeStopSet(TEST_VERSION_CURRENT, "the", "in", "are", "this"))); qp.setEnablePositionIncrements(true); String qtxt = "\"the words in poisitions pos02578 are stopped in this phrasequery\""; // 0 2 5 7 8 int expectedPositions[] = {1, 3, 4, 6, 9}; PhraseQuery pq = (PhraseQuery) qp.parse(qtxt, "a"); // System.out.println("Query text: "+qtxt); // System.out.println("Result: "+pq); Term t[] = pq.getTerms(); int pos[] = pq.getPositions(); for (int i = 0; i < t.length; i++) { // System.out.println(i+". "+t[i]+" pos: "+pos[i]); assertEquals( "term " + i + " = " + t[i] + " has wrong term-position!", expectedPositions[i], pos[i]); } }
BooleanQuery orPhraseQuery(List<SearchTerm> orSearchTerms) { BooleanQuery orTerms = new BooleanQuery(); orTerms.setMaxClauseCount(dvnMaxClauseCount); for (Iterator it = orSearchTerms.iterator(); it.hasNext(); ) { SearchTerm elem = (SearchTerm) it.next(); String[] phrase = getPhrase(elem.getValue().toLowerCase().trim()); if (phrase.length > 1) { BooleanClause partialMatchClause = null; PhraseQuery phraseQuery = new PhraseQuery(); phraseQuery.setSlop(3); for (int i = 0; i < phrase.length; i++) { phraseQuery.add(new Term(elem.getFieldName(), phrase[i].toLowerCase().trim())); } orTerms.add(phraseQuery, BooleanClause.Occur.SHOULD); } else { Term t = new Term(elem.getFieldName(), elem.getValue().toLowerCase().trim()); TermQuery orQuery = new TermQuery(t); orTerms.add(orQuery, BooleanClause.Occur.SHOULD); } } return orTerms; }
BooleanQuery andSearchTermClause(List<SearchTerm> andSearchTerms) { BooleanQuery andTerms = new BooleanQuery(); andTerms.setMaxClauseCount(dvnMaxClauseCount); Query rQuery = null; for (Iterator it = andSearchTerms.iterator(); it.hasNext(); ) { SearchTerm elem = (SearchTerm) it.next(); if (elem.getOperator().equals("<")) { Term end = new Term(elem.getFieldName(), elem.getValue().toLowerCase().trim()); Term begin = null; rQuery = new RangeQuery(begin, end, true); andTerms.add(rQuery, BooleanClause.Occur.MUST); } else if (elem.getOperator().equals(">")) { Term end = null; Term begin = new Term(elem.getFieldName(), elem.getValue().toLowerCase().trim()); rQuery = new RangeQuery(begin, end, true); andTerms.add(rQuery, BooleanClause.Occur.MUST); } else if (elem.getFieldName().equalsIgnoreCase("any")) { andTerms = buildAnyQuery(elem.getValue().toLowerCase().trim()); } else { String[] phrase = getPhrase(elem.getValue().toLowerCase().trim()); if (phrase.length > 1) { PhraseQuery phraseQuery = new PhraseQuery(); phraseQuery.setSlop(0); andTerms.add(partialMatch(elem, 3)); } else { Term t = new Term(elem.getFieldName(), elem.getValue().toLowerCase().trim()); TermQuery andQuery = new TermQuery(t); if (elem.getOperator().equals("=")) { andTerms.add(andQuery, BooleanClause.Occur.MUST); } else if (elem.getOperator().equalsIgnoreCase("-")) { andTerms.add(andQuery, BooleanClause.Occur.MUST_NOT); } } } } return andTerms; }
private Query createQuery(SearchEngineFilter filter) { BooleanQuery fieldQuery = new BooleanQuery(); String key = filter.getKey(); String attachmentKey = key + IIndexerDAO.ATTACHMENT_FIELD_SUFFIX; Object value = filter.getValue(); if (null != value) { if (value instanceof String) { SearchEngineFilter.TextSearchOption option = filter.getTextSearchOption(); if (null == option) { option = SearchEngineFilter.TextSearchOption.AT_LEAST_ONE_WORD; } String stringValue = value.toString(); String[] values = stringValue.split("\\s+"); if (!option.equals(SearchEngineFilter.TextSearchOption.EXACT)) { BooleanClause.Occur bc = BooleanClause.Occur.SHOULD; if (option.equals(SearchEngineFilter.TextSearchOption.ALL_WORDS)) { bc = BooleanClause.Occur.MUST; } else if (option.equals(SearchEngineFilter.TextSearchOption.ANY_WORD)) { bc = BooleanClause.Occur.MUST_NOT; } for (int i = 0; i < values.length; i++) { TermQuery term = new TermQuery(new Term(key, values[i].toLowerCase())); // NOTE: search lower case.... if (filter.isIncludeAttachments()) { BooleanQuery compositeQuery = new BooleanQuery(); compositeQuery.add(term, BooleanClause.Occur.SHOULD); TermQuery termAttachment = new TermQuery(new Term(attachmentKey, values[i].toLowerCase())); compositeQuery.add(termAttachment, BooleanClause.Occur.SHOULD); fieldQuery.add(compositeQuery, bc); } else { fieldQuery.add(term, bc); } } } else { PhraseQuery phraseQuery = new PhraseQuery(); for (int i = 0; i < values.length; i++) { // NOTE: search lower case.... phraseQuery.add(new Term(key, values[i].toLowerCase())); } if (filter.isIncludeAttachments()) { fieldQuery.add(phraseQuery, BooleanClause.Occur.SHOULD); PhraseQuery phraseQuery2 = new PhraseQuery(); for (int i = 0; i < values.length; i++) { // NOTE: search lower case.... phraseQuery2.add(new Term(attachmentKey, values[i].toLowerCase())); } fieldQuery.add(phraseQuery2, BooleanClause.Occur.SHOULD); } else { return phraseQuery; } } } else if (value instanceof Date) { String toString = DateTools.timeToString(((Date) value).getTime(), DateTools.Resolution.MINUTE); TermQuery term = new TermQuery(new Term(filter.getKey(), toString)); fieldQuery.add(term, BooleanClause.Occur.MUST); } else if (value instanceof Number) { TermQuery term = new TermQuery(new Term(filter.getKey(), value.toString())); fieldQuery.add(term, BooleanClause.Occur.MUST); } } else { if (filter.getStart() instanceof Number || filter.getEnd() instanceof Number) { // .............................. TODO } else { String start = null; String end = null; if (filter.getStart() instanceof Date || filter.getEnd() instanceof Date) { if (null != filter.getStart()) { start = DateTools.timeToString( ((Date) filter.getStart()).getTime(), DateTools.Resolution.MINUTE); } if (null != filter.getEnd()) { end = DateTools.timeToString( ((Date) filter.getEnd()).getTime(), DateTools.Resolution.MINUTE); } } else { start = (null != filter.getStart()) ? filter.getStart().toString().toLowerCase() : null; end = (null != filter.getEnd()) ? filter.getEnd().toString().toLowerCase() : null; } BytesRef byteStart = (null != start) ? new BytesRef(start.getBytes()) : null; BytesRef byteEnd = (null != end) ? new BytesRef(end.getBytes()) : null; TermRangeQuery range = new TermRangeQuery(filter.getKey(), byteStart, byteEnd, true, true); fieldQuery.add(range, BooleanClause.Occur.MUST); } } return fieldQuery; }
public Query createQuery(FieldContext fieldContext) { final Query perFieldQuery; final String fieldName = fieldContext.getField(); /* * Store terms per position and detect if for a given position more than one term is present */ TokenStream stream = null; boolean isMultiPhrase = false; Map<Integer, List<Term>> termsPerPosition = new HashMap<Integer, List<Term>>(); final String sentence = phraseContext.getSentence(); try { Reader reader = new StringReader(sentence); stream = queryContext.getQueryAnalyzer().tokenStream(fieldName, reader); CharTermAttribute termAttribute = stream.addAttribute(CharTermAttribute.class); PositionIncrementAttribute positionAttribute = stream.addAttribute(PositionIncrementAttribute.class); stream.reset(); int position = -1; // start at -1 since we apply at least one increment List<Term> termsAtSamePosition = null; while (stream.incrementToken()) { int positionIncrement = 1; if (positionAttribute != null) { positionIncrement = positionAttribute.getPositionIncrement(); } if (positionIncrement > 0) { position += positionIncrement; termsAtSamePosition = termsPerPosition.get(position); } if (termsAtSamePosition == null) { termsAtSamePosition = new ArrayList<Term>(); termsPerPosition.put(position, termsAtSamePosition); } String termString = new String(termAttribute.buffer(), 0, termAttribute.length()); termsAtSamePosition.add(new Term(fieldName, termString)); if (termsAtSamePosition.size() > 1) { isMultiPhrase = true; } } } catch (IOException e) { throw new AssertionFailure("IOException while reading a string. Doh!", e); } finally { if (stream != null) { try { stream.end(); stream.close(); } catch (IOException e) { throw new AssertionFailure("IOException while reading a string. Doh!", e); } } } /* * Create the appropriate query depending on the conditions * note that a MultiPhraseQuery is needed if several terms share the same position * as it will do a OR and not a AND like PhraseQuery */ final int size = termsPerPosition.size(); if (size == 0) { perFieldQuery = new BooleanQuery.Builder().build(); } else if (size <= 1) { final List<Term> terms = termsPerPosition.values().iterator().next(); if (terms.size() == 1) { perFieldQuery = new TermQuery(terms.get(0)); } else { BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); for (Term term : terms) { booleanQueryBuilder.add(new TermQuery(term), BooleanClause.Occur.SHOULD); } perFieldQuery = booleanQueryBuilder.build(); } } else { if (isMultiPhrase) { MultiPhraseQuery query = new MultiPhraseQuery(); query.setSlop(phraseContext.getSlop()); for (Map.Entry<Integer, List<Term>> entry : termsPerPosition.entrySet()) { final List<Term> value = entry.getValue(); query.add(value.toArray(new Term[value.size()]), entry.getKey()); } perFieldQuery = query; } else { PhraseQuery query = new PhraseQuery(); query.setSlop(phraseContext.getSlop()); for (Map.Entry<Integer, List<Term>> entry : termsPerPosition.entrySet()) { final List<Term> value = entry.getValue(); query.add(value.get(0), entry.getKey()); } perFieldQuery = query; } } return fieldContext.getFieldCustomizer().setWrappedQuery(perFieldQuery).createQuery(); }
// Test scores with one field with Term Freqs and one without, otherwise with equal content public void testBasic() throws Exception { Directory dir = newDirectory(); Analyzer analyzer = new MockAnalyzer(random()); IndexWriter writer = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer) .setMaxBufferedDocs(2) .setSimilarity(new SimpleSimilarity()) .setMergePolicy(newLogMergePolicy(2))); StringBuilder sb = new StringBuilder(265); String term = "term"; for (int i = 0; i < 30; i++) { Document d = new Document(); sb.append(term).append(" "); String content = sb.toString(); Field noTf = newField("noTf", content + (i % 2 == 0 ? "" : " notf"), omitType); d.add(noTf); Field tf = newField("tf", content + (i % 2 == 0 ? " tf" : ""), normalType); d.add(tf); writer.addDocument(d); // System.out.println(d); } writer.forceMerge(1); // flush writer.close(); /* * Verify the index */ IndexReader reader = DirectoryReader.open(dir); IndexSearcher searcher = newSearcher(reader); searcher.setSimilarity(new SimpleSimilarity()); Term a = new Term("noTf", term); Term b = new Term("tf", term); Term c = new Term("noTf", "notf"); Term d = new Term("tf", "tf"); TermQuery q1 = new TermQuery(a); TermQuery q2 = new TermQuery(b); TermQuery q3 = new TermQuery(c); TermQuery q4 = new TermQuery(d); PhraseQuery pq = new PhraseQuery(); pq.add(a); pq.add(c); try { searcher.search(pq, 10); fail("did not hit expected exception"); } catch (Exception e) { Throwable cause = e; // If the searcher uses an executor service, the IAE is wrapped into other exceptions while (cause.getCause() != null) { cause = cause.getCause(); } assertTrue("Expected an IAE, got " + cause, cause instanceof IllegalStateException); } searcher.search( q1, new CountingHitCollector() { private Scorer scorer; @Override public final void setScorer(Scorer scorer) { this.scorer = scorer; } @Override public final void collect(int doc) throws IOException { // System.out.println("Q1: Doc=" + doc + " score=" + score); float score = scorer.score(); assertTrue("got score=" + score, score == 1.0f); super.collect(doc); } }); // System.out.println(CountingHitCollector.getCount()); searcher.search( q2, new CountingHitCollector() { private Scorer scorer; @Override public final void setScorer(Scorer scorer) { this.scorer = scorer; } @Override public final void collect(int doc) throws IOException { // System.out.println("Q2: Doc=" + doc + " score=" + score); float score = scorer.score(); assertEquals(1.0f + doc, score, 0.00001f); super.collect(doc); } }); // System.out.println(CountingHitCollector.getCount()); searcher.search( q3, new CountingHitCollector() { private Scorer scorer; @Override public final void setScorer(Scorer scorer) { this.scorer = scorer; } @Override public final void collect(int doc) throws IOException { // System.out.println("Q1: Doc=" + doc + " score=" + score); float score = scorer.score(); assertTrue(score == 1.0f); assertFalse(doc % 2 == 0); super.collect(doc); } }); // System.out.println(CountingHitCollector.getCount()); searcher.search( q4, new CountingHitCollector() { private Scorer scorer; @Override public final void setScorer(Scorer scorer) { this.scorer = scorer; } @Override public final void collect(int doc) throws IOException { float score = scorer.score(); // System.out.println("Q1: Doc=" + doc + " score=" + score); assertTrue(score == 1.0f); assertTrue(doc % 2 == 0); super.collect(doc); } }); // System.out.println(CountingHitCollector.getCount()); BooleanQuery bq = new BooleanQuery(); bq.add(q1, Occur.MUST); bq.add(q4, Occur.MUST); searcher.search( bq, new CountingHitCollector() { @Override public final void collect(int doc) throws IOException { // System.out.println("BQ: Doc=" + doc + " score=" + score); super.collect(doc); } }); assertEquals(15, CountingHitCollector.getCount()); reader.close(); dir.close(); }