private final void populateTerm(
final HQuery hQuery, final boolean isShould, final boolean isMust, final Query subQueryL)
throws FederatedSearchException {
HTerm hTerm = new HTerm();
hTerm.isShould = isShould;
hTerm.isMust = isMust;
hTerm.boost = subQueryL.getBoost();
hQuery.terms.add(hTerm);
if (subQueryL instanceof TermQuery) {
TermQuery lTerm = (TermQuery) subQueryL;
hTerm.type = lTerm.getTerm().field();
hTerm.text = lTerm.getTerm().text();
} else if (subQueryL instanceof FuzzyQuery) {
FuzzyQuery lTerm = (FuzzyQuery) subQueryL;
hTerm.isFuzzy = true;
hTerm.type = lTerm.getTerm().field();
hTerm.text = lTerm.getTerm().text();
} else if (subQueryL instanceof TermRangeQuery) {
TermRangeQuery lTerm = (TermRangeQuery) subQueryL;
hTerm.isFuzzy = false;
hTerm.type = lTerm.getField();
hTerm.minRange = lTerm.getLowerTerm();
hTerm.maxRange = lTerm.getUpperTerm();
} else {
throw new FederatedSearchException(
"HQueryParser: Not Implemented Query :" + subQueryL.getClass().toString());
}
}
/**
* 组装TermRangeQuery
*
* @param elements
* @return
*/
private TermRangeQuery toTermRangeQuery(Element fieldNameEle, LinkedList<Element> elements) {
boolean includeFirst = false;
boolean includeLast = false;
String firstValue = null;
String lastValue = null;
// 检查第一个元素是否是[或者{
Element first = elements.getFirst();
if ('[' == first.type) {
includeFirst = true;
} else if ('{' == first.type) {
includeFirst = false;
} else {
throw new IllegalStateException("表达式异常");
}
// 检查最后一个元素是否是]或者}
Element last = elements.getLast();
if (']' == last.type) {
includeLast = true;
} else if ('}' == last.type) {
includeLast = false;
} else {
throw new IllegalStateException("表达式异常, RangeQuery缺少结束括号");
}
if (elements.size() < 4 || elements.size() > 5) {
throw new IllegalStateException("表达式异常, RangeQuery 错误");
}
// 读出中间部分
Element e2 = elements.get(1);
if ('\'' == e2.type) {
firstValue = e2.toString();
//
Element e3 = elements.get(2);
if (',' != e3.type) {
throw new IllegalStateException("表达式异常, RangeQuery缺少逗号分隔");
}
//
Element e4 = elements.get(3);
if ('\'' == e4.type) {
lastValue = e4.toString();
} else if (e4 != last) {
throw new IllegalStateException("表达式异常，RangeQuery格式错误");
}
} else if (',' == e2.type) {
firstValue = null;
//
Element e3 = elements.get(2);
if ('\'' == e3.type) {
lastValue = e3.toString();
} else {
throw new IllegalStateException("表达式异常，RangeQuery格式错误");
}
} else {
throw new IllegalStateException("表达式异常, RangeQuery格式错误");
}
return TermRangeQuery.newStringRange(
fieldNameEle.toString(), firstValue, lastValue, includeFirst, includeLast);
}
@Override
public TermRangeQuery getQuery(Element e) throws ParserException {
String fieldName = DOMUtils.getAttributeWithInheritance(e, "fieldName");
String lowerTerm = e.hasAttribute("lowerTerm") ? e.getAttribute("lowerTerm") : null;
String upperTerm = e.hasAttribute("upperTerm") ? e.getAttribute("upperTerm") : null;
boolean includeLower = DOMUtils.getAttribute(e, "includeLower", true);
boolean includeUpper = DOMUtils.getAttribute(e, "includeUpper", true);
return TermRangeQuery.newStringRange(
fieldName, lowerTerm, upperTerm, includeLower, includeUpper);
}
/** test sparse filters (match single documents) */
public void testSparse() throws Exception {
for (int i = 0; i < 10; i++) {
int id_start = random().nextInt(ir.maxDoc() - 1);
int id_end = id_start + 1;
Query query =
TermRangeQuery.newStringRange(
"id", Integer.toString(id_start), Integer.toString(id_end), true, true);
Filter expected = new QueryWrapperFilter(query);
Filter actual = new CachingWrapperFilter(expected);
assertFilterEquals(expected, actual);
}
}
@Override
public Query getRangeQuery(String field, String part1, String part2, boolean inclusive)
throws ParseException {
TermRangeQuery query =
(TermRangeQuery) // A
super.getRangeQuery(
field, part1, part2, // A
inclusive); // A
if ("price".equals(field)) {
return NumericRangeQuery.newDoubleRange( // B
"price", // B
Double.parseDouble( // B
query.getLowerTerm()), // B
Double.parseDouble( // B
query.getUpperTerm()), // B
query.includesLower(), // B
query.includesUpper()); // B
} else {
return query; // C
}
}
@Override
protected void doAssertLuceneQuery(
RangeQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
if (getCurrentTypes().length == 0
|| (queryBuilder.fieldName().equals(DATE_FIELD_NAME) == false
&& queryBuilder.fieldName().equals(INT_FIELD_NAME) == false)) {
assertThat(query, instanceOf(TermRangeQuery.class));
TermRangeQuery termRangeQuery = (TermRangeQuery) query;
assertThat(termRangeQuery.getField(), equalTo(queryBuilder.fieldName()));
assertThat(termRangeQuery.getLowerTerm(), equalTo(BytesRefs.toBytesRef(queryBuilder.from())));
assertThat(termRangeQuery.getUpperTerm(), equalTo(BytesRefs.toBytesRef(queryBuilder.to())));
assertThat(termRangeQuery.includesLower(), equalTo(queryBuilder.includeLower()));
assertThat(termRangeQuery.includesUpper(), equalTo(queryBuilder.includeUpper()));
} else if (queryBuilder.fieldName().equals(DATE_FIELD_NAME)) {
// we can't properly test unmapped dates because LateParsingQuery is package private
} else if (queryBuilder.fieldName().equals(INT_FIELD_NAME)) {
assertThat(query, instanceOf(NumericRangeQuery.class));
NumericRangeQuery numericRangeQuery = (NumericRangeQuery) query;
assertThat(numericRangeQuery.getField(), equalTo(queryBuilder.fieldName()));
assertThat(numericRangeQuery.getMin(), equalTo(queryBuilder.from()));
assertThat(numericRangeQuery.getMax(), equalTo(queryBuilder.to()));
assertThat(numericRangeQuery.includesMin(), equalTo(queryBuilder.includeLower()));
assertThat(numericRangeQuery.includesMax(), equalTo(queryBuilder.includeUpper()));
} else {
throw new UnsupportedOperationException();
}
}
/** DateField specific range query */
public Query getRangeQuery(
QParser parser,
SchemaField sf,
Date part1,
Date part2,
boolean minInclusive,
boolean maxInclusive) {
return TermRangeQuery.newStringRange(
sf.getName(),
part1 == null ? null : toInternal(part1),
part2 == null ? null : toInternal(part2),
minInclusive,
maxInclusive);
}
private static JsonObject convertTermRangeQuery(TermRangeQuery query) {
JsonObject interval = new JsonObject();
if (query.getLowerTerm() != null) {
interval.addProperty(
query.includesLower() ? "gte" : "gt", query.getLowerTerm().utf8ToString());
}
if (query.getUpperTerm() != null) {
interval.addProperty(
query.includesUpper() ? "lte" : "lt", query.getUpperTerm().utf8ToString());
}
interval.addProperty("boost", query.getBoost());
JsonObject range =
JsonBuilder.object()
.add("range", JsonBuilder.object().add(query.getField(), interval))
.build();
return wrapQueryForNestedIfRequired(query.getField(), range);
}
/**
* Tests NumericRangeQuery on a multi-valued field (multiple numeric values per document). This
* test ensures, that a classical TermRangeQuery returns exactly the same document numbers as
* NumericRangeQuery (see SOLR-1322 for discussion) and the multiple precision terms per numeric
* value do not interfere with multiple numeric values.
*/
public void testMultiValuedNRQ() throws Exception {
Directory directory = newDirectory();
RandomIndexWriter writer =
new RandomIndexWriter(
random(),
directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMaxBufferedDocs(_TestUtil.nextInt(random(), 50, 1000)));
DecimalFormat format = new DecimalFormat("00000000000", new DecimalFormatSymbols(Locale.US));
int num = atLeast(500);
for (int l = 0; l < num; l++) {
Document doc = new Document();
for (int m = 0, c = random().nextInt(10); m <= c; m++) {
int value = random().nextInt(Integer.MAX_VALUE);
doc.add(newStringField("asc", format.format(value), Field.Store.NO));
doc.add(new IntField("trie", value, Field.Store.NO));
}
writer.addDocument(doc);
}
IndexReader reader = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(reader);
num = atLeast(50);
for (int i = 0; i < num; i++) {
int lower = random().nextInt(Integer.MAX_VALUE);
int upper = random().nextInt(Integer.MAX_VALUE);
if (lower > upper) {
int a = lower;
lower = upper;
upper = a;
}
TermRangeQuery cq =
TermRangeQuery.newStringRange(
"asc", format.format(lower), format.format(upper), true, true);
NumericRangeQuery<Integer> tq =
NumericRangeQuery.newIntRange("trie", lower, upper, true, true);
TopDocs trTopDocs = searcher.search(cq, 1);
TopDocs nrTopDocs = searcher.search(tq, 1);
assertEquals(
"Returned count for NumericRangeQuery and TermRangeQuery must be equal",
trTopDocs.totalHits,
nrTopDocs.totalHits);
}
reader.close();
directory.close();
}
private static void planTermRangeQuery(final StringBuilder builder, final TermRangeQuery query) {
builder.append(query.getLowerTerm());
builder.append(" TO ");
builder.append(query.getUpperTerm());
}
/** @see #toString(Query,IndexSchema) */
public static void toString(Query query, IndexSchema schema, Appendable out, int flags)
throws IOException {
boolean writeBoost = true;
if (query instanceof TermQuery) {
TermQuery q = (TermQuery) query;
Term t = q.getTerm();
FieldType ft = writeFieldName(t.field(), schema, out, flags);
writeFieldVal(t.bytes(), ft, out, flags);
} else if (query instanceof TermRangeQuery) {
TermRangeQuery q = (TermRangeQuery) query;
String fname = q.getField();
FieldType ft = writeFieldName(fname, schema, out, flags);
out.append(q.includesLower() ? '[' : '{');
BytesRef lt = q.getLowerTerm();
BytesRef ut = q.getUpperTerm();
if (lt == null) {
out.append('*');
} else {
writeFieldVal(lt, ft, out, flags);
}
out.append(" TO ");
if (ut == null) {
out.append('*');
} else {
writeFieldVal(ut, ft, out, flags);
}
out.append(q.includesUpper() ? ']' : '}');
} else if (query instanceof NumericRangeQuery) {
NumericRangeQuery q = (NumericRangeQuery) query;
String fname = q.getField();
FieldType ft = writeFieldName(fname, schema, out, flags);
out.append(q.includesMin() ? '[' : '{');
Number lt = q.getMin();
Number ut = q.getMax();
if (lt == null) {
out.append('*');
} else {
out.append(lt.toString());
}
out.append(" TO ");
if (ut == null) {
out.append('*');
} else {
out.append(ut.toString());
}
out.append(q.includesMax() ? ']' : '}');
} else if (query instanceof BooleanQuery) {
BooleanQuery q = (BooleanQuery) query;
boolean needParens = false;
if (q.getBoost() != 1.0 || q.getMinimumNumberShouldMatch() != 0 || q.isCoordDisabled()) {
needParens = true;
}
if (needParens) {
out.append('(');
}
boolean first = true;
for (BooleanClause c : q.clauses()) {
if (!first) {
out.append(' ');
} else {
first = false;
}
if (c.isProhibited()) {
out.append('-');
} else if (c.isRequired()) {
out.append('+');
}
Query subQuery = c.getQuery();
boolean wrapQuery = false;
// TODO: may need to put parens around other types
// of queries too, depending on future syntax.
if (subQuery instanceof BooleanQuery) {
wrapQuery = true;
}
if (wrapQuery) {
out.append('(');
}
toString(subQuery, schema, out, flags);
if (wrapQuery) {
out.append(')');
}
}
if (needParens) {
out.append(')');
}
if (q.getMinimumNumberShouldMatch() > 0) {
out.append('~');
out.append(Integer.toString(q.getMinimumNumberShouldMatch()));
}
if (q.isCoordDisabled()) {
out.append("/no_coord");
}
} else if (query instanceof PrefixQuery) {
PrefixQuery q = (PrefixQuery) query;
Term prefix = q.getPrefix();
FieldType ft = writeFieldName(prefix.field(), schema, out, flags);
out.append(prefix.text());
out.append('*');
} else if (query instanceof WildcardQuery) {
out.append(query.toString());
writeBoost = false;
} else if (query instanceof FuzzyQuery) {
out.append(query.toString());
writeBoost = false;
} else if (query instanceof ConstantScoreQuery) {
out.append(query.toString());
writeBoost = false;
} else if (query instanceof WrappedQuery) {
WrappedQuery q = (WrappedQuery) query;
out.append(q.getOptions());
toString(q.getWrappedQuery(), schema, out, flags);
writeBoost = false; // we don't use the boost on wrapped queries
} else {
out.append(query.getClass().getSimpleName() + '(' + query.toString() + ')');
writeBoost = false;
}
if (writeBoost && query.getBoost() != 1.0f) {
out.append("^");
out.append(Float.toString(query.getBoost()));
}
}
/**
* Extracts all MultiTermQueries for {@code field}, and returns equivalent automata that will
* match terms.
*/
static CharacterRunAutomaton[] extractAutomata(Query query, String field) {
List<CharacterRunAutomaton> list = new ArrayList<>();
if (query instanceof BooleanQuery) {
for (BooleanClause clause : (BooleanQuery) query) {
if (!clause.isProhibited()) {
list.addAll(Arrays.asList(extractAutomata(clause.getQuery(), field)));
}
}
} else if (query instanceof ConstantScoreQuery) {
list.addAll(Arrays.asList(extractAutomata(((ConstantScoreQuery) query).getQuery(), field)));
} else if (query instanceof DisjunctionMaxQuery) {
for (Query sub : ((DisjunctionMaxQuery) query).getDisjuncts()) {
list.addAll(Arrays.asList(extractAutomata(sub, field)));
}
} else if (query instanceof SpanOrQuery) {
for (Query sub : ((SpanOrQuery) query).getClauses()) {
list.addAll(Arrays.asList(extractAutomata(sub, field)));
}
} else if (query instanceof SpanNearQuery) {
for (Query sub : ((SpanNearQuery) query).getClauses()) {
list.addAll(Arrays.asList(extractAutomata(sub, field)));
}
} else if (query instanceof SpanNotQuery) {
list.addAll(Arrays.asList(extractAutomata(((SpanNotQuery) query).getInclude(), field)));
} else if (query instanceof SpanPositionCheckQuery) {
list.addAll(
Arrays.asList(extractAutomata(((SpanPositionCheckQuery) query).getMatch(), field)));
} else if (query instanceof SpanMultiTermQueryWrapper) {
list.addAll(
Arrays.asList(
extractAutomata(((SpanMultiTermQueryWrapper<?>) query).getWrappedQuery(), field)));
} else if (query instanceof AutomatonQuery) {
final AutomatonQuery aq = (AutomatonQuery) query;
if (aq.getField().equals(field)) {
list.add(
new CharacterRunAutomaton(aq.getAutomaton()) {
@Override
public String toString() {
return aq.toString();
}
});
}
} else if (query instanceof PrefixQuery) {
final PrefixQuery pq = (PrefixQuery) query;
Term prefix = pq.getPrefix();
if (prefix.field().equals(field)) {
list.add(
new CharacterRunAutomaton(
Operations.concatenate(
Automata.makeString(prefix.text()), Automata.makeAnyString())) {
@Override
public String toString() {
return pq.toString();
}
});
}
} else if (query instanceof FuzzyQuery) {
final FuzzyQuery fq = (FuzzyQuery) query;
if (fq.getField().equals(field)) {
String utf16 = fq.getTerm().text();
int termText[] = new int[utf16.codePointCount(0, utf16.length())];
for (int cp, i = 0, j = 0; i < utf16.length(); i += Character.charCount(cp)) {
termText[j++] = cp = utf16.codePointAt(i);
}
int termLength = termText.length;
int prefixLength = Math.min(fq.getPrefixLength(), termLength);
String suffix =
UnicodeUtil.newString(termText, prefixLength, termText.length - prefixLength);
LevenshteinAutomata builder = new LevenshteinAutomata(suffix, fq.getTranspositions());
String prefix = UnicodeUtil.newString(termText, 0, prefixLength);
Automaton automaton = builder.toAutomaton(fq.getMaxEdits(), prefix);
list.add(
new CharacterRunAutomaton(automaton) {
@Override
public String toString() {
return fq.toString();
}
});
}
} else if (query instanceof TermRangeQuery) {
final TermRangeQuery tq = (TermRangeQuery) query;
if (tq.getField().equals(field)) {
final CharsRef lowerBound;
if (tq.getLowerTerm() == null) {
lowerBound = null;
} else {
lowerBound = new CharsRef(tq.getLowerTerm().utf8ToString());
}
final CharsRef upperBound;
if (tq.getUpperTerm() == null) {
upperBound = null;
} else {
upperBound = new CharsRef(tq.getUpperTerm().utf8ToString());
}
final boolean includeLower = tq.includesLower();
final boolean includeUpper = tq.includesUpper();
final CharsRef scratch = new CharsRef();
final Comparator<CharsRef> comparator = CharsRef.getUTF16SortedAsUTF8Comparator();
// this is *not* an automaton, but it's very simple
list.add(
new CharacterRunAutomaton(Automata.makeEmpty()) {
@Override
public boolean run(char[] s, int offset, int length) {
scratch.chars = s;
scratch.offset = offset;
scratch.length = length;
if (lowerBound != null) {
int cmp = comparator.compare(scratch, lowerBound);
if (cmp < 0 || (!includeLower && cmp == 0)) {
return false;
}
}
if (upperBound != null) {
int cmp = comparator.compare(scratch, upperBound);
if (cmp > 0 || (!includeUpper && cmp == 0)) {
return false;
}
}
return true;
}
@Override
public String toString() {
return tq.toString();
}
});
}
}
return list.toArray(new CharacterRunAutomaton[list.size()]);
}
/** Returns a random filter over the document set */
protected Filter randomFilter() {
return new QueryWrapperFilter(
TermRangeQuery.newStringRange("field", "a", "" + randomChar(), true, true));
}
private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
final Random rnd = newRandom();
String field = "field" + precisionStep;
int termCountT = 0, termCountC = 0;
for (int i = 0; i < 50; i++) {
long lower = (long) (rnd.nextDouble() * noDocs * distance) + startOffset;
long upper = (long) (rnd.nextDouble() * noDocs * distance) + startOffset;
if (lower > upper) {
long a = lower;
lower = upper;
upper = a;
}
// test inclusive range
NumericRangeQuery<Long> tq =
NumericRangeQuery.newLongRange(field, precisionStep, lower, upper, true, true);
TermRangeQuery cq =
new TermRangeQuery(
field,
NumericUtils.longToPrefixCoded(lower),
NumericUtils.longToPrefixCoded(upper),
true,
true);
TopDocs tTopDocs = searcher.search(tq, 1);
TopDocs cTopDocs = searcher.search(cq, 1);
assertEquals(
"Returned count for NumericRangeQuery and TermRangeQuery must be equal",
cTopDocs.totalHits,
tTopDocs.totalHits);
termCountT += tq.getTotalNumberOfTerms();
termCountC += cq.getTotalNumberOfTerms();
// test exclusive range
tq = NumericRangeQuery.newLongRange(field, precisionStep, lower, upper, false, false);
cq =
new TermRangeQuery(
field,
NumericUtils.longToPrefixCoded(lower),
NumericUtils.longToPrefixCoded(upper),
false,
false);
tTopDocs = searcher.search(tq, 1);
cTopDocs = searcher.search(cq, 1);
assertEquals(
"Returned count for NumericRangeQuery and TermRangeQuery must be equal",
cTopDocs.totalHits,
tTopDocs.totalHits);
termCountT += tq.getTotalNumberOfTerms();
termCountC += cq.getTotalNumberOfTerms();
// test left exclusive range
tq = NumericRangeQuery.newLongRange(field, precisionStep, lower, upper, false, true);
cq =
new TermRangeQuery(
field,
NumericUtils.longToPrefixCoded(lower),
NumericUtils.longToPrefixCoded(upper),
false,
true);
tTopDocs = searcher.search(tq, 1);
cTopDocs = searcher.search(cq, 1);
assertEquals(
"Returned count for NumericRangeQuery and TermRangeQuery must be equal",
cTopDocs.totalHits,
tTopDocs.totalHits);
termCountT += tq.getTotalNumberOfTerms();
termCountC += cq.getTotalNumberOfTerms();
// test right exclusive range
tq = NumericRangeQuery.newLongRange(field, precisionStep, lower, upper, true, false);
cq =
new TermRangeQuery(
field,
NumericUtils.longToPrefixCoded(lower),
NumericUtils.longToPrefixCoded(upper),
true,
false);
tTopDocs = searcher.search(tq, 1);
cTopDocs = searcher.search(cq, 1);
assertEquals(
"Returned count for NumericRangeQuery and TermRangeQuery must be equal",
cTopDocs.totalHits,
tTopDocs.totalHits);
termCountT += tq.getTotalNumberOfTerms();
termCountC += cq.getTotalNumberOfTerms();
}
if (precisionStep == Integer.MAX_VALUE) {
assertEquals(
"Total number of terms should be equal for unlimited precStep", termCountT, termCountC);
} else {
System.out.println("Average number of terms during random search on '" + field + "':");
System.out.println(" Trie query: " + (((double) termCountT) / (50 * 4)));
System.out.println(" Classical query: " + (((double) termCountC) / (50 * 4)));
}
}