@Test
public void testCollectLiterals() throws Exception {
List<Symbol> toCollect =
Arrays.<Symbol>asList(
Literal.newLiteral("foobar"),
Literal.newLiteral(true),
Literal.newLiteral(1),
Literal.newLiteral(4.2));
CollectPhase collectNode =
new CollectPhase(
UUID.randomUUID(), 0, "literals", testRouting, toCollect, EMPTY_PROJECTIONS);
Bucket result = getBucket(collectNode);
assertThat(result, contains(isRow(new BytesRef("foobar"), true, 1, 4.2)));
}
@Test
public void testAnyEqArrayReference() throws Exception {
Reference ref = createReference("d_array", new ArrayType(DataTypes.DOUBLE));
Literal doubleLiteral = Literal.newLiteral(1.5d);
Query query = convert(whereClause(AnyEqOperator.NAME, doubleLiteral, ref));
assertThat(query.toString(), is("d_array:[1.5 TO 1.5]"));
}
private TermsFilter termsFilter(String columnName, Literal arrayLiteral) {
TermsFilter termsFilter;
Object values = arrayLiteral.value();
if (values instanceof Collection) {
termsFilter =
new TermsFilter(
columnName,
getBytesRefs((Collection) values, TermBuilder.forType(arrayLiteral.valueType())));
} else {
termsFilter =
new TermsFilter(
columnName,
getBytesRefs((Object[]) values, TermBuilder.forType(arrayLiteral.valueType())));
}
return termsFilter;
}
@Override
public Query apply(Function input, Context context) throws IOException {
Tuple<Reference, Literal> tuple = prepare(input);
if (tuple == null) {
return null;
}
String field = tuple.v1().info().ident().columnIdent().fqn();
Literal literal = tuple.v2();
CollectionType dataType = ((CollectionType) literal.valueType());
Set values = (Set) literal.value();
DataType innerType = dataType.innerType();
BytesRef[] terms = getBytesRefs(values, TermBuilder.forType(innerType));
TermsFilter termsFilter = new TermsFilter(field, terms);
return new FilteredQuery(Queries.newMatchAllQuery(), termsFilter);
}
/**
* When using PCRE features, switch to different regex implementation on top of java.util.regex.
*/
@Test
public void testRegexQueryPcre() throws Exception {
Reference value = createReference("foo", DataTypes.STRING);
Literal pattern = Literal.newLiteral(new BytesRef("\\D"));
Query query = convert(whereClause(RegexpMatchOperator.NAME, value, pattern));
assertThat(query, instanceOf(RegexQuery.class));
}
@Test
public void testIdQuery() throws Exception {
Reference ref = createReference("_id", DataTypes.STRING);
Query query = convert(whereClause(EqOperator.NAME, ref, Literal.newLiteral("i1")));
assertThat(query, instanceOf(TermQuery.class));
assertThat(query.toString(), is("_uid:default#i1"));
}
@Override
protected Query applyArrayReference(
Reference arrayReference, Literal literal, Context context) throws IOException {
QueryBuilderHelper builder =
QueryBuilderHelper.forType(((CollectionType) arrayReference.valueType()).innerType());
return builder.eq(arrayReference.ident().columnIdent().fqn(), literal.value());
}
@Nullable
private Function rewriteAndValidateFields(Function function, Context context) {
if (function.arguments().size() == 2) {
Symbol left = function.arguments().get(0);
Symbol right = function.arguments().get(1);
if (left.symbolType() == SymbolType.REFERENCE && right.symbolType().isValueSymbol()) {
Reference ref = (Reference) left;
if (ref.info().ident().columnIdent().equals(DocSysColumns.ID)) {
function.setArgument(
0,
new Reference(DocSysColumns.forTable(ref.ident().tableIdent(), DocSysColumns.UID)));
function.setArgument(
1,
Literal.newLiteral(
Uid.createUid(
Constants.DEFAULT_MAPPING_TYPE, ValueSymbolVisitor.STRING.process(right))));
} else {
String unsupportedMessage =
context.unsupportedMessage(ref.info().ident().columnIdent().name());
if (unsupportedMessage != null) {
throw new UnsupportedFeatureException(unsupportedMessage);
}
}
}
}
return function;
}
/**
* Make sure we still sport the fast Lucene regular expression engine when not using PCRE
* features.
*/
@Test
public void testRegexQueryFast() throws Exception {
Reference value = createReference("foo", DataTypes.STRING);
Literal pattern = Literal.newLiteral(new BytesRef("[a-z]"));
Query query = convert(whereClause(RegexpMatchOperator.NAME, value, pattern));
assertThat(query, instanceOf(XConstantScoreQuery.class));
assertThat(((XConstantScoreQuery) query).getFilter(), instanceOf(RegexpFilter.class));
}
@Test
public void testNormalizeSymbolReferenceAndLiteralFalse() throws Exception {
OrOperator operator = new OrOperator();
Function function =
new Function(
operator.info(), Arrays.<Symbol>asList(new Reference(), Literal.newLiteral(false)));
Symbol normalizedSymbol = operator.normalizeSymbol(function);
assertThat(normalizedSymbol, instanceOf(Reference.class));
}
@Override
protected Query applyArrayReference(
Reference arrayReference, Literal literal, Context context) throws IOException {
// 1 < ANY (array_col) --> array_col > 1
return rangeQuery.toQuery(
arrayReference,
((CollectionType) arrayReference.valueType()).innerType(),
literal.value());
}
@Override
protected Query applyArrayReference(
Reference arrayReference, Literal literal, Context context) throws IOException {
String notLike =
negateWildcard(convertWildcardToRegex(BytesRefs.toString(literal.value())));
return new RegexpQuery(
new Term(arrayReference.info().ident().columnIdent().fqn(), notLike),
RegexpFlag.COMPLEMENT.value());
}
@Test
public void testCollectWithFalseWhereClause() throws Exception {
CollectPhase collectNode =
new CollectPhase(
UUID.randomUUID(),
0,
"whereClause",
testRouting,
TO_COLLECT_TEST_REF,
EMPTY_PROJECTIONS);
collectNode.whereClause(
new WhereClause(
new Function(
AndOperator.INFO,
Arrays.<Symbol>asList(Literal.newLiteral(false), Literal.newLiteral(false)))));
Bucket result = getBucket(collectNode);
assertThat(result.size(), is(0));
}
@Test
public void testAnyEqArrayLiteral() throws Exception {
Reference ref = createReference("d", DataTypes.DOUBLE);
Literal doubleArrayLiteral =
Literal.newLiteral(new Object[] {-1.5d, 0.0d, 1.5d}, new ArrayType(DataTypes.DOUBLE));
Query query = convert(whereClause(AnyEqOperator.NAME, ref, doubleArrayLiteral));
assertThat(query, instanceOf(FilteredQuery.class));
assertThat(((FilteredQuery) query).getFilter(), instanceOf(TermsFilter.class));
}
@Test
public void testNormalizeSymbolReferenceAndLiteral() throws Exception {
OrOperator operator = new OrOperator();
Function function =
new Function(
operator.info(), Arrays.<Symbol>asList(new Reference(), Literal.newLiteral(true)));
Symbol normalizedSymbol = operator.normalizeSymbol(function);
assertLiteralSymbol(normalizedSymbol, true);
}
@Test
public void testCollectWithTrueWhereClause() throws Exception {
CollectPhase collectNode =
new CollectPhase(
UUID.randomUUID(),
0,
"whereClause",
testRouting,
TO_COLLECT_TEST_REF,
EMPTY_PROJECTIONS);
collectNode.whereClause(
new WhereClause(
new Function(
AndOperator.INFO,
Arrays.<Symbol>asList(Literal.newLiteral(true), Literal.newLiteral(true)))));
collectNode.maxRowGranularity(RowGranularity.NODE);
Bucket result = getBucket(collectNode);
assertThat(result, contains(isRow((short) 1)));
}
@Test
public void testAnyGreaterAndSmaller() throws Exception {
Reference arrayRef = createReference("d_array", new ArrayType(DataTypes.DOUBLE));
Literal doubleLiteral = Literal.newLiteral(1.5d);
Reference ref = createReference("d", DataTypes.DOUBLE);
Literal arrayLiteral =
Literal.newLiteral(new Object[] {1.2d, 3.5d}, new ArrayType(DataTypes.DOUBLE));
// 1.5d < ANY (d_array)
Query ltQuery = convert(whereClause(AnyLtOperator.NAME, doubleLiteral, arrayRef));
assertThat(ltQuery.toString(), is("d_array:{1.5 TO *}"));
// d < ANY ([1.2, 3.5])
Query ltQuery2 = convert(whereClause(AnyLtOperator.NAME, ref, arrayLiteral));
assertThat(ltQuery2.toString(), is("(d:{* TO 1.2} d:{* TO 3.5})~1"));
// 1.5d <= ANY (d_array)
Query lteQuery = convert(whereClause(AnyLteOperator.NAME, doubleLiteral, arrayRef));
assertThat(lteQuery.toString(), is("d_array:[1.5 TO *}"));
// d <= ANY ([1.2, 3.5])
Query lteQuery2 = convert(whereClause(AnyLteOperator.NAME, ref, arrayLiteral));
assertThat(lteQuery2.toString(), is("(d:{* TO 1.2] d:{* TO 3.5])~1"));
// 1.5d > ANY (d_array)
Query gtQuery = convert(whereClause(AnyGtOperator.NAME, doubleLiteral, arrayRef));
assertThat(gtQuery.toString(), is("d_array:{* TO 1.5}"));
// d > ANY ([1.2, 3.5])
Query gtQuery2 = convert(whereClause(AnyGtOperator.NAME, ref, arrayLiteral));
assertThat(gtQuery2.toString(), is("(d:{1.2 TO *} d:{3.5 TO *})~1"));
// 1.5d >= ANY (d_array)
Query gteQuery = convert(whereClause(AnyGteOperator.NAME, doubleLiteral, arrayRef));
assertThat(gteQuery.toString(), is("d_array:{* TO 1.5]"));
// d >= ANY ([1.2, 3.5])
Query gteQuery2 = convert(whereClause(AnyGteOperator.NAME, ref, arrayLiteral));
assertThat(gteQuery2.toString(), is("(d:[1.2 TO *} d:[3.5 TO *})~1"));
}
@Override
protected Query applyArrayLiteral(Reference reference, Literal arrayLiteral, Context context)
throws IOException {
// col like ANY (['a', 'b']) --> or(like(col, 'a'), like(col, 'b'))
BooleanQuery booleanQuery = new BooleanQuery();
booleanQuery.setMinimumNumberShouldMatch(1);
for (Object value : toIterable(arrayLiteral.value())) {
booleanQuery.add(
likeQuery.toQuery(reference, value, context), BooleanClause.Occur.SHOULD);
}
return booleanQuery;
}
@Override
public Query apply(Function input, Context context) {
Tuple<Reference, Literal> tuple = super.prepare(input);
if (tuple == null) {
return null;
}
Reference reference = tuple.v1();
Literal literal = tuple.v2();
String columnName = reference.info().ident().columnIdent().fqn();
if (DataTypes.isCollectionType(reference.valueType())
&& DataTypes.isCollectionType(literal.valueType())) {
// create boolean filter with term filters to pre-filter the result before applying the
// functionQuery.
BooleanFilter boolTermsFilter = new BooleanFilter();
DataType type = literal.valueType();
while (DataTypes.isCollectionType(type)) {
type = ((CollectionType) type).innerType();
}
QueryBuilderHelper builder = QueryBuilderHelper.forType(type);
Object value = literal.value();
buildTermsQuery(boolTermsFilter, value, columnName, builder);
if (boolTermsFilter.clauses().isEmpty()) {
// all values are null...
return genericFunctionQuery(input, context);
}
// wrap boolTermsFilter and genericFunction filter in an additional BooleanFilter to
// control the ordering of the filters
// termsFilter is applied first
// afterwards the more expensive genericFunctionFilter
BooleanFilter filterClauses = new BooleanFilter();
filterClauses.add(boolTermsFilter, BooleanClause.Occur.MUST);
filterClauses.add(genericFunctionFilter(input, context), BooleanClause.Occur.MUST);
return new FilteredQuery(Queries.newMatchAllQuery(), filterClauses);
}
QueryBuilderHelper builder = QueryBuilderHelper.forType(tuple.v1().valueType());
return builder.eq(columnName, tuple.v2().value());
}
@Test
public void testGteQuery() throws Exception {
Query query =
convert(
new WhereClause(
createFunction(
GteOperator.NAME,
DataTypes.BOOLEAN,
createReference("x", DataTypes.INTEGER),
Literal.newLiteral(10))));
assertThat(query, instanceOf(NumericRangeQuery.class));
assertThat(query.toString(), is("x:[10 TO *}"));
}
@Override
protected Query applyArrayLiteral(Reference reference, Literal arrayLiteral, Context context)
throws IOException {
// col < ANY ([1,2,3]) --> or(col<1, col<2, col<3)
BooleanQuery booleanQuery = new BooleanQuery();
booleanQuery.setMinimumNumberShouldMatch(1);
for (Object value : toIterable(arrayLiteral.value())) {
booleanQuery.add(
inverseRangeQuery.toQuery(reference, reference.valueType(), value),
BooleanClause.Occur.SHOULD);
}
return booleanQuery;
}
@Test
public void testEqOnTwoArraysBecomesGenericFunctionQueryAllValuesNull() throws Exception {
DataType longArray = new ArrayType(DataTypes.LONG);
Query query =
convert(
new WhereClause(
createFunction(
EqOperator.NAME,
DataTypes.BOOLEAN,
createReference("x", longArray),
Literal.newLiteral(longArray, new Object[] {null, null, null}))));
assertThat(query, instanceOf(FilteredQuery.class));
}
@Test
public void testWhereRefEqNullWithDifferentTypes() throws Exception {
for (DataType type : DataTypes.PRIMITIVE_TYPES) {
Reference foo = createReference("foo", type);
Query query = convert(whereClause(EqOperator.NAME, foo, Literal.newLiteral(type, null)));
// must always become a MatchNoDocsQuery
// string: term query with null would cause NPE
// int/numeric: rangeQuery from null to null would match all
// bool: term would match false too because of the condition in the eq query builder
assertThat(query, instanceOf(MatchNoDocsQuery.class));
}
}
@Test
public void testAnyOnArrayLiteral() throws Exception {
Reference ref = createReference("d", DataTypes.STRING);
Literal stringArrayLiteral =
Literal.newLiteral(
new Object[] {new BytesRef("a"), new BytesRef("b"), new BytesRef("c")},
new ArrayType(DataTypes.STRING));
// col != ANY (1,2,3)
Query neqQuery = convert(whereClause(AnyNeqOperator.NAME, ref, stringArrayLiteral));
assertThat(neqQuery, instanceOf(FilteredQuery.class));
assertThat(((FilteredQuery) neqQuery).getFilter(), instanceOf(BooleanFilter.class));
BooleanFilter filter = (BooleanFilter) ((FilteredQuery) neqQuery).getFilter();
assertThat(filter.toString(), is("BooleanFilter(-BooleanFilter(+d:a +d:b +d:c))"));
// col like any (1,2,3)
Query likeQuery = convert(whereClause(AnyLikeOperator.NAME, ref, stringArrayLiteral));
assertThat(likeQuery, instanceOf(BooleanQuery.class));
BooleanQuery likeBQuery = (BooleanQuery) likeQuery;
assertThat(likeBQuery.clauses().size(), is(3));
for (int i = 0; i < 2; i++) {
// like --> XConstantScoreQuery with regexp-filter
Query filteredQuery = likeBQuery.clauses().get(i).getQuery();
assertThat(filteredQuery, instanceOf(XConstantScoreQuery.class));
assertThat(((XConstantScoreQuery) filteredQuery).getFilter(), instanceOf(RegexpFilter.class));
}
// col not like any (1,2,3)
Query notLikeQuery = convert(whereClause(AnyNotLikeOperator.NAME, ref, stringArrayLiteral));
assertThat(notLikeQuery, instanceOf(BooleanQuery.class));
BooleanQuery notLikeBQuery = (BooleanQuery) notLikeQuery;
assertThat(notLikeBQuery.clauses(), hasSize(1));
BooleanClause clause = notLikeBQuery.clauses().get(0);
assertThat(clause.getOccur(), is(BooleanClause.Occur.MUST_NOT));
assertThat(((BooleanQuery) clause.getQuery()).clauses(), hasSize(3));
for (BooleanClause innerClause : ((BooleanQuery) clause.getQuery()).clauses()) {
assertThat(innerClause.getOccur(), is(BooleanClause.Occur.MUST));
assertThat(innerClause.getQuery(), instanceOf(XConstantScoreQuery.class));
assertThat(
((XConstantScoreQuery) innerClause.getQuery()).getFilter(),
instanceOf(RegexpFilter.class));
}
// col < any (1,2,3)
Query ltQuery2 = convert(whereClause(AnyLtOperator.NAME, ref, stringArrayLiteral));
assertThat(ltQuery2, instanceOf(BooleanQuery.class));
BooleanQuery ltBQuery = (BooleanQuery) ltQuery2;
assertThat(ltBQuery.toString(), is("(d:{* TO a} d:{* TO b} d:{* TO c})~1"));
}
@Test
public void testConditionThatReturnsNull() throws Exception {
BlobContainer container = new BlobContainer(tempFolder.newFolder());
createFile(container, "417de3231e23dcd6d224ff60918024bc6c59aa58");
Input<Boolean> condition = Literal.<Boolean>newLiteral((Boolean) null);
CollectingProjector projector =
getProjector(
container,
Arrays.<Input<?>>asList(digestExpression, ctimeExpression),
Arrays.<CollectExpression<File, ?>>asList(digestExpression, ctimeExpression),
condition);
Bucket result = projector.result().get();
assertThat(result.size(), is(0));
}
@Test
public void testWhereRefInSetLiteralIsConvertedToBooleanQuery() throws Exception {
DataType dataType = new SetType(DataTypes.INTEGER);
Reference foo = createReference("foo", DataTypes.INTEGER);
WhereClause whereClause =
new WhereClause(
createFunction(
InOperator.NAME,
DataTypes.BOOLEAN,
foo,
Literal.newLiteral(dataType, Sets.newHashSet(1, 3))));
Query query = convert(whereClause);
assertThat(query, instanceOf(FilteredQuery.class));
assertThat(((FilteredQuery) query).getFilter(), instanceOf(TermsFilter.class));
}
@Test
public void testCollectShardExpressionsLiteralsAndNodeExpressions() throws Exception {
CollectPhase collectNode =
new CollectPhase(
UUID.randomUUID(),
0,
"shardCollect",
shardRouting(0, 1),
Arrays.asList(testShardIdReference, Literal.newLiteral(true), testNodeReference),
EMPTY_PROJECTIONS);
collectNode.maxRowGranularity(RowGranularity.SHARD);
Bucket result = getBucket(collectNode);
assertThat(result.size(), is(2));
assertThat(result, containsInAnyOrder(isRow(0, true, (short) 1), isRow(1, true, (short) 1)));
}
@Test
public void testEqOnArrayWithTooManyClauses() throws Exception {
Object[] values = new Object[2000]; // should trigger the TooManyClauses exception
Arrays.fill(values, 10L);
DataType longArray = new ArrayType(DataTypes.LONG);
Query query =
convert(
new WhereClause(
createFunction(
EqOperator.NAME,
DataTypes.BOOLEAN,
createReference("x", longArray),
Literal.newLiteral(longArray, values))));
assertThat(query, instanceOf(FilteredQuery.class));
}
@Override
protected Query applyArrayReference(
Reference arrayReference, Literal literal, Context context) throws IOException {
// 1 != any ( col ) --> gt 1 or lt 1
String columnName = arrayReference.info().ident().columnIdent().fqn();
Object value = literal.value();
QueryBuilderHelper builder = QueryBuilderHelper.forType(arrayReference.valueType());
BooleanQuery query = new BooleanQuery();
query.setMinimumNumberShouldMatch(1);
query.add(
builder.rangeQuery(columnName, value, null, false, false), BooleanClause.Occur.SHOULD);
query.add(
builder.rangeQuery(columnName, null, value, false, false), BooleanClause.Occur.SHOULD);
return query;
}
@Override
protected Query applyArrayLiteral(Reference reference, Literal arrayLiteral, Context context)
throws IOException {
// col != ANY ([1,2,3]) --> not(col=1 and col=2 and col=3)
String columnName = reference.info().ident().columnIdent().fqn();
QueryBuilderHelper builder = QueryBuilderHelper.forType(reference.valueType());
BooleanFilter filter = new BooleanFilter();
BooleanFilter notFilter = new BooleanFilter();
for (Object value : toIterable(arrayLiteral.value())) {
notFilter.add(builder.eqFilter(columnName, value), BooleanClause.Occur.MUST);
}
filter.add(notFilter, BooleanClause.Occur.MUST_NOT);
return new FilteredQuery(Queries.newMatchAllQuery(), filter);
}