@Test
public void reverseAnotherNestedGroupByOnNestedFieldWithFilterTestWithReverseNestedNoPath()
throws Exception {
Aggregations result =
query(
String.format(
"SELECT COUNT(*) FROM %s/nestedType GROUP BY nested(message.info),filter('myFilter',message.info = 'a'),reverse_nested(comment.data,'~comment')",
TEST_INDEX));
InternalNested nested = result.get("message.info@NESTED");
InternalFilter filter = nested.getAggregations().get("myFilter@FILTER");
Terms infos = filter.getAggregations().get("message.info");
Assert.assertEquals(1, infos.getBuckets().size());
for (Terms.Bucket bucket : infos.getBuckets()) {
InternalReverseNested reverseNested =
bucket.getAggregations().get("comment.data@NESTED_REVERSED");
InternalNested innerNested = reverseNested.getAggregations().get("comment.data@NESTED");
Terms terms = innerNested.getAggregations().get("comment.data");
Terms.Bucket internalBucket = terms.getBuckets().get(0);
long count = ((ValueCount) internalBucket.getAggregations().get("COUNT(*)")).getValue();
String key = internalBucket.getKey();
if (key.equalsIgnoreCase("ab")) {
Assert.assertEquals(2, count);
} else {
throw new Exception(String.format("Unexpected key. expected: only a . found: %s", key));
}
}
}
@Test
public void testDocCount_asSubAgg() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
terms("terms")
.field("tag")
.order(Order.term(true))
.subAggregation(
histogram("histo")
.field(SINGLE_VALUED_FIELD_NAME)
.interval(interval)
.extendedBounds((long) minRandomValue, (long) maxRandomValue))
.subAggregation(maxBucket("max_bucket").setBucketsPaths("histo>_count")))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
List<Terms.Bucket> termsBuckets = terms.getBuckets();
assertThat(termsBuckets.size(), equalTo(interval));
for (int i = 0; i < interval; ++i) {
Terms.Bucket termsBucket = termsBuckets.get(i);
assertThat(termsBucket, notNullValue());
assertThat((String) termsBucket.getKey(), equalTo("tag" + (i % interval)));
Histogram histo = termsBucket.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(histo.getName(), equalTo("histo"));
List<? extends Bucket> buckets = histo.getBuckets();
List<String> maxKeys = new ArrayList<>();
double maxValue = Double.NEGATIVE_INFINITY;
for (int j = 0; j < numValueBuckets; ++j) {
Histogram.Bucket bucket = buckets.get(j);
assertThat(bucket, notNullValue());
assertThat(((Number) bucket.getKey()).longValue(), equalTo((long) j * interval));
if (bucket.getDocCount() > maxValue) {
maxValue = bucket.getDocCount();
maxKeys = new ArrayList<>();
maxKeys.add(bucket.getKeyAsString());
} else if (bucket.getDocCount() == maxValue) {
maxKeys.add(bucket.getKeyAsString());
}
}
InternalBucketMetricValue maxBucketValue = termsBucket.getAggregations().get("max_bucket");
assertThat(maxBucketValue, notNullValue());
assertThat(maxBucketValue.getName(), equalTo("max_bucket"));
assertThat(maxBucketValue.value(), equalTo(maxValue));
assertThat(maxBucketValue.keys(), equalTo(maxKeys.toArray(new String[maxKeys.size()])));
}
}
@Test
public void testChildrenAggs() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("test")
.setQuery(matchQuery("randomized", true))
.addAggregation(
terms("category")
.field("category")
.size(0)
.subAggregation(
children("to_comment")
.childType("comment")
.subAggregation(
terms("commenters")
.field("commenter")
.size(0)
.subAggregation(topHits("top_comments")))))
.get();
assertSearchResponse(searchResponse);
Terms categoryTerms = searchResponse.getAggregations().get("category");
assertThat(categoryTerms.getBuckets().size(), equalTo(categoryToControl.size()));
for (Map.Entry<String, Control> entry1 : categoryToControl.entrySet()) {
Terms.Bucket categoryBucket = categoryTerms.getBucketByKey(entry1.getKey());
assertThat(categoryBucket.getKey(), equalTo(entry1.getKey()));
assertThat(categoryBucket.getDocCount(), equalTo((long) entry1.getValue().articleIds.size()));
Children childrenBucket = categoryBucket.getAggregations().get("to_comment");
assertThat(childrenBucket.getName(), equalTo("to_comment"));
assertThat(childrenBucket.getDocCount(), equalTo((long) entry1.getValue().commentIds.size()));
assertThat(
(long) childrenBucket.getProperty("_count"),
equalTo((long) entry1.getValue().commentIds.size()));
Terms commentersTerms = childrenBucket.getAggregations().get("commenters");
assertThat((Terms) childrenBucket.getProperty("commenters"), sameInstance(commentersTerms));
assertThat(
commentersTerms.getBuckets().size(),
equalTo(entry1.getValue().commenterToCommentId.size()));
for (Map.Entry<String, Set<String>> entry2 :
entry1.getValue().commenterToCommentId.entrySet()) {
Terms.Bucket commentBucket = commentersTerms.getBucketByKey(entry2.getKey());
assertThat(commentBucket.getKey(), equalTo(entry2.getKey()));
assertThat(commentBucket.getDocCount(), equalTo((long) entry2.getValue().size()));
TopHits topHits = commentBucket.getAggregations().get("top_comments");
for (SearchHit searchHit : topHits.getHits().getHits()) {
assertThat(entry2.getValue().contains(searchHit.getId()), is(true));
}
}
}
}
@Test
public void script_SingleValue_WithSubAggregator_Inherited() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms")
.executionHint(randomExecutionHint())
.script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value")
.subAggregation(count("count")))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
for (int i = 0; i < 5; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
assertThat(bucket.getDocCount(), equalTo(1l));
ValueCount valueCount = bucket.getAggregations().get("count");
assertThat(valueCount, notNullValue());
assertThat(valueCount.getValue(), equalTo(1l));
}
}
@Test
public void singleValuedField_OrderedBySingleValueSubAggregationAsc() throws Exception {
boolean asc = true;
SearchResponse response =
client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms")
.executionHint(randomExecutionHint())
.field(SINGLE_VALUED_FIELD_NAME)
.order(Terms.Order.aggregation("avg_i", asc))
.subAggregation(avg("avg_i").field("i")))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
int i = 0;
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
assertThat(bucket.getDocCount(), equalTo(1l));
Avg avg = bucket.getAggregations().get("avg_i");
assertThat(avg, notNullValue());
assertThat(avg.getValue(), equalTo((double) i));
i++;
}
}
@Test
public void singleValuedField_OrderedBySingleBucketSubAggregationAsc() throws Exception {
boolean asc = randomBoolean();
SearchResponse response =
client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("tags")
.executionHint(randomExecutionHint())
.field("tag")
.order(Terms.Order.aggregation("filter", asc))
.subAggregation(filter("filter").filter(FilterBuilders.matchAllFilter())))
.execute()
.actionGet();
assertSearchResponse(response);
Terms tags = response.getAggregations().get("tags");
assertThat(tags, notNullValue());
assertThat(tags.getName(), equalTo("tags"));
assertThat(tags.getBuckets().size(), equalTo(2));
Iterator<Terms.Bucket> iters = tags.getBuckets().iterator();
Terms.Bucket tag = iters.next();
assertThat(tag, notNullValue());
assertThat(key(tag), equalTo(asc ? "less" : "more"));
assertThat(tag.getDocCount(), equalTo(asc ? 2l : 3l));
Filter filter = tag.getAggregations().get("filter");
assertThat(filter, notNullValue());
assertThat(filter.getDocCount(), equalTo(asc ? 2l : 3l));
tag = iters.next();
assertThat(tag, notNullValue());
assertThat(key(tag), equalTo(asc ? "more" : "less"));
assertThat(tag.getDocCount(), equalTo(asc ? 3l : 2l));
filter = tag.getAggregations().get("filter");
assertThat(filter, notNullValue());
assertThat(filter.getDocCount(), equalTo(asc ? 3l : 2l));
}
public void testNestNestedAggs() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx_nested_nested_aggs")
.addAggregation(
nested("level1", "nested1")
.subAggregation(
terms("a")
.field("nested1.a")
.collectMode(aggCollectionMode)
.subAggregation(
nested("level2", "nested1.nested2")
.subAggregation(sum("sum").field("nested1.nested2.b")))))
.get();
assertSearchResponse(response);
Nested level1 = response.getAggregations().get("level1");
assertThat(level1, notNullValue());
assertThat(level1.getName(), equalTo("level1"));
assertThat(level1.getDocCount(), equalTo(2L));
StringTerms a = level1.getAggregations().get("a");
Terms.Bucket bBucket = a.getBucketByKey("a");
assertThat(bBucket.getDocCount(), equalTo(1L));
Nested level2 = bBucket.getAggregations().get("level2");
assertThat(level2.getDocCount(), equalTo(1L));
Sum sum = level2.getAggregations().get("sum");
assertThat(sum.getValue(), equalTo(2d));
a = level1.getAggregations().get("a");
bBucket = a.getBucketByKey("b");
assertThat(bBucket.getDocCount(), equalTo(1L));
level2 = bBucket.getAggregations().get("level2");
assertThat(level2.getDocCount(), equalTo(1L));
sum = level2.getAggregations().get("sum");
assertThat(sum.getValue(), equalTo(2d));
}
@Test
public void multiValuedField_WithValueScript_WithInheritedSubAggregator() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms")
.executionHint(randomExecutionHint())
.field(MULTI_VALUED_FIELD_NAME)
.script("'foo_' + _value")
.subAggregation(count("count")))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(6));
for (int i = 0; i < 6; i++) {
Terms.Bucket bucket = terms.getBucketByKey("foo_val" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("foo_val" + i));
if (i == 0 | i == 5) {
assertThat(bucket.getDocCount(), equalTo(1l));
ValueCount valueCount = bucket.getAggregations().get("count");
assertThat(valueCount, notNullValue());
assertThat(valueCount.getValue(), equalTo(2l));
} else {
assertThat(bucket.getDocCount(), equalTo(2l));
ValueCount valueCount = bucket.getAggregations().get("count");
assertThat(valueCount, notNullValue());
assertThat("term[" + key(bucket) + "]", valueCount.getValue(), equalTo(4l));
}
}
}
@Test
public void groupByTest() throws Exception {
Aggregations result =
query(String.format("SELECT COUNT(*) FROM %s/account GROUP BY gender", TEST_INDEX));
Terms gender = result.get("gender");
for (Terms.Bucket bucket : gender.getBuckets()) {
String key = bucket.getKey();
long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue();
if (key.equalsIgnoreCase("m")) {
Assert.assertEquals(507, count);
} else if (key.equalsIgnoreCase("f")) {
Assert.assertEquals(493, count);
} else {
throw new Exception(String.format("Unexpected key. expected: m OR f. found: %s", key));
}
}
}
@Test
public void testMetric_topLevel() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
terms("terms")
.field("tag")
.subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME)))
.addAggregation(maxBucket("max_bucket").setBucketsPaths("terms>sum"))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
List<Terms.Bucket> buckets = terms.getBuckets();
assertThat(buckets.size(), equalTo(interval));
List<String> maxKeys = new ArrayList<>();
double maxValue = Double.NEGATIVE_INFINITY;
for (int i = 0; i < interval; ++i) {
Terms.Bucket bucket = buckets.get(i);
assertThat(bucket, notNullValue());
assertThat((String) bucket.getKey(), equalTo("tag" + (i % interval)));
assertThat(bucket.getDocCount(), greaterThan(0l));
Sum sum = bucket.getAggregations().get("sum");
assertThat(sum, notNullValue());
if (sum.value() > maxValue) {
maxValue = sum.value();
maxKeys = new ArrayList<>();
maxKeys.add(bucket.getKeyAsString());
} else if (sum.value() == maxValue) {
maxKeys.add(bucket.getKeyAsString());
}
}
InternalBucketMetricValue maxBucketValue = response.getAggregations().get("max_bucket");
assertThat(maxBucketValue, notNullValue());
assertThat(maxBucketValue.getName(), equalTo("max_bucket"));
assertThat(maxBucketValue.value(), equalTo(maxValue));
assertThat(maxBucketValue.keys(), equalTo(maxKeys.toArray(new String[maxKeys.size()])));
}
@Test
public void reverseToRootGroupByOnNestedFieldWithFilterTestWithReverseNestedOnHistogram()
throws Exception {
Aggregations result =
query(
String.format(
"SELECT COUNT(*) FROM %s/nestedType GROUP BY nested(message.info),filter('myFilter',message.info = 'a'),histogram('field'='myNum','reverse_nested'='','interval'='2' , 'alias' = 'someAlias' )",
TEST_INDEX));
InternalNested nested = result.get("message.info@NESTED");
InternalFilter filter = nested.getAggregations().get("myFilter@FILTER");
Terms infos = filter.getAggregations().get("message.info");
Assert.assertEquals(1, infos.getBuckets().size());
for (Terms.Bucket bucket : infos.getBuckets()) {
InternalReverseNested reverseNested = bucket.getAggregations().get("someAlias@NESTED");
InternalHistogram histogram = reverseNested.getAggregations().get("someAlias");
Assert.assertEquals(2, histogram.getBuckets().size());
}
}
@Test
public void reverseToRootGroupByOnNestedFieldWithFilterAndSumOnReverseNestedField()
throws Exception {
Aggregations result =
query(
String.format(
"SELECT sum(reverse_nested(myNum)) bla FROM %s/nestedType GROUP BY nested(message.info),filter('myFilter',message.info = 'a')",
TEST_INDEX));
InternalNested nested = result.get("message.info@NESTED");
InternalFilter filter = nested.getAggregations().get("myFilter@FILTER");
Terms infos = filter.getAggregations().get("message.info");
Assert.assertEquals(1, infos.getBuckets().size());
for (Terms.Bucket bucket : infos.getBuckets()) {
InternalReverseNested reverseNested = bucket.getAggregations().get("myNum@NESTED");
InternalSum sum = reverseNested.getAggregations().get("bla");
Assert.assertEquals(5.0, sum.getValue(), 0.000001);
}
}
@Test
public void orderByDescTest()
throws IOException, SqlParseException, SQLFeatureNotSupportedException {
ArrayList<Long> agesCount = new ArrayList<>();
Aggregations result =
query(
String.format(
"SELECT COUNT(*) FROM %s/account GROUP BY age ORDER BY COUNT(*) DESC", TEST_INDEX));
Terms age = result.get("age");
for (Terms.Bucket bucket : age.getBuckets()) {
agesCount.add(((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue());
}
ArrayList<Long> sortedAgesCount = (ArrayList<Long>) agesCount.clone();
Collections.sort(sortedAgesCount, Collections.reverseOrder());
Assert.assertTrue("The list is not ordered descending", agesCount.equals(agesCount));
}
public Map<QualityProfileKey, Multimap<String, FacetValue>> getStatsByProfileKey(
List<QualityProfileKey> keys) {
String[] stringKeys = new String[keys.size()];
for (int i = 0; i < keys.size(); i++) {
stringKeys[i] = keys.get(i).toString();
}
SearchResponse response =
getClient()
.prepareSearch(this.getIndexName())
.setQuery(
QueryBuilders.filteredQuery(
QueryBuilders.matchAllQuery(),
FilterBuilders.termsFilter(
ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field(), stringKeys)))
.addAggregation(
AggregationBuilders.terms(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field())
.field(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field())
.subAggregation(
AggregationBuilders.terms(
ActiveRuleNormalizer.ActiveRuleField.INHERITANCE.field())
.field(ActiveRuleNormalizer.ActiveRuleField.INHERITANCE.field()))
.subAggregation(
AggregationBuilders.terms(
ActiveRuleNormalizer.ActiveRuleField.SEVERITY.field())
.field(ActiveRuleNormalizer.ActiveRuleField.SEVERITY.field())))
.setSize(0)
.setTypes(this.getIndexType())
.get();
Map<QualityProfileKey, Multimap<String, FacetValue>> stats =
new HashMap<QualityProfileKey, Multimap<String, FacetValue>>();
Aggregation aggregation =
response.getAggregations().get(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field());
for (Terms.Bucket value : ((Terms) aggregation).getBuckets()) {
stats.put(
QualityProfileKey.parse(value.getKey()),
this.processAggregations(value.getAggregations()));
}
return stats;
}
@Override
public void testOrderByEmptyAggregation() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("idx")
.setQuery(matchAllQuery())
.addAggregation(
terms("terms")
.field("value")
.order(
Terms.Order.compound(
Terms.Order.aggregation("filter>percentiles.99", true)))
.subAggregation(
filter("filter", termQuery("value", 100))
.subAggregation(
percentiles("percentiles")
.method(PercentilesMethod.TDIGEST)
.field("value"))))
.get();
assertHitCount(searchResponse, 10);
Terms terms = searchResponse.getAggregations().get("terms");
assertThat(terms, notNullValue());
List<Terms.Bucket> buckets = terms.getBuckets();
assertThat(buckets, notNullValue());
assertThat(buckets.size(), equalTo(10));
for (int i = 0; i < 10; i++) {
Terms.Bucket bucket = buckets.get(i);
assertThat(bucket, notNullValue());
assertThat(bucket.getKeyAsNumber(), equalTo((long) i + 1));
assertThat(bucket.getDocCount(), equalTo(1L));
Filter filter = bucket.getAggregations().get("filter");
assertThat(filter, notNullValue());
assertThat(filter.getDocCount(), equalTo(0L));
Percentiles percentiles = filter.getAggregations().get("percentiles");
assertThat(percentiles, notNullValue());
assertThat(percentiles.percentile(99), equalTo(Double.NaN));
}
}
@Test
public void groupByOnNestedFieldWithFilterTest() throws Exception {
Aggregations result =
query(
String.format(
"SELECT COUNT(*) FROM %s/nestedType GROUP BY nested(message.info),filter('myFilter',message.info = 'a')",
TEST_INDEX));
InternalNested nested = result.get("message.info@NESTED");
InternalFilter filter = nested.getAggregations().get("myFilter@FILTER");
Terms infos = filter.getAggregations().get("message.info");
Assert.assertEquals(1, infos.getBuckets().size());
for (Terms.Bucket bucket : infos.getBuckets()) {
String key = bucket.getKey();
long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue();
if (key.equalsIgnoreCase("a")) {
Assert.assertEquals(2, count);
} else {
throw new Exception(String.format("Unexpected key. expected: only a . found: %s", key));
}
}
}
@Test
public void testSubAggregations() throws Exception {
Set expectedAges =
new HashSet<>(ContiguousSet.create(Range.closed(20, 40), DiscreteDomain.integers()));
final String query =
String.format(
"SELECT /*! DOCS_WITH_AGGREGATION(10) */"
+ " * FROM %s/account GROUP BY (gender, age), (state) LIMIT 0,10",
TEST_INDEX);
Map<String, Set<Integer>> buckets = new HashMap<>();
SqlElasticSearchRequestBuilder select = getSearchRequestBuilder(query);
SearchResponse response = (SearchResponse) select.get();
Aggregations result = response.getAggregations();
Terms gender = result.get("gender");
for (Terms.Bucket genderBucket : gender.getBuckets()) {
String genderKey = genderBucket.getKey();
buckets.put(genderKey, new HashSet<Integer>());
Terms ageBuckets = (Terms) genderBucket.getAggregations().get("age");
for (Terms.Bucket ageBucket : ageBuckets.getBuckets()) {
buckets.get(genderKey).add(Integer.parseInt(ageBucket.getKey()));
}
}
Assert.assertEquals(2, buckets.keySet().size());
Assert.assertEquals(expectedAges, buckets.get("m"));
Assert.assertEquals(expectedAges, buckets.get("f"));
Terms state = result.get("state");
for (Terms.Bucket stateBucket : state.getBuckets()) {
if (stateBucket.getKey().equalsIgnoreCase("ak")) {
Assert.assertTrue("There are 22 entries for state ak", stateBucket.getDocCount() == 22);
}
}
Assert.assertEquals(response.getHits().totalHits(), 1000);
Assert.assertEquals(response.getHits().hits().length, 10);
}
@Override
public void testOrderByEmptyAggregation() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("idx")
.setQuery(matchAllQuery())
.addAggregation(
terms("terms")
.field("value")
.order(Order.compound(Order.aggregation("filter>stats.avg", true)))
.subAggregation(
filter("filter", termQuery("value", 100))
.subAggregation(stats("stats").field("value"))))
.get();
assertHitCount(searchResponse, 10);
Terms terms = searchResponse.getAggregations().get("terms");
assertThat(terms, notNullValue());
List<Terms.Bucket> buckets = terms.getBuckets();
assertThat(buckets, notNullValue());
assertThat(buckets.size(), equalTo(10));
for (int i = 0; i < 10; i++) {
Terms.Bucket bucket = buckets.get(i);
assertThat(bucket, notNullValue());
assertThat(bucket.getKeyAsNumber(), equalTo((long) i + 1));
assertThat(bucket.getDocCount(), equalTo(1L));
Filter filter = bucket.getAggregations().get("filter");
assertThat(filter, notNullValue());
assertThat(filter.getDocCount(), equalTo(0L));
Stats stats = filter.getAggregations().get("stats");
assertThat(stats, notNullValue());
assertThat(stats.getMin(), equalTo(Double.POSITIVE_INFINITY));
assertThat(stats.getMax(), equalTo(Double.NEGATIVE_INFINITY));
assertThat(stats.getAvg(), equalTo(Double.NaN));
assertThat(stats.getSum(), equalTo(0.0));
assertThat(stats.getCount(), equalTo(0L));
}
}
@Test
public void multipleGroupBysWithSize() throws Exception {
Set expectedAges =
new HashSet<Integer>(ContiguousSet.create(Range.closed(20, 40), DiscreteDomain.integers()));
Map<String, Set<Integer>> buckets = new HashMap<>();
Aggregations result =
query(
String.format(
"SELECT COUNT(*) FROM %s/account GROUP BY gender, terms('alias'='ageAgg','field'='age','size'=3)",
TEST_INDEX));
Terms gender = result.get("gender");
Assert.assertEquals(2, gender.getBuckets().size());
for (Terms.Bucket genderBucket : gender.getBuckets()) {
String genderKey = genderBucket.getKey();
buckets.put(genderKey, new HashSet<Integer>());
Terms ageBuckets = genderBucket.getAggregations().get("ageAgg");
Assert.assertEquals(3, ageBuckets.getBuckets().size());
}
}
public void testXContentResponse() throws Exception {
String type = randomBoolean() ? "string" : "long";
String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}";
SharedSignificantTermsTestMethods.index01Docs(type, settings, this);
SearchResponse response =
client()
.prepareSearch(INDEX_NAME)
.setTypes(DOC_TYPE)
.addAggregation(
new TermsBuilder("class")
.field(CLASS_FIELD)
.subAggregation(new SignificantTermsBuilder("sig_terms").field(TEXT_FIELD)))
.execute()
.actionGet();
assertSearchResponse(response);
StringTerms classes = (StringTerms) response.getAggregations().get("class");
assertThat(classes.getBuckets().size(), equalTo(2));
for (Terms.Bucket classBucket : classes.getBuckets()) {
Map<String, Aggregation> aggs = classBucket.getAggregations().asMap();
assertTrue(aggs.containsKey("sig_terms"));
SignificantTerms agg = (SignificantTerms) aggs.get("sig_terms");
assertThat(agg.getBuckets().size(), equalTo(1));
String term = agg.iterator().next().getKeyAsString();
String classTerm = classBucket.getKeyAsString();
assertTrue(term.equals(classTerm));
}
XContentBuilder responseBuilder = XContentFactory.jsonBuilder();
classes.toXContent(responseBuilder, null);
String result = null;
if (type.equals("long")) {
result =
"\"class\"{\"doc_count_error_upper_bound\":0,\"sum_other_doc_count\":0,\"buckets\":[{\"key\":\"0\",\"doc_count\":4,\"sig_terms\":{\"doc_count\":4,\"buckets\":[{\"key\":0,\"key_as_string\":\"0\",\"doc_count\":4,\"score\":0.39999999999999997,\"bg_count\":5}]}},{\"key\":\"1\",\"doc_count\":3,\"sig_terms\":{\"doc_count\":3,\"buckets\":[{\"key\":1,\"key_as_string\":\"1\",\"doc_count\":3,\"score\":0.75,\"bg_count\":4}]}}]}";
} else {
result =
"\"class\"{\"doc_count_error_upper_bound\":0,\"sum_other_doc_count\":0,\"buckets\":[{\"key\":\"0\",\"doc_count\":4,\"sig_terms\":{\"doc_count\":4,\"buckets\":[{\"key\":\"0\",\"doc_count\":4,\"score\":0.39999999999999997,\"bg_count\":5}]}},{\"key\":\"1\",\"doc_count\":3,\"sig_terms\":{\"doc_count\":3,\"buckets\":[{\"key\":\"1\",\"doc_count\":3,\"score\":0.75,\"bg_count\":4}]}}]}";
}
assertThat(responseBuilder.string(), equalTo(result));
}
@Test
public void multipleGroupByTest() throws Exception {
Set expectedAges =
new HashSet<Integer>(ContiguousSet.create(Range.closed(20, 40), DiscreteDomain.integers()));
Map<String, Set<Integer>> buckets = new HashMap<>();
Aggregations result =
query(String.format("SELECT COUNT(*) FROM %s/account GROUP BY gender, age", TEST_INDEX));
Terms gender = result.get("gender");
for (Terms.Bucket genderBucket : gender.getBuckets()) {
String genderKey = genderBucket.getKey();
buckets.put(genderKey, new HashSet<Integer>());
Terms ageBuckets = (Terms) genderBucket.getAggregations().get("age");
for (Terms.Bucket ageBucket : ageBuckets.getBuckets()) {
buckets.get(genderKey).add(Integer.parseInt(ageBucket.getKey()));
}
}
Assert.assertEquals(2, buckets.keySet().size());
Assert.assertEquals(expectedAges, buckets.get("m"));
Assert.assertEquals(expectedAges, buckets.get("f"));
}
public void testScriptScore() throws ExecutionException, InterruptedException, IOException {
indexRandomFrequencies01(randomBoolean() ? "string" : "long");
ScriptHeuristic.ScriptHeuristicBuilder scriptHeuristicBuilder =
getScriptSignificanceHeuristicBuilder();
ensureYellow();
SearchResponse response =
client()
.prepareSearch(INDEX_NAME)
.addAggregation(
new TermsBuilder("class")
.field(CLASS_FIELD)
.subAggregation(
new SignificantTermsBuilder("mySignificantTerms")
.field(TEXT_FIELD)
.executionHint(randomExecutionHint())
.significanceHeuristic(scriptHeuristicBuilder)
.minDocCount(1)
.shardSize(2)
.size(2)))
.execute()
.actionGet();
assertSearchResponse(response);
for (Terms.Bucket classBucket :
((Terms) response.getAggregations().get("class")).getBuckets()) {
for (SignificantTerms.Bucket bucket :
((SignificantTerms) classBucket.getAggregations().get("mySignificantTerms"))
.getBuckets()) {
assertThat(
bucket.getSignificanceScore(),
is(
(double) bucket.getSubsetDf()
+ bucket.getSubsetSize()
+ bucket.getSupersetDf()
+ bucket.getSupersetSize()));
}
}
}
public void testNestedSameDocIdProcessedMultipleTime() throws Exception {
assertAcked(
prepareCreate("idx4")
.setSettings(
Settings.builder()
.put(SETTING_NUMBER_OF_SHARDS, 1)
.put(SETTING_NUMBER_OF_REPLICAS, 0))
.addMapping(
"product",
"categories",
"type=text",
"name",
"type=text",
"property",
"type=nested"));
ensureGreen("idx4");
client()
.prepareIndex("idx4", "product", "1")
.setSource(
jsonBuilder()
.startObject()
.field("name", "product1")
.field("categories", "1", "2", "3", "4")
.startArray("property")
.startObject()
.field("id", 1)
.endObject()
.startObject()
.field("id", 2)
.endObject()
.startObject()
.field("id", 3)
.endObject()
.endArray()
.endObject())
.get();
client()
.prepareIndex("idx4", "product", "2")
.setSource(
jsonBuilder()
.startObject()
.field("name", "product2")
.field("categories", "1", "2")
.startArray("property")
.startObject()
.field("id", 1)
.endObject()
.startObject()
.field("id", 5)
.endObject()
.startObject()
.field("id", 4)
.endObject()
.endArray()
.endObject())
.get();
refresh();
SearchResponse response =
client()
.prepareSearch("idx4")
.setTypes("product")
.addAggregation(
terms("category")
.field("categories")
.subAggregation(
nested("property", "property")
.subAggregation(terms("property_id").field("property.id"))))
.get();
assertNoFailures(response);
assertHitCount(response, 2);
Terms category = response.getAggregations().get("category");
assertThat(category.getBuckets().size(), equalTo(4));
Terms.Bucket bucket = category.getBucketByKey("1");
assertThat(bucket.getDocCount(), equalTo(2L));
Nested property = bucket.getAggregations().get("property");
assertThat(property.getDocCount(), equalTo(6L));
Terms propertyId = property.getAggregations().get("property_id");
assertThat(propertyId.getBuckets().size(), equalTo(5));
assertThat(propertyId.getBucketByKey("1").getDocCount(), equalTo(2L));
assertThat(propertyId.getBucketByKey("2").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("3").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("4").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("5").getDocCount(), equalTo(1L));
bucket = category.getBucketByKey("2");
assertThat(bucket.getDocCount(), equalTo(2L));
property = bucket.getAggregations().get("property");
assertThat(property.getDocCount(), equalTo(6L));
propertyId = property.getAggregations().get("property_id");
assertThat(propertyId.getBuckets().size(), equalTo(5));
assertThat(propertyId.getBucketByKey("1").getDocCount(), equalTo(2L));
assertThat(propertyId.getBucketByKey("2").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("3").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("4").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("5").getDocCount(), equalTo(1L));
bucket = category.getBucketByKey("3");
assertThat(bucket.getDocCount(), equalTo(1L));
property = bucket.getAggregations().get("property");
assertThat(property.getDocCount(), equalTo(3L));
propertyId = property.getAggregations().get("property_id");
assertThat(propertyId.getBuckets().size(), equalTo(3));
assertThat(propertyId.getBucketByKey("1").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("2").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("3").getDocCount(), equalTo(1L));
bucket = category.getBucketByKey("4");
assertThat(bucket.getDocCount(), equalTo(1L));
property = bucket.getAggregations().get("property");
assertThat(property.getDocCount(), equalTo(3L));
propertyId = property.getAggregations().get("property_id");
assertThat(propertyId.getBuckets().size(), equalTo(3));
assertThat(propertyId.getBucketByKey("1").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("2").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("3").getDocCount(), equalTo(1L));
}
// Test based on: https://github.com/elastic/elasticsearch/issues/9280
public void testParentFilterResolvedCorrectly() throws Exception {
XContentBuilder mapping =
jsonBuilder()
.startObject()
.startObject("provider")
.startObject("properties")
.startObject("comments")
.field("type", "nested")
.startObject("properties")
.startObject("cid")
.field("type", "long")
.endObject()
.startObject("identifier")
.field("type", "keyword")
.endObject()
.startObject("tags")
.field("type", "nested")
.startObject("properties")
.startObject("tid")
.field("type", "long")
.endObject()
.startObject("name")
.field("type", "keyword")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.startObject("dates")
.field("type", "object")
.startObject("properties")
.startObject("day")
.field("type", "date")
.field("format", "dateOptionalTime")
.endObject()
.startObject("month")
.field("type", "object")
.startObject("properties")
.startObject("end")
.field("type", "date")
.field("format", "dateOptionalTime")
.endObject()
.startObject("start")
.field("type", "date")
.field("format", "dateOptionalTime")
.endObject()
.startObject("label")
.field("type", "keyword")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject();
assertAcked(
prepareCreate("idx2")
.setSettings(
Settings.builder()
.put(SETTING_NUMBER_OF_SHARDS, 1)
.put(SETTING_NUMBER_OF_REPLICAS, 0))
.addMapping("provider", mapping));
ensureGreen("idx2");
List<IndexRequestBuilder> indexRequests = new ArrayList<>(2);
indexRequests.add(
client()
.prepareIndex("idx2", "provider", "1")
.setSource(
"{\"dates\": {\"month\": {\"label\": \"2014-11\", \"end\": \"2014-11-30\", \"start\": \"2014-11-01\"}, \"day\": \"2014-11-30\"}, \"comments\": [{\"cid\": 3,\"identifier\": \"29111\"}, {\"cid\": 4,\"tags\": [{\"tid\" :44,\"name\": \"Roles\"}], \"identifier\": \"29101\"}]}"));
indexRequests.add(
client()
.prepareIndex("idx2", "provider", "2")
.setSource(
"{\"dates\": {\"month\": {\"label\": \"2014-12\", \"end\": \"2014-12-31\", \"start\": \"2014-12-01\"}, \"day\": \"2014-12-03\"}, \"comments\": [{\"cid\": 1, \"identifier\": \"29111\"}, {\"cid\": 2,\"tags\": [{\"tid\" : 22, \"name\": \"DataChannels\"}], \"identifier\": \"29101\"}]}"));
indexRandom(true, indexRequests);
SearchResponse response =
client()
.prepareSearch("idx2")
.setTypes("provider")
.addAggregation(
terms("startDate")
.field("dates.month.start")
.subAggregation(
terms("endDate")
.field("dates.month.end")
.subAggregation(
terms("period")
.field("dates.month.label")
.subAggregation(
nested("ctxt_idfier_nested", "comments")
.subAggregation(
filter(
"comment_filter",
termQuery("comments.identifier", "29111"))
.subAggregation(
nested("nested_tags", "comments.tags")
.subAggregation(
terms("tag")
.field(
"comments.tags.name"))))))))
.get();
assertNoFailures(response);
assertHitCount(response, 2);
Terms startDate = response.getAggregations().get("startDate");
assertThat(startDate.getBuckets().size(), equalTo(2));
Terms.Bucket bucket = startDate.getBucketByKey("2014-11-01T00:00:00.000Z");
assertThat(bucket.getDocCount(), equalTo(1L));
Terms endDate = bucket.getAggregations().get("endDate");
bucket = endDate.getBucketByKey("2014-11-30T00:00:00.000Z");
assertThat(bucket.getDocCount(), equalTo(1L));
Terms period = bucket.getAggregations().get("period");
bucket = period.getBucketByKey("2014-11");
assertThat(bucket.getDocCount(), equalTo(1L));
Nested comments = bucket.getAggregations().get("ctxt_idfier_nested");
assertThat(comments.getDocCount(), equalTo(2L));
Filter filter = comments.getAggregations().get("comment_filter");
assertThat(filter.getDocCount(), equalTo(1L));
Nested nestedTags = filter.getAggregations().get("nested_tags");
assertThat(nestedTags.getDocCount(), equalTo(0L)); // This must be 0
Terms tags = nestedTags.getAggregations().get("tag");
assertThat(tags.getBuckets().size(), equalTo(0)); // and this must be empty
bucket = startDate.getBucketByKey("2014-12-01T00:00:00.000Z");
assertThat(bucket.getDocCount(), equalTo(1L));
endDate = bucket.getAggregations().get("endDate");
bucket = endDate.getBucketByKey("2014-12-31T00:00:00.000Z");
assertThat(bucket.getDocCount(), equalTo(1L));
period = bucket.getAggregations().get("period");
bucket = period.getBucketByKey("2014-12");
assertThat(bucket.getDocCount(), equalTo(1L));
comments = bucket.getAggregations().get("ctxt_idfier_nested");
assertThat(comments.getDocCount(), equalTo(2L));
filter = comments.getAggregations().get("comment_filter");
assertThat(filter.getDocCount(), equalTo(1L));
nestedTags = filter.getAggregations().get("nested_tags");
assertThat(nestedTags.getDocCount(), equalTo(0L)); // This must be 0
tags = nestedTags.getAggregations().get("tag");
assertThat(tags.getBuckets().size(), equalTo(0)); // and this must be empty
}
@Test
public void testParentWithMultipleBuckets() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("test")
.setQuery(matchQuery("randomized", false))
.addAggregation(
terms("category")
.field("category")
.size(0)
.subAggregation(
children("to_comment")
.childType("comment")
.subAggregation(topHits("top_comments").addSort("_id", SortOrder.ASC))))
.get();
assertSearchResponse(searchResponse);
Terms categoryTerms = searchResponse.getAggregations().get("category");
assertThat(categoryTerms.getBuckets().size(), equalTo(3));
for (Terms.Bucket bucket : categoryTerms.getBuckets()) {
logger.info("bucket=" + bucket.getKey());
Children childrenBucket = bucket.getAggregations().get("to_comment");
TopHits topHits = childrenBucket.getAggregations().get("top_comments");
logger.info("total_hits={}", topHits.getHits().getTotalHits());
for (SearchHit searchHit : topHits.getHits()) {
logger.info(
"hit= {} {} {}", searchHit.sortValues()[0], searchHit.getType(), searchHit.getId());
}
}
Terms.Bucket categoryBucket = categoryTerms.getBucketByKey("a");
assertThat(categoryBucket.getKey(), equalTo("a"));
assertThat(categoryBucket.getDocCount(), equalTo(3l));
Children childrenBucket = categoryBucket.getAggregations().get("to_comment");
assertThat(childrenBucket.getName(), equalTo("to_comment"));
assertThat(childrenBucket.getDocCount(), equalTo(2l));
TopHits topHits = childrenBucket.getAggregations().get("top_comments");
assertThat(topHits.getHits().totalHits(), equalTo(2l));
assertThat(topHits.getHits().getAt(0).sortValues()[0].toString(), equalTo("a"));
assertThat(topHits.getHits().getAt(0).getId(), equalTo("a"));
assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
assertThat(topHits.getHits().getAt(1).sortValues()[0].toString(), equalTo("c"));
assertThat(topHits.getHits().getAt(1).getId(), equalTo("c"));
assertThat(topHits.getHits().getAt(1).getType(), equalTo("comment"));
categoryBucket = categoryTerms.getBucketByKey("b");
assertThat(categoryBucket.getKey(), equalTo("b"));
assertThat(categoryBucket.getDocCount(), equalTo(2l));
childrenBucket = categoryBucket.getAggregations().get("to_comment");
assertThat(childrenBucket.getName(), equalTo("to_comment"));
assertThat(childrenBucket.getDocCount(), equalTo(1l));
topHits = childrenBucket.getAggregations().get("top_comments");
assertThat(topHits.getHits().totalHits(), equalTo(1l));
assertThat(topHits.getHits().getAt(0).getId(), equalTo("c"));
assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
categoryBucket = categoryTerms.getBucketByKey("c");
assertThat(categoryBucket.getKey(), equalTo("c"));
assertThat(categoryBucket.getDocCount(), equalTo(2l));
childrenBucket = categoryBucket.getAggregations().get("to_comment");
assertThat(childrenBucket.getName(), equalTo("to_comment"));
assertThat(childrenBucket.getDocCount(), equalTo(1l));
topHits = childrenBucket.getAggregations().get("top_comments");
assertThat(topHits.getHits().totalHits(), equalTo(1l));
assertThat(topHits.getHits().getAt(0).getId(), equalTo("c"));
assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
}
public void testPlugin() throws Exception {
String type = randomBoolean() ? "string" : "long";
String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}";
SharedSignificantTermsTestMethods.index01Docs(type, settings, this);
SearchResponse response =
client()
.prepareSearch(INDEX_NAME)
.setTypes(DOC_TYPE)
.addAggregation(
new TermsBuilder("class")
.field(CLASS_FIELD)
.subAggregation(
(new SignificantTermsBuilder("sig_terms"))
.field(TEXT_FIELD)
.significanceHeuristic(new SimpleHeuristic.SimpleHeuristicBuilder())
.minDocCount(1)))
.execute()
.actionGet();
assertSearchResponse(response);
StringTerms classes = (StringTerms) response.getAggregations().get("class");
assertThat(classes.getBuckets().size(), equalTo(2));
for (Terms.Bucket classBucket : classes.getBuckets()) {
Map<String, Aggregation> aggs = classBucket.getAggregations().asMap();
assertTrue(aggs.containsKey("sig_terms"));
SignificantTerms agg = (SignificantTerms) aggs.get("sig_terms");
assertThat(agg.getBuckets().size(), equalTo(2));
Iterator<SignificantTerms.Bucket> bucketIterator = agg.iterator();
SignificantTerms.Bucket sigBucket = bucketIterator.next();
String term = sigBucket.getKeyAsString();
String classTerm = classBucket.getKeyAsString();
assertTrue(term.equals(classTerm));
assertThat(sigBucket.getSignificanceScore(), closeTo(2.0, 1.e-8));
sigBucket = bucketIterator.next();
assertThat(sigBucket.getSignificanceScore(), closeTo(1.0, 1.e-8));
}
// we run the same test again but this time we do not call assertSearchResponse() before the
// assertions
// the reason is that this would trigger toXContent and we would like to check that this has no
// potential side effects
response =
client()
.prepareSearch(INDEX_NAME)
.setTypes(DOC_TYPE)
.addAggregation(
new TermsBuilder("class")
.field(CLASS_FIELD)
.subAggregation(
(new SignificantTermsBuilder("sig_terms"))
.field(TEXT_FIELD)
.significanceHeuristic(new SimpleHeuristic.SimpleHeuristicBuilder())
.minDocCount(1)))
.execute()
.actionGet();
classes = (StringTerms) response.getAggregations().get("class");
assertThat(classes.getBuckets().size(), equalTo(2));
for (Terms.Bucket classBucket : classes.getBuckets()) {
Map<String, Aggregation> aggs = classBucket.getAggregations().asMap();
assertTrue(aggs.containsKey("sig_terms"));
SignificantTerms agg = (SignificantTerms) aggs.get("sig_terms");
assertThat(agg.getBuckets().size(), equalTo(2));
Iterator<SignificantTerms.Bucket> bucketIterator = agg.iterator();
SignificantTerms.Bucket sigBucket = bucketIterator.next();
String term = sigBucket.getKeyAsString();
String classTerm = classBucket.getKeyAsString();
assertTrue(term.equals(classTerm));
assertThat(sigBucket.getSignificanceScore(), closeTo(2.0, 1.e-8));
sigBucket = bucketIterator.next();
assertThat(sigBucket.getSignificanceScore(), closeTo(1.0, 1.e-8));
}
}
@Test
public void singleValuedField_OrderedBySubAggregationAsc_MultiHierarchyLevels() throws Exception {
boolean asc = randomBoolean();
SearchResponse response =
client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("tags")
.executionHint(randomExecutionHint())
.field("tag")
.order(Terms.Order.aggregation("filter1>filter2>stats.max", asc))
.subAggregation(
filter("filter1")
.filter(FilterBuilders.matchAllFilter())
.subAggregation(
filter("filter2")
.filter(FilterBuilders.matchAllFilter())
.subAggregation(stats("stats").field("i")))))
.execute()
.actionGet();
assertSearchResponse(response);
Terms tags = response.getAggregations().get("tags");
assertThat(tags, notNullValue());
assertThat(tags.getName(), equalTo("tags"));
assertThat(tags.getBuckets().size(), equalTo(2));
Iterator<Terms.Bucket> iters = tags.getBuckets().iterator();
// the max for "more" is 2
// the max for "less" is 4
Terms.Bucket tag = iters.next();
assertThat(tag, notNullValue());
assertThat(key(tag), equalTo(asc ? "more" : "less"));
assertThat(tag.getDocCount(), equalTo(asc ? 3l : 2l));
Filter filter1 = tag.getAggregations().get("filter1");
assertThat(filter1, notNullValue());
assertThat(filter1.getDocCount(), equalTo(asc ? 3l : 2l));
Filter filter2 = filter1.getAggregations().get("filter2");
assertThat(filter2, notNullValue());
assertThat(filter2.getDocCount(), equalTo(asc ? 3l : 2l));
Stats stats = filter2.getAggregations().get("stats");
assertThat(stats, notNullValue());
assertThat(stats.getMax(), equalTo(asc ? 2.0 : 4.0));
tag = iters.next();
assertThat(tag, notNullValue());
assertThat(key(tag), equalTo(asc ? "less" : "more"));
assertThat(tag.getDocCount(), equalTo(asc ? 2l : 3l));
filter1 = tag.getAggregations().get("filter1");
assertThat(filter1, notNullValue());
assertThat(filter1.getDocCount(), equalTo(asc ? 2l : 3l));
filter2 = filter1.getAggregations().get("filter2");
assertThat(filter2, notNullValue());
assertThat(filter2.getDocCount(), equalTo(asc ? 2l : 3l));
stats = filter2.getAggregations().get("stats");
assertThat(stats, notNullValue());
assertThat(stats.getMax(), equalTo(asc ? 4.0 : 2.0));
}