@Test
public void testSimpleSubAggregations() throws Exception {
final String query =
String.format(
"SELECT /*! DOCS_WITH_AGGREGATION(10) */ * FROM %s/account GROUP BY (gender), (state) ",
TEST_INDEX);
SqlElasticSearchRequestBuilder select = getSearchRequestBuilder(query);
SearchResponse response = (SearchResponse) select.get();
Aggregations result = response.getAggregations();
Terms gender = result.get("gender");
for (Terms.Bucket genderBucket : gender.getBuckets()) {
String genderKey = genderBucket.getKey();
Assert.assertTrue("Gender should be m or f", genderKey.equals("m") || genderKey.equals("f"));
}
Assert.assertEquals(2, gender.getBuckets().size());
Terms state = result.get("state");
for (Terms.Bucket stateBucket : state.getBuckets()) {
if (stateBucket.getKey().equalsIgnoreCase("ak")) {
Assert.assertTrue("There are 22 entries for state ak", stateBucket.getDocCount() == 22);
}
}
Assert.assertEquals(response.getHits().totalHits(), 1000);
Assert.assertEquals(response.getHits().hits().length, 10);
}
@Test
public void testChildrenAggs() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("test")
.setQuery(matchQuery("randomized", true))
.addAggregation(
terms("category")
.field("category")
.size(0)
.subAggregation(
children("to_comment")
.childType("comment")
.subAggregation(
terms("commenters")
.field("commenter")
.size(0)
.subAggregation(topHits("top_comments")))))
.get();
assertSearchResponse(searchResponse);
Terms categoryTerms = searchResponse.getAggregations().get("category");
assertThat(categoryTerms.getBuckets().size(), equalTo(categoryToControl.size()));
for (Map.Entry<String, Control> entry1 : categoryToControl.entrySet()) {
Terms.Bucket categoryBucket = categoryTerms.getBucketByKey(entry1.getKey());
assertThat(categoryBucket.getKey(), equalTo(entry1.getKey()));
assertThat(categoryBucket.getDocCount(), equalTo((long) entry1.getValue().articleIds.size()));
Children childrenBucket = categoryBucket.getAggregations().get("to_comment");
assertThat(childrenBucket.getName(), equalTo("to_comment"));
assertThat(childrenBucket.getDocCount(), equalTo((long) entry1.getValue().commentIds.size()));
assertThat(
(long) childrenBucket.getProperty("_count"),
equalTo((long) entry1.getValue().commentIds.size()));
Terms commentersTerms = childrenBucket.getAggregations().get("commenters");
assertThat((Terms) childrenBucket.getProperty("commenters"), sameInstance(commentersTerms));
assertThat(
commentersTerms.getBuckets().size(),
equalTo(entry1.getValue().commenterToCommentId.size()));
for (Map.Entry<String, Set<String>> entry2 :
entry1.getValue().commenterToCommentId.entrySet()) {
Terms.Bucket commentBucket = commentersTerms.getBucketByKey(entry2.getKey());
assertThat(commentBucket.getKey(), equalTo(entry2.getKey()));
assertThat(commentBucket.getDocCount(), equalTo((long) entry2.getValue().size()));
TopHits topHits = commentBucket.getAggregations().get("top_comments");
for (SearchHit searchHit : topHits.getHits().getHits()) {
assertThat(entry2.getValue().contains(searchHit.getId()), is(true));
}
}
}
}
@Test
public void reverseAnotherNestedGroupByOnNestedFieldWithFilterTestWithReverseNestedNoPath()
throws Exception {
Aggregations result =
query(
String.format(
"SELECT COUNT(*) FROM %s/nestedType GROUP BY nested(message.info),filter('myFilter',message.info = 'a'),reverse_nested(comment.data,'~comment')",
TEST_INDEX));
InternalNested nested = result.get("message.info@NESTED");
InternalFilter filter = nested.getAggregations().get("myFilter@FILTER");
Terms infos = filter.getAggregations().get("message.info");
Assert.assertEquals(1, infos.getBuckets().size());
for (Terms.Bucket bucket : infos.getBuckets()) {
InternalReverseNested reverseNested =
bucket.getAggregations().get("comment.data@NESTED_REVERSED");
InternalNested innerNested = reverseNested.getAggregations().get("comment.data@NESTED");
Terms terms = innerNested.getAggregations().get("comment.data");
Terms.Bucket internalBucket = terms.getBuckets().get(0);
long count = ((ValueCount) internalBucket.getAggregations().get("COUNT(*)")).getValue();
String key = internalBucket.getKey();
if (key.equalsIgnoreCase("ab")) {
Assert.assertEquals(2, count);
} else {
throw new Exception(String.format("Unexpected key. expected: only a . found: %s", key));
}
}
}
public Map<String, Long> countByField(IndexField indexField, FilterBuilder filter) {
Map<String, Long> counts = new HashMap<>();
SearchRequestBuilder request =
client
.prepareSearch(this.getIndexName())
.setTypes(this.getIndexType())
.setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filter))
.setSize(0)
.addAggregation(
AggregationBuilders.terms(indexField.field())
.field(indexField.field())
.order(Terms.Order.count(false))
.size(Integer.MAX_VALUE)
.minDocCount(0));
SearchResponse response = request.get();
Terms values = response.getAggregations().get(indexField.field());
for (Terms.Bucket value : values.getBuckets()) {
counts.put(value.getKey(), value.getDocCount());
}
return counts;
}
@Test
public void testDocCount_asSubAgg() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
terms("terms")
.field("tag")
.order(Order.term(true))
.subAggregation(
histogram("histo")
.field(SINGLE_VALUED_FIELD_NAME)
.interval(interval)
.extendedBounds((long) minRandomValue, (long) maxRandomValue))
.subAggregation(maxBucket("max_bucket").setBucketsPaths("histo>_count")))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
List<Terms.Bucket> termsBuckets = terms.getBuckets();
assertThat(termsBuckets.size(), equalTo(interval));
for (int i = 0; i < interval; ++i) {
Terms.Bucket termsBucket = termsBuckets.get(i);
assertThat(termsBucket, notNullValue());
assertThat((String) termsBucket.getKey(), equalTo("tag" + (i % interval)));
Histogram histo = termsBucket.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(histo.getName(), equalTo("histo"));
List<? extends Bucket> buckets = histo.getBuckets();
List<String> maxKeys = new ArrayList<>();
double maxValue = Double.NEGATIVE_INFINITY;
for (int j = 0; j < numValueBuckets; ++j) {
Histogram.Bucket bucket = buckets.get(j);
assertThat(bucket, notNullValue());
assertThat(((Number) bucket.getKey()).longValue(), equalTo((long) j * interval));
if (bucket.getDocCount() > maxValue) {
maxValue = bucket.getDocCount();
maxKeys = new ArrayList<>();
maxKeys.add(bucket.getKeyAsString());
} else if (bucket.getDocCount() == maxValue) {
maxKeys.add(bucket.getKeyAsString());
}
}
InternalBucketMetricValue maxBucketValue = termsBucket.getAggregations().get("max_bucket");
assertThat(maxBucketValue, notNullValue());
assertThat(maxBucketValue.getName(), equalTo("max_bucket"));
assertThat(maxBucketValue.value(), equalTo(maxValue));
assertThat(maxBucketValue.keys(), equalTo(maxKeys.toArray(new String[maxKeys.size()])));
}
}
@Test
public void testSubAggregations() throws Exception {
Set expectedAges =
new HashSet<>(ContiguousSet.create(Range.closed(20, 40), DiscreteDomain.integers()));
final String query =
String.format(
"SELECT /*! DOCS_WITH_AGGREGATION(10) */"
+ " * FROM %s/account GROUP BY (gender, age), (state) LIMIT 0,10",
TEST_INDEX);
Map<String, Set<Integer>> buckets = new HashMap<>();
SqlElasticSearchRequestBuilder select = getSearchRequestBuilder(query);
SearchResponse response = (SearchResponse) select.get();
Aggregations result = response.getAggregations();
Terms gender = result.get("gender");
for (Terms.Bucket genderBucket : gender.getBuckets()) {
String genderKey = genderBucket.getKey();
buckets.put(genderKey, new HashSet<Integer>());
Terms ageBuckets = (Terms) genderBucket.getAggregations().get("age");
for (Terms.Bucket ageBucket : ageBuckets.getBuckets()) {
buckets.get(genderKey).add(Integer.parseInt(ageBucket.getKey()));
}
}
Assert.assertEquals(2, buckets.keySet().size());
Assert.assertEquals(expectedAges, buckets.get("m"));
Assert.assertEquals(expectedAges, buckets.get("f"));
Terms state = result.get("state");
for (Terms.Bucket stateBucket : state.getBuckets()) {
if (stateBucket.getKey().equalsIgnoreCase("ak")) {
Assert.assertTrue("There are 22 entries for state ak", stateBucket.getDocCount() == 22);
}
}
Assert.assertEquals(response.getHits().totalHits(), 1000);
Assert.assertEquals(response.getHits().hits().length, 10);
}
@Test
public void multipleGroupByTest() throws Exception {
Set expectedAges =
new HashSet<Integer>(ContiguousSet.create(Range.closed(20, 40), DiscreteDomain.integers()));
Map<String, Set<Integer>> buckets = new HashMap<>();
Aggregations result =
query(String.format("SELECT COUNT(*) FROM %s/account GROUP BY gender, age", TEST_INDEX));
Terms gender = result.get("gender");
for (Terms.Bucket genderBucket : gender.getBuckets()) {
String genderKey = genderBucket.getKey();
buckets.put(genderKey, new HashSet<Integer>());
Terms ageBuckets = (Terms) genderBucket.getAggregations().get("age");
for (Terms.Bucket ageBucket : ageBuckets.getBuckets()) {
buckets.get(genderKey).add(Integer.parseInt(ageBucket.getKey()));
}
}
Assert.assertEquals(2, buckets.keySet().size());
Assert.assertEquals(expectedAges, buckets.get("m"));
Assert.assertEquals(expectedAges, buckets.get("f"));
}
/*
* 所有姓氏为"Smith"的员工的最大的共同点(兴趣爱好)是什么
* 聚合变成只包含和查询语句想匹配的文档
*/
@Test
public void queryAggregationTest() {
Client client = ClientTemplate.getClient();
SearchResponse response =
client
.prepareSearch("fccs")
.setTypes("employee")
.setQuery(QueryBuilders.matchQuery("lastName", "Smith"))
.addAggregation(AggregationBuilders.terms("by_interests").field("interests"))
.get();
Terms terms = response.getAggregations().get("by_interests");
for (Terms.Bucket bucket : terms.getBuckets()) {
System.out.println("key:" + bucket.getKey() + " doc_count:" + bucket.getDocCount());
}
client.close();
}
@Test
public void groupByTest() throws Exception {
Aggregations result =
query(String.format("SELECT COUNT(*) FROM %s/account GROUP BY gender", TEST_INDEX));
Terms gender = result.get("gender");
for (Terms.Bucket bucket : gender.getBuckets()) {
String key = bucket.getKey();
long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue();
if (key.equalsIgnoreCase("m")) {
Assert.assertEquals(507, count);
} else if (key.equalsIgnoreCase("f")) {
Assert.assertEquals(493, count);
} else {
throw new Exception(String.format("Unexpected key. expected: m OR f. found: %s", key));
}
}
}
@Test
public void testMetric_topLevel() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
terms("terms")
.field("tag")
.subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME)))
.addAggregation(maxBucket("max_bucket").setBucketsPaths("terms>sum"))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
List<Terms.Bucket> buckets = terms.getBuckets();
assertThat(buckets.size(), equalTo(interval));
List<String> maxKeys = new ArrayList<>();
double maxValue = Double.NEGATIVE_INFINITY;
for (int i = 0; i < interval; ++i) {
Terms.Bucket bucket = buckets.get(i);
assertThat(bucket, notNullValue());
assertThat((String) bucket.getKey(), equalTo("tag" + (i % interval)));
assertThat(bucket.getDocCount(), greaterThan(0l));
Sum sum = bucket.getAggregations().get("sum");
assertThat(sum, notNullValue());
if (sum.value() > maxValue) {
maxValue = sum.value();
maxKeys = new ArrayList<>();
maxKeys.add(bucket.getKeyAsString());
} else if (sum.value() == maxValue) {
maxKeys.add(bucket.getKeyAsString());
}
}
InternalBucketMetricValue maxBucketValue = response.getAggregations().get("max_bucket");
assertThat(maxBucketValue, notNullValue());
assertThat(maxBucketValue.getName(), equalTo("max_bucket"));
assertThat(maxBucketValue.value(), equalTo(maxValue));
assertThat(maxBucketValue.keys(), equalTo(maxKeys.toArray(new String[maxKeys.size()])));
}
// Response helpers
protected Multimap<String, FacetValue> processAggregations(Aggregations aggregations) {
Multimap<String, FacetValue> stats = ArrayListMultimap.create();
if (aggregations != null) {
for (Aggregation aggregation : aggregations.asList()) {
if (aggregation instanceof StringTerms) {
for (Terms.Bucket value : ((Terms) aggregation).getBuckets()) {
FacetValue facetValue = new FacetValue(value.getKey(), value.getDocCount());
stats.put(aggregation.getName(), facetValue);
}
} else if (aggregation instanceof InternalValueCount) {
InternalValueCount count = (InternalValueCount) aggregation;
FacetValue facetValue = new FacetValue(count.getName(), count.getValue());
stats.put(count.getName(), facetValue);
}
}
}
return stats;
}
public Map<QualityProfileKey, Multimap<String, FacetValue>> getStatsByProfileKey(
List<QualityProfileKey> keys) {
String[] stringKeys = new String[keys.size()];
for (int i = 0; i < keys.size(); i++) {
stringKeys[i] = keys.get(i).toString();
}
SearchResponse response =
getClient()
.prepareSearch(this.getIndexName())
.setQuery(
QueryBuilders.filteredQuery(
QueryBuilders.matchAllQuery(),
FilterBuilders.termsFilter(
ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field(), stringKeys)))
.addAggregation(
AggregationBuilders.terms(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field())
.field(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field())
.subAggregation(
AggregationBuilders.terms(
ActiveRuleNormalizer.ActiveRuleField.INHERITANCE.field())
.field(ActiveRuleNormalizer.ActiveRuleField.INHERITANCE.field()))
.subAggregation(
AggregationBuilders.terms(
ActiveRuleNormalizer.ActiveRuleField.SEVERITY.field())
.field(ActiveRuleNormalizer.ActiveRuleField.SEVERITY.field())))
.setSize(0)
.setTypes(this.getIndexType())
.get();
Map<QualityProfileKey, Multimap<String, FacetValue>> stats =
new HashMap<QualityProfileKey, Multimap<String, FacetValue>>();
Aggregation aggregation =
response.getAggregations().get(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field());
for (Terms.Bucket value : ((Terms) aggregation).getBuckets()) {
stats.put(
QualityProfileKey.parse(value.getKey()),
this.processAggregations(value.getAggregations()));
}
return stats;
}
@Test
public void groupByOnNestedFieldWithFilterTest() throws Exception {
Aggregations result =
query(
String.format(
"SELECT COUNT(*) FROM %s/nestedType GROUP BY nested(message.info),filter('myFilter',message.info = 'a')",
TEST_INDEX));
InternalNested nested = result.get("message.info@NESTED");
InternalFilter filter = nested.getAggregations().get("myFilter@FILTER");
Terms infos = filter.getAggregations().get("message.info");
Assert.assertEquals(1, infos.getBuckets().size());
for (Terms.Bucket bucket : infos.getBuckets()) {
String key = bucket.getKey();
long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue();
if (key.equalsIgnoreCase("a")) {
Assert.assertEquals(2, count);
} else {
throw new Exception(String.format("Unexpected key. expected: only a . found: %s", key));
}
}
}
@Test
public void multipleGroupBysWithSize() throws Exception {
Set expectedAges =
new HashSet<Integer>(ContiguousSet.create(Range.closed(20, 40), DiscreteDomain.integers()));
Map<String, Set<Integer>> buckets = new HashMap<>();
Aggregations result =
query(
String.format(
"SELECT COUNT(*) FROM %s/account GROUP BY gender, terms('alias'='ageAgg','field'='age','size'=3)",
TEST_INDEX));
Terms gender = result.get("gender");
Assert.assertEquals(2, gender.getBuckets().size());
for (Terms.Bucket genderBucket : gender.getBuckets()) {
String genderKey = genderBucket.getKey();
buckets.put(genderKey, new HashSet<Integer>());
Terms ageBuckets = genderBucket.getAggregations().get("ageAgg");
Assert.assertEquals(3, ageBuckets.getBuckets().size());
}
}
@Override
public Map<String, Integer> countTermsByField(SearchCriteria searchCriteria, String field) {
ElasticSearchCriteria criteria = (ElasticSearchCriteria) searchCriteria;
if (criteria == null) return null;
SearchRequestBuilder srb = criteria2builder(criteria);
srb.setFrom(0);
srb.setSize(0);
TermsBuilder tb = AggregationBuilders.terms(field);
tb.field(field);
srb.addAggregation(tb);
try {
SearchResponse response = srb.execute().get();
StringTerms aggr = response.getAggregations().get(field);
Map<String, Integer> result = new LinkedHashMap<>();
for (Terms.Bucket bucket : aggr.getBuckets()) {
result.put(bucket.getKey(), (int) bucket.getDocCount());
}
return result;
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
return Collections.emptyMap();
}
private String key(Terms.Bucket bucket) {
return randomBoolean() ? bucket.getKey() : bucket.getKeyAsText().string();
}
@Test
public void testParentWithMultipleBuckets() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("test")
.setQuery(matchQuery("randomized", false))
.addAggregation(
terms("category")
.field("category")
.size(0)
.subAggregation(
children("to_comment")
.childType("comment")
.subAggregation(topHits("top_comments").addSort("_id", SortOrder.ASC))))
.get();
assertSearchResponse(searchResponse);
Terms categoryTerms = searchResponse.getAggregations().get("category");
assertThat(categoryTerms.getBuckets().size(), equalTo(3));
for (Terms.Bucket bucket : categoryTerms.getBuckets()) {
logger.info("bucket=" + bucket.getKey());
Children childrenBucket = bucket.getAggregations().get("to_comment");
TopHits topHits = childrenBucket.getAggregations().get("top_comments");
logger.info("total_hits={}", topHits.getHits().getTotalHits());
for (SearchHit searchHit : topHits.getHits()) {
logger.info(
"hit= {} {} {}", searchHit.sortValues()[0], searchHit.getType(), searchHit.getId());
}
}
Terms.Bucket categoryBucket = categoryTerms.getBucketByKey("a");
assertThat(categoryBucket.getKey(), equalTo("a"));
assertThat(categoryBucket.getDocCount(), equalTo(3l));
Children childrenBucket = categoryBucket.getAggregations().get("to_comment");
assertThat(childrenBucket.getName(), equalTo("to_comment"));
assertThat(childrenBucket.getDocCount(), equalTo(2l));
TopHits topHits = childrenBucket.getAggregations().get("top_comments");
assertThat(topHits.getHits().totalHits(), equalTo(2l));
assertThat(topHits.getHits().getAt(0).sortValues()[0].toString(), equalTo("a"));
assertThat(topHits.getHits().getAt(0).getId(), equalTo("a"));
assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
assertThat(topHits.getHits().getAt(1).sortValues()[0].toString(), equalTo("c"));
assertThat(topHits.getHits().getAt(1).getId(), equalTo("c"));
assertThat(topHits.getHits().getAt(1).getType(), equalTo("comment"));
categoryBucket = categoryTerms.getBucketByKey("b");
assertThat(categoryBucket.getKey(), equalTo("b"));
assertThat(categoryBucket.getDocCount(), equalTo(2l));
childrenBucket = categoryBucket.getAggregations().get("to_comment");
assertThat(childrenBucket.getName(), equalTo("to_comment"));
assertThat(childrenBucket.getDocCount(), equalTo(1l));
topHits = childrenBucket.getAggregations().get("top_comments");
assertThat(topHits.getHits().totalHits(), equalTo(1l));
assertThat(topHits.getHits().getAt(0).getId(), equalTo("c"));
assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
categoryBucket = categoryTerms.getBucketByKey("c");
assertThat(categoryBucket.getKey(), equalTo("c"));
assertThat(categoryBucket.getDocCount(), equalTo(2l));
childrenBucket = categoryBucket.getAggregations().get("to_comment");
assertThat(childrenBucket.getName(), equalTo("to_comment"));
assertThat(childrenBucket.getDocCount(), equalTo(1l));
topHits = childrenBucket.getAggregations().get("top_comments");
assertThat(topHits.getHits().totalHits(), equalTo(1l));
assertThat(topHits.getHits().getAt(0).getId(), equalTo("c"));
assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
}