@Test
public void testChildrenAggs() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("test")
.setQuery(matchQuery("randomized", true))
.addAggregation(
terms("category")
.field("category")
.size(0)
.subAggregation(
children("to_comment")
.childType("comment")
.subAggregation(
terms("commenters")
.field("commenter")
.size(0)
.subAggregation(topHits("top_comments")))))
.get();
assertSearchResponse(searchResponse);
Terms categoryTerms = searchResponse.getAggregations().get("category");
assertThat(categoryTerms.getBuckets().size(), equalTo(categoryToControl.size()));
for (Map.Entry<String, Control> entry1 : categoryToControl.entrySet()) {
Terms.Bucket categoryBucket = categoryTerms.getBucketByKey(entry1.getKey());
assertThat(categoryBucket.getKey(), equalTo(entry1.getKey()));
assertThat(categoryBucket.getDocCount(), equalTo((long) entry1.getValue().articleIds.size()));
Children childrenBucket = categoryBucket.getAggregations().get("to_comment");
assertThat(childrenBucket.getName(), equalTo("to_comment"));
assertThat(childrenBucket.getDocCount(), equalTo((long) entry1.getValue().commentIds.size()));
assertThat(
(long) childrenBucket.getProperty("_count"),
equalTo((long) entry1.getValue().commentIds.size()));
Terms commentersTerms = childrenBucket.getAggregations().get("commenters");
assertThat((Terms) childrenBucket.getProperty("commenters"), sameInstance(commentersTerms));
assertThat(
commentersTerms.getBuckets().size(),
equalTo(entry1.getValue().commenterToCommentId.size()));
for (Map.Entry<String, Set<String>> entry2 :
entry1.getValue().commenterToCommentId.entrySet()) {
Terms.Bucket commentBucket = commentersTerms.getBucketByKey(entry2.getKey());
assertThat(commentBucket.getKey(), equalTo(entry2.getKey()));
assertThat(commentBucket.getDocCount(), equalTo((long) entry2.getValue().size()));
TopHits topHits = commentBucket.getAggregations().get("top_comments");
for (SearchHit searchHit : topHits.getHits().getHits()) {
assertThat(entry2.getValue().contains(searchHit.getId()), is(true));
}
}
}
}
@Test
public void multiValuedField_WithValueScript_NotUnique() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms")
.executionHint(randomExecutionHint())
.field(MULTI_VALUED_FIELD_NAME)
.script("_value.substring(0,3)"))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(1));
Terms.Bucket bucket = terms.getBucketByKey("val");
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val"));
assertThat(bucket.getDocCount(), equalTo(5l));
}
@Test
public void script_SingleValue_WithSubAggregator_Inherited() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms")
.executionHint(randomExecutionHint())
.script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value")
.subAggregation(count("count")))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
for (int i = 0; i < 5; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
assertThat(bucket.getDocCount(), equalTo(1l));
ValueCount valueCount = bucket.getAggregations().get("count");
assertThat(valueCount, notNullValue());
assertThat(valueCount.getValue(), equalTo(1l));
}
}
@Test
public void script_MultiValued() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms")
.executionHint(randomExecutionHint())
.script("doc['" + MULTI_VALUED_FIELD_NAME + "'].values"))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(6));
for (int i = 0; i < 6; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
if (i == 0 || i == 5) {
assertThat(bucket.getDocCount(), equalTo(1l));
} else {
assertThat(bucket.getDocCount(), equalTo(2l));
}
}
}
@Test
public void partiallyUnmapped() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx", "idx_unmapped")
.setTypes("type")
.addAggregation(
terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
for (int i = 0; i < 5; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
assertThat(bucket.getDocCount(), equalTo(1l));
}
}
@Test
public void stringTermsNestedIntoPerBucketAggregator() throws Exception {
// no execution hint so that the logic that decides whether or not to use ordinals is executed
SearchResponse response =
client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
filter("filter")
.filter(termFilter(MULTI_VALUED_FIELD_NAME, "val3"))
.subAggregation(terms("terms").field(MULTI_VALUED_FIELD_NAME)))
.execute()
.actionGet();
assertThat(response.getFailedShards(), equalTo(0));
Filter filter = response.getAggregations().get("filter");
Terms terms = filter.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(3));
for (int i = 2; i <= 4; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
assertThat(bucket.getDocCount(), equalTo(i == 3 ? 2L : 1L));
}
}
@Test
public void singleValueField_WithMaxSize() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.setTypes("high_card_type")
.addAggregation(
terms("terms")
.executionHint(randomExecutionHint())
.field(SINGLE_VALUED_FIELD_NAME)
.size(20)
.order(
Terms.Order.term(
true))) // we need to sort by terms cause we're checking the first 20
// values
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(20));
for (int i = 0; i < 20; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val" + Strings.padStart(i + "", 3, '0'));
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + Strings.padStart(i + "", 3, '0')));
assertThat(bucket.getDocCount(), equalTo(1l));
}
}
public void testUnmappedTerms() {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(terms("my_terms").field("non_existing_field").missing("bar"))
.get();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("my_terms");
assertEquals(1, terms.getBuckets().size());
assertEquals(2, terms.getBucketByKey("bar").getDocCount());
}
public void testDoubleTerms() {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(terms("my_terms").field("double").missing(4.5))
.get();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("my_terms");
assertEquals(2, terms.getBuckets().size());
assertEquals(1, terms.getBucketByKey("4.5").getDocCount());
assertEquals(1, terms.getBucketByKey("5.5").getDocCount());
response =
client()
.prepareSearch("idx")
.addAggregation(terms("my_terms").field("double").missing(5.5))
.get();
assertSearchResponse(response);
terms = response.getAggregations().get("my_terms");
assertEquals(1, terms.getBuckets().size());
assertEquals(2, terms.getBucketByKey("5.5").getDocCount());
}
private void assertDocCountErrorWithinBounds(
int size, SearchResponse accurateResponse, SearchResponse testResponse) {
Terms accurateTerms = accurateResponse.getAggregations().get("terms");
assertThat(accurateTerms, notNullValue());
assertThat(accurateTerms.getName(), equalTo("terms"));
assertThat(accurateTerms.getDocCountError(), equalTo(0L));
Terms testTerms = testResponse.getAggregations().get("terms");
assertThat(testTerms, notNullValue());
assertThat(testTerms.getName(), equalTo("terms"));
assertThat(testTerms.getDocCountError(), greaterThanOrEqualTo(0L));
Collection<Bucket> testBuckets = testTerms.getBuckets();
assertThat(testBuckets.size(), lessThanOrEqualTo(size));
assertThat(accurateTerms.getBuckets().size(), greaterThanOrEqualTo(testBuckets.size()));
for (Terms.Bucket testBucket : testBuckets) {
assertThat(testBucket, notNullValue());
Terms.Bucket accurateBucket = accurateTerms.getBucketByKey(testBucket.getKeyAsString());
assertThat(accurateBucket, notNullValue());
assertThat(accurateBucket.getDocCountError(), equalTo(0L));
assertThat(testBucket.getDocCountError(), lessThanOrEqualTo(testTerms.getDocCountError()));
assertThat(
testBucket.getDocCount() + testBucket.getDocCountError(),
greaterThanOrEqualTo(accurateBucket.getDocCount()));
assertThat(
testBucket.getDocCount() - testBucket.getDocCountError(),
lessThanOrEqualTo(accurateBucket.getDocCount()));
}
for (Terms.Bucket accurateBucket : accurateTerms.getBuckets()) {
assertThat(accurateBucket, notNullValue());
Terms.Bucket testBucket = accurateTerms.getBucketByKey(accurateBucket.getKeyAsString());
if (testBucket == null) {
assertThat(accurateBucket.getDocCount(), lessThanOrEqualTo(testTerms.getDocCountError()));
}
}
}
@Test
public void multiValuedField_WithValueScript_WithInheritedSubAggregator() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(
terms("terms")
.executionHint(randomExecutionHint())
.field(MULTI_VALUED_FIELD_NAME)
.script("'foo_' + _value")
.subAggregation(count("count")))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(6));
for (int i = 0; i < 6; i++) {
Terms.Bucket bucket = terms.getBucketByKey("foo_val" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("foo_val" + i));
if (i == 0 | i == 5) {
assertThat(bucket.getDocCount(), equalTo(1l));
ValueCount valueCount = bucket.getAggregations().get("count");
assertThat(valueCount, notNullValue());
assertThat(valueCount.getValue(), equalTo(2l));
} else {
assertThat(bucket.getDocCount(), equalTo(2l));
ValueCount valueCount = bucket.getAggregations().get("count");
assertThat(valueCount, notNullValue());
assertThat("term[" + key(bucket) + "]", valueCount.getValue(), equalTo(4l));
}
}
}
public void testNestedSameDocIdProcessedMultipleTime() throws Exception {
assertAcked(
prepareCreate("idx4")
.setSettings(
Settings.builder()
.put(SETTING_NUMBER_OF_SHARDS, 1)
.put(SETTING_NUMBER_OF_REPLICAS, 0))
.addMapping(
"product",
"categories",
"type=text",
"name",
"type=text",
"property",
"type=nested"));
ensureGreen("idx4");
client()
.prepareIndex("idx4", "product", "1")
.setSource(
jsonBuilder()
.startObject()
.field("name", "product1")
.field("categories", "1", "2", "3", "4")
.startArray("property")
.startObject()
.field("id", 1)
.endObject()
.startObject()
.field("id", 2)
.endObject()
.startObject()
.field("id", 3)
.endObject()
.endArray()
.endObject())
.get();
client()
.prepareIndex("idx4", "product", "2")
.setSource(
jsonBuilder()
.startObject()
.field("name", "product2")
.field("categories", "1", "2")
.startArray("property")
.startObject()
.field("id", 1)
.endObject()
.startObject()
.field("id", 5)
.endObject()
.startObject()
.field("id", 4)
.endObject()
.endArray()
.endObject())
.get();
refresh();
SearchResponse response =
client()
.prepareSearch("idx4")
.setTypes("product")
.addAggregation(
terms("category")
.field("categories")
.subAggregation(
nested("property", "property")
.subAggregation(terms("property_id").field("property.id"))))
.get();
assertNoFailures(response);
assertHitCount(response, 2);
Terms category = response.getAggregations().get("category");
assertThat(category.getBuckets().size(), equalTo(4));
Terms.Bucket bucket = category.getBucketByKey("1");
assertThat(bucket.getDocCount(), equalTo(2L));
Nested property = bucket.getAggregations().get("property");
assertThat(property.getDocCount(), equalTo(6L));
Terms propertyId = property.getAggregations().get("property_id");
assertThat(propertyId.getBuckets().size(), equalTo(5));
assertThat(propertyId.getBucketByKey("1").getDocCount(), equalTo(2L));
assertThat(propertyId.getBucketByKey("2").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("3").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("4").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("5").getDocCount(), equalTo(1L));
bucket = category.getBucketByKey("2");
assertThat(bucket.getDocCount(), equalTo(2L));
property = bucket.getAggregations().get("property");
assertThat(property.getDocCount(), equalTo(6L));
propertyId = property.getAggregations().get("property_id");
assertThat(propertyId.getBuckets().size(), equalTo(5));
assertThat(propertyId.getBucketByKey("1").getDocCount(), equalTo(2L));
assertThat(propertyId.getBucketByKey("2").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("3").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("4").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("5").getDocCount(), equalTo(1L));
bucket = category.getBucketByKey("3");
assertThat(bucket.getDocCount(), equalTo(1L));
property = bucket.getAggregations().get("property");
assertThat(property.getDocCount(), equalTo(3L));
propertyId = property.getAggregations().get("property_id");
assertThat(propertyId.getBuckets().size(), equalTo(3));
assertThat(propertyId.getBucketByKey("1").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("2").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("3").getDocCount(), equalTo(1L));
bucket = category.getBucketByKey("4");
assertThat(bucket.getDocCount(), equalTo(1L));
property = bucket.getAggregations().get("property");
assertThat(property.getDocCount(), equalTo(3L));
propertyId = property.getAggregations().get("property_id");
assertThat(propertyId.getBuckets().size(), equalTo(3));
assertThat(propertyId.getBucketByKey("1").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("2").getDocCount(), equalTo(1L));
assertThat(propertyId.getBucketByKey("3").getDocCount(), equalTo(1L));
}
// Test based on: https://github.com/elastic/elasticsearch/issues/9280
public void testParentFilterResolvedCorrectly() throws Exception {
XContentBuilder mapping =
jsonBuilder()
.startObject()
.startObject("provider")
.startObject("properties")
.startObject("comments")
.field("type", "nested")
.startObject("properties")
.startObject("cid")
.field("type", "long")
.endObject()
.startObject("identifier")
.field("type", "keyword")
.endObject()
.startObject("tags")
.field("type", "nested")
.startObject("properties")
.startObject("tid")
.field("type", "long")
.endObject()
.startObject("name")
.field("type", "keyword")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.startObject("dates")
.field("type", "object")
.startObject("properties")
.startObject("day")
.field("type", "date")
.field("format", "dateOptionalTime")
.endObject()
.startObject("month")
.field("type", "object")
.startObject("properties")
.startObject("end")
.field("type", "date")
.field("format", "dateOptionalTime")
.endObject()
.startObject("start")
.field("type", "date")
.field("format", "dateOptionalTime")
.endObject()
.startObject("label")
.field("type", "keyword")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject();
assertAcked(
prepareCreate("idx2")
.setSettings(
Settings.builder()
.put(SETTING_NUMBER_OF_SHARDS, 1)
.put(SETTING_NUMBER_OF_REPLICAS, 0))
.addMapping("provider", mapping));
ensureGreen("idx2");
List<IndexRequestBuilder> indexRequests = new ArrayList<>(2);
indexRequests.add(
client()
.prepareIndex("idx2", "provider", "1")
.setSource(
"{\"dates\": {\"month\": {\"label\": \"2014-11\", \"end\": \"2014-11-30\", \"start\": \"2014-11-01\"}, \"day\": \"2014-11-30\"}, \"comments\": [{\"cid\": 3,\"identifier\": \"29111\"}, {\"cid\": 4,\"tags\": [{\"tid\" :44,\"name\": \"Roles\"}], \"identifier\": \"29101\"}]}"));
indexRequests.add(
client()
.prepareIndex("idx2", "provider", "2")
.setSource(
"{\"dates\": {\"month\": {\"label\": \"2014-12\", \"end\": \"2014-12-31\", \"start\": \"2014-12-01\"}, \"day\": \"2014-12-03\"}, \"comments\": [{\"cid\": 1, \"identifier\": \"29111\"}, {\"cid\": 2,\"tags\": [{\"tid\" : 22, \"name\": \"DataChannels\"}], \"identifier\": \"29101\"}]}"));
indexRandom(true, indexRequests);
SearchResponse response =
client()
.prepareSearch("idx2")
.setTypes("provider")
.addAggregation(
terms("startDate")
.field("dates.month.start")
.subAggregation(
terms("endDate")
.field("dates.month.end")
.subAggregation(
terms("period")
.field("dates.month.label")
.subAggregation(
nested("ctxt_idfier_nested", "comments")
.subAggregation(
filter(
"comment_filter",
termQuery("comments.identifier", "29111"))
.subAggregation(
nested("nested_tags", "comments.tags")
.subAggregation(
terms("tag")
.field(
"comments.tags.name"))))))))
.get();
assertNoFailures(response);
assertHitCount(response, 2);
Terms startDate = response.getAggregations().get("startDate");
assertThat(startDate.getBuckets().size(), equalTo(2));
Terms.Bucket bucket = startDate.getBucketByKey("2014-11-01T00:00:00.000Z");
assertThat(bucket.getDocCount(), equalTo(1L));
Terms endDate = bucket.getAggregations().get("endDate");
bucket = endDate.getBucketByKey("2014-11-30T00:00:00.000Z");
assertThat(bucket.getDocCount(), equalTo(1L));
Terms period = bucket.getAggregations().get("period");
bucket = period.getBucketByKey("2014-11");
assertThat(bucket.getDocCount(), equalTo(1L));
Nested comments = bucket.getAggregations().get("ctxt_idfier_nested");
assertThat(comments.getDocCount(), equalTo(2L));
Filter filter = comments.getAggregations().get("comment_filter");
assertThat(filter.getDocCount(), equalTo(1L));
Nested nestedTags = filter.getAggregations().get("nested_tags");
assertThat(nestedTags.getDocCount(), equalTo(0L)); // This must be 0
Terms tags = nestedTags.getAggregations().get("tag");
assertThat(tags.getBuckets().size(), equalTo(0)); // and this must be empty
bucket = startDate.getBucketByKey("2014-12-01T00:00:00.000Z");
assertThat(bucket.getDocCount(), equalTo(1L));
endDate = bucket.getAggregations().get("endDate");
bucket = endDate.getBucketByKey("2014-12-31T00:00:00.000Z");
assertThat(bucket.getDocCount(), equalTo(1L));
period = bucket.getAggregations().get("period");
bucket = period.getBucketByKey("2014-12");
assertThat(bucket.getDocCount(), equalTo(1L));
comments = bucket.getAggregations().get("ctxt_idfier_nested");
assertThat(comments.getDocCount(), equalTo(2L));
filter = comments.getAggregations().get("comment_filter");
assertThat(filter.getDocCount(), equalTo(1L));
nestedTags = filter.getAggregations().get("nested_tags");
assertThat(nestedTags.getDocCount(), equalTo(0L)); // This must be 0
tags = nestedTags.getAggregations().get("tag");
assertThat(tags.getBuckets().size(), equalTo(0)); // and this must be empty
}
// test long/double/string terms aggs with high number of buckets that require array growth
public void testDuelTerms() throws Exception {
final int numDocs = scaledRandomIntBetween(1000, 2000);
final int maxNumTerms = randomIntBetween(10, 5000);
final IntOpenHashSet valuesSet = new IntOpenHashSet();
cluster().wipeIndices("idx");
prepareCreate("idx")
.addMapping(
"type",
jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("string_values")
.field("type", "string")
.field("index", "not_analyzed")
.startObject("fields")
.startObject("doc_values")
.field("type", "string")
.field("index", "no")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.endObject()
.endObject()
.startObject("long_values")
.field("type", "long")
.endObject()
.startObject("double_values")
.field("type", "double")
.endObject()
.endObject()
.endObject())
.execute()
.actionGet();
List<IndexRequestBuilder> indexingRequests = Lists.newArrayList();
for (int i = 0; i < numDocs; ++i) {
final int[] values = new int[randomInt(4)];
for (int j = 0; j < values.length; ++j) {
values[j] = randomInt(maxNumTerms - 1) - 1000;
valuesSet.add(values[j]);
}
XContentBuilder source =
jsonBuilder().startObject().field("num", randomDouble()).startArray("long_values");
for (int j = 0; j < values.length; ++j) {
source = source.value(values[j]);
}
source = source.endArray().startArray("double_values");
for (int j = 0; j < values.length; ++j) {
source = source.value((double) values[j]);
}
source = source.endArray().startArray("string_values");
for (int j = 0; j < values.length; ++j) {
source = source.value(Integer.toString(values[j]));
}
source = source.endArray().endObject();
indexingRequests.add(client().prepareIndex("idx", "type").setSource(source));
}
indexRandom(true, indexingRequests);
assertNoFailures(
client()
.admin()
.indices()
.prepareRefresh("idx")
.setIndicesOptions(IndicesOptions.lenientExpandOpen())
.execute()
.get());
TermsAggregatorFactory.ExecutionMode[] globalOrdinalModes =
new TermsAggregatorFactory.ExecutionMode[] {
TermsAggregatorFactory.ExecutionMode.GLOBAL_ORDINALS_HASH,
TermsAggregatorFactory.ExecutionMode.GLOBAL_ORDINALS
};
SearchResponse resp =
client()
.prepareSearch("idx")
.addAggregation(
terms("long")
.field("long_values")
.size(maxNumTerms)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.subAggregation(min("min").field("num")))
.addAggregation(
terms("double")
.field("double_values")
.size(maxNumTerms)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.subAggregation(max("max").field("num")))
.addAggregation(
terms("string_map")
.field("string_values")
.collectMode(randomFrom(SubAggCollectionMode.values()))
.executionHint(TermsAggregatorFactory.ExecutionMode.MAP.toString())
.size(maxNumTerms)
.subAggregation(stats("stats").field("num")))
.addAggregation(
terms("string_global_ordinals")
.field("string_values")
.collectMode(randomFrom(SubAggCollectionMode.values()))
.executionHint(
globalOrdinalModes[randomInt(globalOrdinalModes.length - 1)].toString())
.size(maxNumTerms)
.subAggregation(extendedStats("stats").field("num")))
.addAggregation(
terms("string_global_ordinals_doc_values")
.field("string_values.doc_values")
.collectMode(randomFrom(SubAggCollectionMode.values()))
.executionHint(
globalOrdinalModes[randomInt(globalOrdinalModes.length - 1)].toString())
.size(maxNumTerms)
.subAggregation(extendedStats("stats").field("num")))
.execute()
.actionGet();
assertAllSuccessful(resp);
assertEquals(numDocs, resp.getHits().getTotalHits());
final Terms longTerms = resp.getAggregations().get("long");
final Terms doubleTerms = resp.getAggregations().get("double");
final Terms stringMapTerms = resp.getAggregations().get("string_map");
final Terms stringGlobalOrdinalsTerms = resp.getAggregations().get("string_global_ordinals");
final Terms stringGlobalOrdinalsDVTerms =
resp.getAggregations().get("string_global_ordinals_doc_values");
assertEquals(valuesSet.size(), longTerms.getBuckets().size());
assertEquals(valuesSet.size(), doubleTerms.getBuckets().size());
assertEquals(valuesSet.size(), stringMapTerms.getBuckets().size());
assertEquals(valuesSet.size(), stringGlobalOrdinalsTerms.getBuckets().size());
assertEquals(valuesSet.size(), stringGlobalOrdinalsDVTerms.getBuckets().size());
for (Terms.Bucket bucket : longTerms.getBuckets()) {
final Terms.Bucket doubleBucket =
doubleTerms.getBucketByKey(
Double.toString(Long.parseLong(bucket.getKeyAsText().string())));
final Terms.Bucket stringMapBucket =
stringMapTerms.getBucketByKey(bucket.getKeyAsText().string());
final Terms.Bucket stringGlobalOrdinalsBucket =
stringGlobalOrdinalsTerms.getBucketByKey(bucket.getKeyAsText().string());
final Terms.Bucket stringGlobalOrdinalsDVBucket =
stringGlobalOrdinalsDVTerms.getBucketByKey(bucket.getKeyAsText().string());
assertNotNull(doubleBucket);
assertNotNull(stringMapBucket);
assertNotNull(stringGlobalOrdinalsBucket);
assertNotNull(stringGlobalOrdinalsDVBucket);
assertEquals(bucket.getDocCount(), doubleBucket.getDocCount());
assertEquals(bucket.getDocCount(), stringMapBucket.getDocCount());
assertEquals(bucket.getDocCount(), stringGlobalOrdinalsBucket.getDocCount());
assertEquals(bucket.getDocCount(), stringGlobalOrdinalsDVBucket.getDocCount());
}
}
@Test
public void testParentWithMultipleBuckets() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("test")
.setQuery(matchQuery("randomized", false))
.addAggregation(
terms("category")
.field("category")
.size(0)
.subAggregation(
children("to_comment")
.childType("comment")
.subAggregation(topHits("top_comments").addSort("_id", SortOrder.ASC))))
.get();
assertSearchResponse(searchResponse);
Terms categoryTerms = searchResponse.getAggregations().get("category");
assertThat(categoryTerms.getBuckets().size(), equalTo(3));
for (Terms.Bucket bucket : categoryTerms.getBuckets()) {
logger.info("bucket=" + bucket.getKey());
Children childrenBucket = bucket.getAggregations().get("to_comment");
TopHits topHits = childrenBucket.getAggregations().get("top_comments");
logger.info("total_hits={}", topHits.getHits().getTotalHits());
for (SearchHit searchHit : topHits.getHits()) {
logger.info(
"hit= {} {} {}", searchHit.sortValues()[0], searchHit.getType(), searchHit.getId());
}
}
Terms.Bucket categoryBucket = categoryTerms.getBucketByKey("a");
assertThat(categoryBucket.getKey(), equalTo("a"));
assertThat(categoryBucket.getDocCount(), equalTo(3l));
Children childrenBucket = categoryBucket.getAggregations().get("to_comment");
assertThat(childrenBucket.getName(), equalTo("to_comment"));
assertThat(childrenBucket.getDocCount(), equalTo(2l));
TopHits topHits = childrenBucket.getAggregations().get("top_comments");
assertThat(topHits.getHits().totalHits(), equalTo(2l));
assertThat(topHits.getHits().getAt(0).sortValues()[0].toString(), equalTo("a"));
assertThat(topHits.getHits().getAt(0).getId(), equalTo("a"));
assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
assertThat(topHits.getHits().getAt(1).sortValues()[0].toString(), equalTo("c"));
assertThat(topHits.getHits().getAt(1).getId(), equalTo("c"));
assertThat(topHits.getHits().getAt(1).getType(), equalTo("comment"));
categoryBucket = categoryTerms.getBucketByKey("b");
assertThat(categoryBucket.getKey(), equalTo("b"));
assertThat(categoryBucket.getDocCount(), equalTo(2l));
childrenBucket = categoryBucket.getAggregations().get("to_comment");
assertThat(childrenBucket.getName(), equalTo("to_comment"));
assertThat(childrenBucket.getDocCount(), equalTo(1l));
topHits = childrenBucket.getAggregations().get("top_comments");
assertThat(topHits.getHits().totalHits(), equalTo(1l));
assertThat(topHits.getHits().getAt(0).getId(), equalTo("c"));
assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
categoryBucket = categoryTerms.getBucketByKey("c");
assertThat(categoryBucket.getKey(), equalTo("c"));
assertThat(categoryBucket.getDocCount(), equalTo(2l));
childrenBucket = categoryBucket.getAggregations().get("to_comment");
assertThat(childrenBucket.getName(), equalTo("to_comment"));
assertThat(childrenBucket.getDocCount(), equalTo(1l));
topHits = childrenBucket.getAggregations().get("top_comments");
assertThat(topHits.getHits().totalHits(), equalTo(1l));
assertThat(topHits.getHits().getAt(0).getId(), equalTo("c"));
assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
}
@Test
public void singleValueField_WithRegexFiltering() throws Exception {
// include without exclude
// we should be left with: val000, val001, val002, val003, val004, val005, val006, val007,
// val008, val009
SearchResponse response =
client()
.prepareSearch("idx")
.setTypes("high_card_type")
.addAggregation(terms("terms").field(SINGLE_VALUED_FIELD_NAME).include("val00.+"))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(10));
for (int i = 0; i < 10; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val00" + i));
assertThat(bucket.getDocCount(), equalTo(1l));
}
// include and exclude
// we should be left with: val002, val003, val004, val005, val006, val007, val008, val009
response =
client()
.prepareSearch("idx")
.setTypes("high_card_type")
.addAggregation(
terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.include("val00.+")
.exclude("(val000|val001)"))
.execute()
.actionGet();
assertSearchResponse(response);
terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(8));
for (int i = 2; i < 10; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val00" + i));
assertThat(bucket.getDocCount(), equalTo(1l));
}
// exclude without include
// we should be left with: val000, val001, val002, val003, val004, val005, val006, val007,
// val008, val009
response =
client()
.prepareSearch("idx")
.setTypes("high_card_type")
.addAggregation(terms("terms").field(SINGLE_VALUED_FIELD_NAME).exclude("val0[1-9]+.+"))
.execute()
.actionGet();
assertSearchResponse(response);
terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(10));
for (int i = 0; i < 10; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val00" + i));
assertThat(bucket.getDocCount(), equalTo(1l));
}
}
@Test
public void singleValueField_WithRegexFiltering_WithFlags() throws Exception {
// include without exclude
// we should be left with: val000, val001, val002, val003, val004, val005, val006, val007,
// val008, val009
// with case insensitive flag on the include regex
SearchResponse response =
client()
.prepareSearch("idx")
.setTypes("high_card_type")
.addAggregation(
terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.include("VAL00.+", Pattern.CASE_INSENSITIVE))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(10));
for (int i = 0; i < 10; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val00" + i));
assertThat(bucket.getDocCount(), equalTo(1l));
}
// include and exclude
// we should be left with: val002, val003, val004, val005, val006, val007, val008, val009
// with multi-flag masking on the exclude regex
response =
client()
.prepareSearch("idx")
.setTypes("high_card_type")
.addAggregation(
terms("terms")
.field(SINGLE_VALUED_FIELD_NAME)
.include("val00.+")
.exclude(
"( val000 | VAL001 )#this is a comment",
Pattern.CASE_INSENSITIVE | Pattern.COMMENTS))
.execute()
.actionGet();
assertSearchResponse(response);
terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(8));
for (int i = 2; i < 10; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val00" + i));
assertThat(bucket.getDocCount(), equalTo(1l));
}
// exclude without include
// we should be left with: val000, val001, val002, val003, val004, val005, val006, val007,
// val008, val009
// with a "no flag" flag
response =
client()
.prepareSearch("idx")
.setTypes("high_card_type")
.addAggregation(
terms("terms").field(SINGLE_VALUED_FIELD_NAME).exclude("val0[1-9]+.+", 0))
.execute()
.actionGet();
assertSearchResponse(response);
terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(10));
for (int i = 0; i < 10; i++) {
Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val00" + i));
assertThat(bucket.getDocCount(), equalTo(1l));
}
}
