public void testLargeNumbersOfPercentileBuckets() throws Exception {
// test high numbers of percentile buckets to make sure paging and release work correctly
createIndex("idx");
final int numDocs = scaledRandomIntBetween(2500, 5000);
logger.info("Indexing [" + numDocs + "] docs");
List<IndexRequestBuilder> indexingRequests = Lists.newArrayList();
for (int i = 0; i < numDocs; ++i) {
indexingRequests.add(
client()
.prepareIndex("idx", "type", Integer.toString(i))
.setSource("double_value", randomDouble()));
}
indexRandom(true, indexingRequests);
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
terms("terms")
.field("double_value")
.collectMode(randomFrom(SubAggCollectionMode.values()))
.subAggregation(percentiles("pcts").field("double_value")))
.execute()
.actionGet();
assertAllSuccessful(response);
assertEquals(numDocs, response.getHits().getTotalHits());
}
@Before
public void init() throws Exception {
createIndex("idx");
IndexRequestBuilder[] lowCardBuilders = new IndexRequestBuilder[5]; // TODO randomize the size?
for (int i = 0; i < lowCardBuilders.length; i++) {
lowCardBuilders[i] =
client()
.prepareIndex("idx", "type")
.setSource(
jsonBuilder()
.startObject()
.field(SINGLE_VALUED_FIELD_NAME, "val" + i)
.field("i", i)
.field("tag", i < lowCardBuilders.length / 2 + 1 ? "more" : "less")
.startArray(MULTI_VALUED_FIELD_NAME)
.value("val" + i)
.value("val" + (i + 1))
.endArray()
.endObject());
}
indexRandom(true, lowCardBuilders);
IndexRequestBuilder[] highCardBuilders =
new IndexRequestBuilder[100]; // TODO randomize the size?
for (int i = 0; i < highCardBuilders.length; i++) {
highCardBuilders[i] =
client()
.prepareIndex("idx", "high_card_type")
.setSource(
jsonBuilder()
.startObject()
.field(SINGLE_VALUED_FIELD_NAME, "val" + Strings.padStart(i + "", 3, '0'))
.startArray(MULTI_VALUED_FIELD_NAME)
.value("val" + Strings.padStart(i + "", 3, '0'))
.value("val" + Strings.padStart((i + 1) + "", 3, '0'))
.endArray()
.endObject());
}
indexRandom(true, highCardBuilders);
createIndex("idx_unmapped");
ensureSearchable();
}
// https://github.com/elasticsearch/elasticsearch/issues/6435
public void testReduce() throws Exception {
createIndex("idx");
final int value = randomIntBetween(0, 10);
indexRandom(true, client().prepareIndex("idx", "type").setSource("f", value));
ensureYellow("idx"); // only one document let's make sure all shards have an active primary
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
filter("filter")
.filter(FilterBuilders.matchAllFilter())
.subAggregation(
range("range")
.field("f")
.addUnboundedTo(6)
.addUnboundedFrom(6)
.subAggregation(sum("sum").field("f"))))
.execute()
.actionGet();
assertSearchResponse(response);
Filter filter = response.getAggregations().get("filter");
assertNotNull(filter);
assertEquals(1, filter.getDocCount());
Range range = filter.getAggregations().get("range");
assertThat(range, notNullValue());
assertThat(range.getName(), equalTo("range"));
assertThat(range.getBuckets().size(), equalTo(2));
Range.Bucket bucket = range.getBucketByKey("*-6.0");
assertThat(bucket, notNullValue());
assertThat(bucket.getKey(), equalTo("*-6.0"));
assertThat(bucket.getFrom().doubleValue(), equalTo(Double.NEGATIVE_INFINITY));
assertThat(bucket.getTo().doubleValue(), equalTo(6.0));
assertThat(bucket.getDocCount(), equalTo(value < 6 ? 1L : 0L));
Sum sum = bucket.getAggregations().get("sum");
assertEquals(value < 6 ? value : 0, sum.getValue(), 0d);
bucket = range.getBucketByKey("6.0-*");
assertThat(bucket, notNullValue());
assertThat(bucket.getKey(), equalTo("6.0-*"));
assertThat(bucket.getFrom().doubleValue(), equalTo(6.0));
assertThat(bucket.getTo().doubleValue(), equalTo(Double.POSITIVE_INFINITY));
assertThat(bucket.getDocCount(), equalTo(value >= 6 ? 1L : 0L));
sum = bucket.getAggregations().get("sum");
assertEquals(value >= 6 ? value : 0, sum.getValue(), 0d);
}
@Test
public void emptyAggregation() throws Exception {
prepareCreate("empty_bucket_idx")
.addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=integer")
.execute()
.actionGet();
List<IndexRequestBuilder> builders = new ArrayList<IndexRequestBuilder>();
for (int i = 0; i < 2; i++) {
builders.add(
client()
.prepareIndex("empty_bucket_idx", "type", "" + i)
.setSource(
jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, i * 2).endObject()));
}
indexRandom(true, builders.toArray(new IndexRequestBuilder[builders.size()]));
SearchResponse searchResponse =
client()
.prepareSearch("empty_bucket_idx")
.setQuery(matchAllQuery())
.addAggregation(
histogram("histo")
.field(SINGLE_VALUED_FIELD_NAME)
.interval(1l)
.minDocCount(0)
.subAggregation(terms("terms")))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(2l));
Histogram histo = searchResponse.getAggregations().get("histo");
assertThat(histo, Matchers.notNullValue());
Histogram.Bucket bucket = histo.getBucketByKey(1l);
assertThat(bucket, Matchers.notNullValue());
Terms terms = bucket.getAggregations().get("terms");
assertThat(terms, Matchers.notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().isEmpty(), is(true));
}
// test long/double/string terms aggs with high number of buckets that require array growth
public void testDuelTerms() throws Exception {
final int numDocs = scaledRandomIntBetween(1000, 2000);
final int maxNumTerms = randomIntBetween(10, 5000);
final IntOpenHashSet valuesSet = new IntOpenHashSet();
cluster().wipeIndices("idx");
prepareCreate("idx")
.addMapping(
"type",
jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("string_values")
.field("type", "string")
.field("index", "not_analyzed")
.startObject("fields")
.startObject("doc_values")
.field("type", "string")
.field("index", "no")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.endObject()
.endObject()
.startObject("long_values")
.field("type", "long")
.endObject()
.startObject("double_values")
.field("type", "double")
.endObject()
.endObject()
.endObject())
.execute()
.actionGet();
List<IndexRequestBuilder> indexingRequests = Lists.newArrayList();
for (int i = 0; i < numDocs; ++i) {
final int[] values = new int[randomInt(4)];
for (int j = 0; j < values.length; ++j) {
values[j] = randomInt(maxNumTerms - 1) - 1000;
valuesSet.add(values[j]);
}
XContentBuilder source =
jsonBuilder().startObject().field("num", randomDouble()).startArray("long_values");
for (int j = 0; j < values.length; ++j) {
source = source.value(values[j]);
}
source = source.endArray().startArray("double_values");
for (int j = 0; j < values.length; ++j) {
source = source.value((double) values[j]);
}
source = source.endArray().startArray("string_values");
for (int j = 0; j < values.length; ++j) {
source = source.value(Integer.toString(values[j]));
}
source = source.endArray().endObject();
indexingRequests.add(client().prepareIndex("idx", "type").setSource(source));
}
indexRandom(true, indexingRequests);
assertNoFailures(
client()
.admin()
.indices()
.prepareRefresh("idx")
.setIndicesOptions(IndicesOptions.lenientExpandOpen())
.execute()
.get());
TermsAggregatorFactory.ExecutionMode[] globalOrdinalModes =
new TermsAggregatorFactory.ExecutionMode[] {
TermsAggregatorFactory.ExecutionMode.GLOBAL_ORDINALS_HASH,
TermsAggregatorFactory.ExecutionMode.GLOBAL_ORDINALS
};
SearchResponse resp =
client()
.prepareSearch("idx")
.addAggregation(
terms("long")
.field("long_values")
.size(maxNumTerms)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.subAggregation(min("min").field("num")))
.addAggregation(
terms("double")
.field("double_values")
.size(maxNumTerms)
.collectMode(randomFrom(SubAggCollectionMode.values()))
.subAggregation(max("max").field("num")))
.addAggregation(
terms("string_map")
.field("string_values")
.collectMode(randomFrom(SubAggCollectionMode.values()))
.executionHint(TermsAggregatorFactory.ExecutionMode.MAP.toString())
.size(maxNumTerms)
.subAggregation(stats("stats").field("num")))
.addAggregation(
terms("string_global_ordinals")
.field("string_values")
.collectMode(randomFrom(SubAggCollectionMode.values()))
.executionHint(
globalOrdinalModes[randomInt(globalOrdinalModes.length - 1)].toString())
.size(maxNumTerms)
.subAggregation(extendedStats("stats").field("num")))
.addAggregation(
terms("string_global_ordinals_doc_values")
.field("string_values.doc_values")
.collectMode(randomFrom(SubAggCollectionMode.values()))
.executionHint(
globalOrdinalModes[randomInt(globalOrdinalModes.length - 1)].toString())
.size(maxNumTerms)
.subAggregation(extendedStats("stats").field("num")))
.execute()
.actionGet();
assertAllSuccessful(resp);
assertEquals(numDocs, resp.getHits().getTotalHits());
final Terms longTerms = resp.getAggregations().get("long");
final Terms doubleTerms = resp.getAggregations().get("double");
final Terms stringMapTerms = resp.getAggregations().get("string_map");
final Terms stringGlobalOrdinalsTerms = resp.getAggregations().get("string_global_ordinals");
final Terms stringGlobalOrdinalsDVTerms =
resp.getAggregations().get("string_global_ordinals_doc_values");
assertEquals(valuesSet.size(), longTerms.getBuckets().size());
assertEquals(valuesSet.size(), doubleTerms.getBuckets().size());
assertEquals(valuesSet.size(), stringMapTerms.getBuckets().size());
assertEquals(valuesSet.size(), stringGlobalOrdinalsTerms.getBuckets().size());
assertEquals(valuesSet.size(), stringGlobalOrdinalsDVTerms.getBuckets().size());
for (Terms.Bucket bucket : longTerms.getBuckets()) {
final Terms.Bucket doubleBucket =
doubleTerms.getBucketByKey(
Double.toString(Long.parseLong(bucket.getKeyAsText().string())));
final Terms.Bucket stringMapBucket =
stringMapTerms.getBucketByKey(bucket.getKeyAsText().string());
final Terms.Bucket stringGlobalOrdinalsBucket =
stringGlobalOrdinalsTerms.getBucketByKey(bucket.getKeyAsText().string());
final Terms.Bucket stringGlobalOrdinalsDVBucket =
stringGlobalOrdinalsDVTerms.getBucketByKey(bucket.getKeyAsText().string());
assertNotNull(doubleBucket);
assertNotNull(stringMapBucket);
assertNotNull(stringGlobalOrdinalsBucket);
assertNotNull(stringGlobalOrdinalsDVBucket);
assertEquals(bucket.getDocCount(), doubleBucket.getDocCount());
assertEquals(bucket.getDocCount(), stringMapBucket.getDocCount());
assertEquals(bucket.getDocCount(), stringGlobalOrdinalsBucket.getDocCount());
assertEquals(bucket.getDocCount(), stringGlobalOrdinalsDVBucket.getDocCount());
}
}