@Test
public void docCountDerivativeWithGaps() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("empty_bucket_idx")
.setQuery(matchAllQuery())
.addAggregation(
histogram("histo")
.field(SINGLE_VALUED_FIELD_NAME)
.interval(1)
.subAggregation(derivative("deriv").setBucketsPaths("_count")))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(numDocsEmptyIdx));
InternalHistogram<Bucket> deriv = searchResponse.getAggregations().get("histo");
assertThat(deriv, Matchers.notNullValue());
assertThat(deriv.getName(), equalTo("histo"));
List<Bucket> buckets = deriv.getBuckets();
assertThat(buckets.size(), equalTo(valueCounts_empty.length));
for (int i = 0; i < valueCounts_empty.length; i++) {
Histogram.Bucket bucket = buckets.get(i);
checkBucketKeyAndDocCount("Bucket " + i, bucket, i, valueCounts_empty[i]);
SimpleValue docCountDeriv = bucket.getAggregations().get("deriv");
if (firstDerivValueCounts_empty[i] == null) {
assertThat(docCountDeriv, nullValue());
} else {
assertThat(docCountDeriv.value(), equalTo(firstDerivValueCounts_empty[i]));
}
}
}
@Test
public void partiallyUnmapped() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx", "idx_unmapped")
.addAggregation(
histogram("histo")
.field(SINGLE_VALUED_FIELD_NAME)
.interval(interval)
.subAggregation(derivative("deriv").setBucketsPaths("_count")))
.execute()
.actionGet();
assertSearchResponse(response);
InternalHistogram<Bucket> deriv = response.getAggregations().get("histo");
assertThat(deriv, notNullValue());
assertThat(deriv.getName(), equalTo("histo"));
List<? extends Bucket> buckets = deriv.getBuckets();
assertThat(deriv.getBuckets().size(), equalTo(numValueBuckets));
for (int i = 0; i < numValueBuckets; ++i) {
Histogram.Bucket bucket = buckets.get(i);
checkBucketKeyAndDocCount("Bucket " + i, bucket, i * interval, valueCounts[i]);
SimpleValue docCountDeriv = bucket.getAggregations().get("deriv");
if (i > 0) {
assertThat(docCountDeriv, notNullValue());
assertThat(docCountDeriv.value(), equalTo((double) firstDerivValueCounts[i - 1]));
} else {
assertThat(docCountDeriv, nullValue());
}
}
}
@Override
@Test
public void testEmptyAggregation() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("empty_bucket_idx")
.setQuery(matchAllQuery())
.addAggregation(
histogram("histo")
.field("value")
.interval(1l)
.minDocCount(0)
.subAggregation(max("max")))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(2l));
Histogram histo = searchResponse.getAggregations().get("histo");
assertThat(histo, notNullValue());
Histogram.Bucket bucket = histo.getBuckets().get(1);
assertThat(bucket, notNullValue());
Max max = bucket.getAggregations().get("max");
assertThat(max, notNullValue());
assertThat(max.getName(), equalTo("max"));
assertThat(max.getValue(), equalTo(Double.NEGATIVE_INFINITY));
}
@Test
public void testEmptyAggregation() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("empty_bucket_idx")
.setQuery(matchAllQuery())
.addAggregation(
histogram("histo")
.field("value")
.interval(1l)
.emptyBuckets(true)
.subAggregation(min("min")))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(2l));
Histogram histo = searchResponse.getAggregations().get("histo");
assertThat(histo, notNullValue());
Histogram.Bucket bucket = histo.getByKey(1l);
assertThat(bucket, notNullValue());
Min min = bucket.getAggregations().get("min");
assertThat(min, notNullValue());
assertThat(min.getName(), equalTo("min"));
assertThat(min.getValue(), equalTo(Double.POSITIVE_INFINITY));
}
void assertBasicAggregationWorks(String indexName) {
// histogram on a long
SearchResponse searchRsp =
client()
.prepareSearch(indexName)
.addAggregation(AggregationBuilders.histogram("histo").field("long_sort").interval(10))
.get();
ElasticsearchAssertions.assertSearchResponse(searchRsp);
Histogram histo = searchRsp.getAggregations().get("histo");
assertNotNull(histo);
long totalCount = 0;
for (Histogram.Bucket bucket : histo.getBuckets()) {
totalCount += bucket.getDocCount();
}
assertEquals(totalCount, searchRsp.getHits().getTotalHits());
// terms on a boolean
searchRsp =
client()
.prepareSearch(indexName)
.addAggregation(AggregationBuilders.terms("bool_terms").field("bool"))
.get();
Terms terms = searchRsp.getAggregations().get("bool_terms");
totalCount = 0;
for (Terms.Bucket bucket : terms.getBuckets()) {
totalCount += bucket.getDocCount();
}
assertEquals(totalCount, searchRsp.getHits().getTotalHits());
}
@Override
public void testEmptyAggregation() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("empty_bucket_idx")
.setQuery(matchAllQuery())
.addAggregation(
histogram("histo")
.field("value")
.interval(1l)
.minDocCount(0)
.subAggregation(avg("avg")))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(2l));
Histogram histo = searchResponse.getAggregations().get("histo");
assertThat(histo, notNullValue());
Histogram.Bucket bucket = histo.getBuckets().get(1);
assertThat(bucket, notNullValue());
Avg avg = bucket.getAggregations().get("avg");
assertThat(avg, notNullValue());
assertThat(avg.getName(), equalTo("avg"));
assertThat(Double.isNaN(avg.getValue()), is(true));
}
public void testOrderBySubAggregation() {
boolean asc = randomBoolean();
SearchResponse searchResponse =
client()
.prepareSearch("idx")
.setQuery(matchAllQuery())
.addAggregation(
histogram("histo")
.field("value")
.interval(2L)
.subAggregation(
randomCompression(
percentiles("percentiles").field("value").percentiles(99)))
.order(Order.aggregation("percentiles", "99", asc)))
.execute()
.actionGet();
assertHitCount(searchResponse, 10);
Histogram histo = searchResponse.getAggregations().get("histo");
double previous = asc ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
for (Histogram.Bucket bucket : histo.getBuckets()) {
Percentiles percentiles = bucket.getAggregations().get("percentiles");
double p99 = percentiles.percentile(99);
if (asc) {
assertThat(p99, greaterThanOrEqualTo(previous));
} else {
assertThat(p99, lessThanOrEqualTo(previous));
}
previous = p99;
}
}
@Override
public void testEmptyAggregation() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("empty_bucket_idx")
.setQuery(matchAllQuery())
.addAggregation(
histogram("histo")
.field("value")
.interval(1L)
.minDocCount(0)
.subAggregation(stats("stats").field("value")))
.execute()
.actionGet();
assertShardExecutionState(searchResponse, 0);
assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L));
Histogram histo = searchResponse.getAggregations().get("histo");
assertThat(histo, notNullValue());
Histogram.Bucket bucket = histo.getBuckets().get(1);
assertThat(bucket, notNullValue());
Stats stats = bucket.getAggregations().get("stats");
assertThat(stats, notNullValue());
assertThat(stats.getName(), equalTo("stats"));
assertThat(stats.getCount(), equalTo(0L));
assertThat(stats.getSum(), equalTo(0.0));
assertThat(stats.getMin(), equalTo(Double.POSITIVE_INFINITY));
assertThat(stats.getMax(), equalTo(Double.NEGATIVE_INFINITY));
assertThat(Double.isNaN(stats.getAvg()), is(true));
}
public void testEmptyAggregation() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("empty_bucket_idx")
.setQuery(matchAllQuery())
.addAggregation(
histogram("histo")
.field("value")
.interval(1L)
.minDocCount(0)
.subAggregation(nested("nested", "nested")))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L));
Histogram histo = searchResponse.getAggregations().get("histo");
assertThat(histo, Matchers.notNullValue());
Histogram.Bucket bucket = histo.getBuckets().get(1);
assertThat(bucket, Matchers.notNullValue());
Nested nested = bucket.getAggregations().get("nested");
assertThat(nested, Matchers.notNullValue());
assertThat(nested.getName(), equalTo("nested"));
assertThat(nested.getDocCount(), is(0L));
}
@Test
public void emptyAggregation() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("empty_bucket_idx")
.setQuery(matchAllQuery())
.addAggregation(
histogram("histo")
.field("value")
.interval(1l)
.minDocCount(0)
.subAggregation(filters("filters").filter("all", matchAllQuery())))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(2l));
Histogram histo = searchResponse.getAggregations().get("histo");
assertThat(histo, Matchers.notNullValue());
Histogram.Bucket bucket = histo.getBuckets().get(1);
assertThat(bucket, Matchers.notNullValue());
Filters filters = bucket.getAggregations().get("filters");
assertThat(filters, notNullValue());
Filters.Bucket all = filters.getBucketByKey("all");
assertThat(all, Matchers.notNullValue());
assertThat(all.getKeyAsString(), equalTo("all"));
assertThat(all.getDocCount(), is(0l));
}
@Override
public void testEmptyAggregation() throws Exception {
SearchResponse searchResponse =
client()
.prepareSearch("empty_bucket_idx")
.setQuery(matchAllQuery())
.addAggregation(
histogram("histo")
.field("value")
.interval(1L)
.minDocCount(0)
.subAggregation(
randomCompression(percentiles("percentiles").field("value"))
.percentiles(10, 15)))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L));
Histogram histo = searchResponse.getAggregations().get("histo");
assertThat(histo, notNullValue());
Histogram.Bucket bucket = histo.getBuckets().get(1);
assertThat(bucket, notNullValue());
Percentiles percentiles = bucket.getAggregations().get("percentiles");
assertThat(percentiles, notNullValue());
assertThat(percentiles.getName(), equalTo("percentiles"));
assertThat(percentiles.percentile(10), equalTo(Double.NaN));
assertThat(percentiles.percentile(15), equalTo(Double.NaN));
}
// Duel between histograms and scripted terms
public void testDuelTermsHistogram() throws Exception {
createIndex("idx");
final int numDocs = scaledRandomIntBetween(500, 5000);
final int maxNumTerms = randomIntBetween(10, 2000);
final int interval = randomIntBetween(1, 100);
final Integer[] values = new Integer[maxNumTerms];
for (int i = 0; i < values.length; ++i) {
values[i] = randomInt(maxNumTerms * 3) - maxNumTerms;
}
for (int i = 0; i < numDocs; ++i) {
XContentBuilder source =
jsonBuilder().startObject().field("num", randomDouble()).startArray("values");
final int numValues = randomInt(4);
for (int j = 0; j < numValues; ++j) {
source = source.value(randomFrom(values));
}
source = source.endArray().endObject();
client().prepareIndex("idx", "type").setSource(source).execute().actionGet();
}
assertNoFailures(
client()
.admin()
.indices()
.prepareRefresh("idx")
.setIndicesOptions(IndicesOptions.lenientExpandOpen())
.execute()
.get());
SearchResponse resp =
client()
.prepareSearch("idx")
.addAggregation(
terms("terms")
.field("values")
.collectMode(randomFrom(SubAggCollectionMode.values()))
.script("floor(_value / interval)")
.param("interval", interval)
.size(maxNumTerms))
.addAggregation(histogram("histo").field("values").interval(interval))
.execute()
.actionGet();
assertSearchResponse(resp);
Terms terms = resp.getAggregations().get("terms");
assertThat(terms, notNullValue());
Histogram histo = resp.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(terms.getBuckets().size(), equalTo(histo.getBuckets().size()));
for (Terms.Bucket bucket : terms.getBuckets()) {
final long key = bucket.getKeyAsNumber().longValue() * interval;
final Histogram.Bucket histoBucket = histo.getBucketByKey(key);
assertEquals(bucket.getDocCount(), histoBucket.getDocCount());
}
}
private void checkBucketKeyAndDocCount(
final String msg,
final Histogram.Bucket bucket,
final long expectedKey,
final long expectedDocCount) {
assertThat(msg, bucket, notNullValue());
assertThat(msg + " key", ((Number) bucket.getKey()).longValue(), equalTo(expectedKey));
assertThat(msg + " docCount", bucket.getDocCount(), equalTo(expectedDocCount));
}
@Test
public void multiValueAggDerivative() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
histogram("histo")
.field(SINGLE_VALUED_FIELD_NAME)
.interval(interval)
.subAggregation(stats("stats").field(SINGLE_VALUED_FIELD_NAME))
.subAggregation(derivative("deriv").setBucketsPaths("stats.sum")))
.execute()
.actionGet();
assertSearchResponse(response);
InternalHistogram<Bucket> deriv = response.getAggregations().get("histo");
assertThat(deriv, notNullValue());
assertThat(deriv.getName(), equalTo("histo"));
assertThat(deriv.getBuckets().size(), equalTo(numValueBuckets));
Object[] propertiesKeys = (Object[]) deriv.getProperty("_key");
Object[] propertiesDocCounts = (Object[]) deriv.getProperty("_count");
Object[] propertiesSumCounts = (Object[]) deriv.getProperty("stats.sum");
List<Bucket> buckets = new ArrayList<Bucket>(deriv.getBuckets());
Long expectedSumPreviousBucket = Long.MIN_VALUE; // start value, gets
// overwritten
for (int i = 0; i < numValueBuckets; ++i) {
Histogram.Bucket bucket = buckets.get(i);
checkBucketKeyAndDocCount("Bucket " + i, bucket, i * interval, valueCounts[i]);
Stats stats = bucket.getAggregations().get("stats");
assertThat(stats, notNullValue());
long expectedSum = valueCounts[i] * (i * interval);
assertThat(stats.getSum(), equalTo((double) expectedSum));
SimpleValue sumDeriv = bucket.getAggregations().get("deriv");
if (i > 0) {
assertThat(sumDeriv, notNullValue());
long sumDerivValue = expectedSum - expectedSumPreviousBucket;
assertThat(sumDeriv.value(), equalTo((double) sumDerivValue));
assertThat(
(double)
bucket.getProperty(
"histo", AggregationPath.parse("deriv.value").getPathElementsAsStringList()),
equalTo((double) sumDerivValue));
} else {
assertThat(sumDeriv, nullValue());
}
expectedSumPreviousBucket = expectedSum;
assertThat((long) propertiesKeys[i], equalTo((long) i * interval));
assertThat((long) propertiesDocCounts[i], equalTo(valueCounts[i]));
assertThat((double) propertiesSumCounts[i], equalTo((double) expectedSum));
}
}
/** test first and second derivative on the sing */
@Test
public void singleValuedField_normalised() {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
histogram("histo")
.field(SINGLE_VALUED_FIELD_NAME)
.interval(interval)
.minDocCount(0)
.subAggregation(derivative("deriv").setBucketsPaths("_count").unit("1"))
.subAggregation(derivative("2nd_deriv").setBucketsPaths("deriv").unit("10")))
.execute()
.actionGet();
assertSearchResponse(response);
InternalHistogram<Bucket> deriv = response.getAggregations().get("histo");
assertThat(deriv, notNullValue());
assertThat(deriv.getName(), equalTo("histo"));
List<? extends Bucket> buckets = deriv.getBuckets();
assertThat(buckets.size(), equalTo(numValueBuckets));
for (int i = 0; i < numValueBuckets; ++i) {
Histogram.Bucket bucket = buckets.get(i);
checkBucketKeyAndDocCount("Bucket " + i, bucket, i * interval, valueCounts[i]);
Derivative docCountDeriv = bucket.getAggregations().get("deriv");
if (i > 0) {
assertThat(docCountDeriv, notNullValue());
assertThat(
docCountDeriv.value(), closeTo((double) (firstDerivValueCounts[i - 1]), 0.00001));
assertThat(
docCountDeriv.normalizedValue(),
closeTo((double) (firstDerivValueCounts[i - 1]) / 5, 0.00001));
} else {
assertThat(docCountDeriv, nullValue());
}
Derivative docCount2ndDeriv = bucket.getAggregations().get("2nd_deriv");
if (i > 1) {
assertThat(docCount2ndDeriv, notNullValue());
assertThat(
docCount2ndDeriv.value(), closeTo((double) (secondDerivValueCounts[i - 2]), 0.00001));
assertThat(
docCount2ndDeriv.normalizedValue(),
closeTo((double) (secondDerivValueCounts[i - 2]) * 2, 0.00001));
} else {
assertThat(docCount2ndDeriv, nullValue());
}
}
}
public void testSingleValuedFieldNormalised() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
dateHistogram("histo")
.field("date")
.dateHistogramInterval(DateHistogramInterval.MONTH)
.minDocCount(0)
.subAggregation(derivative("deriv", "_count").unit(DateHistogramInterval.DAY)))
.execute()
.actionGet();
assertSearchResponse(response);
Histogram deriv = response.getAggregations().get("histo");
assertThat(deriv, notNullValue());
assertThat(deriv.getName(), equalTo("histo"));
List<? extends Bucket> buckets = deriv.getBuckets();
assertThat(buckets.size(), equalTo(3));
DateTime key = new DateTime(2012, 1, 1, 0, 0, DateTimeZone.UTC);
Histogram.Bucket bucket = buckets.get(0);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(1L));
Derivative docCountDeriv = bucket.getAggregations().get("deriv");
assertThat(docCountDeriv, nullValue());
key = new DateTime(2012, 2, 1, 0, 0, DateTimeZone.UTC);
bucket = buckets.get(1);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(2L));
docCountDeriv = bucket.getAggregations().get("deriv");
assertThat(docCountDeriv, notNullValue());
assertThat(docCountDeriv.value(), closeTo(1d, 0.00001));
assertThat(docCountDeriv.normalizedValue(), closeTo(1d / 31d, 0.00001));
key = new DateTime(2012, 3, 1, 0, 0, DateTimeZone.UTC);
bucket = buckets.get(2);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(3L));
docCountDeriv = bucket.getAggregations().get("deriv");
assertThat(docCountDeriv, notNullValue());
assertThat(docCountDeriv.value(), closeTo(1d, 0.00001));
assertThat(docCountDeriv.normalizedValue(), closeTo(1d / 29d, 0.00001));
}
@Test
public void singleValueAggDerivativeWithGaps_random() throws Exception {
GapPolicy gapPolicy = randomFrom(GapPolicy.values());
SearchResponse searchResponse =
client()
.prepareSearch("empty_bucket_idx_rnd")
.setQuery(matchAllQuery())
.addAggregation(
histogram("histo")
.field(SINGLE_VALUED_FIELD_NAME)
.interval(1)
.extendedBounds(0l, (long) numBuckets_empty_rnd - 1)
.subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME))
.subAggregation(
derivative("deriv").setBucketsPaths("sum").gapPolicy(gapPolicy)))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(numDocsEmptyIdx_rnd));
InternalHistogram<Bucket> deriv = searchResponse.getAggregations().get("histo");
assertThat(deriv, Matchers.notNullValue());
assertThat(deriv.getName(), equalTo("histo"));
List<Bucket> buckets = deriv.getBuckets();
assertThat(buckets.size(), equalTo(numBuckets_empty_rnd));
double lastSumValue = Double.NaN;
for (int i = 0; i < valueCounts_empty_rnd.length; i++) {
Histogram.Bucket bucket = buckets.get(i);
checkBucketKeyAndDocCount("Bucket " + i, bucket, i, valueCounts_empty_rnd[i]);
Sum sum = bucket.getAggregations().get("sum");
double thisSumValue = sum.value();
if (bucket.getDocCount() == 0) {
thisSumValue = gapPolicy == GapPolicy.INSERT_ZEROS ? 0 : Double.NaN;
}
SimpleValue sumDeriv = bucket.getAggregations().get("deriv");
if (i == 0) {
assertThat(sumDeriv, nullValue());
} else {
double expectedDerivative = thisSumValue - lastSumValue;
if (Double.isNaN(expectedDerivative)) {
assertThat(sumDeriv.value(), equalTo(expectedDerivative));
} else {
assertThat(sumDeriv.value(), closeTo(expectedDerivative, 0.00001));
}
}
lastSumValue = thisSumValue;
}
}
private static void assertBucket(
Histogram.Bucket bucket,
DateTime expectedKey,
long expectedDocCount,
Matcher<Object> derivativeMatcher,
Double derivative,
Double normalizedDerivative) {
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(expectedKey));
assertThat(bucket.getDocCount(), equalTo(expectedDocCount));
Derivative docCountDeriv = bucket.getAggregations().get("deriv");
assertThat(docCountDeriv, derivativeMatcher);
if (docCountDeriv != null) {
assertThat(docCountDeriv.value(), closeTo(derivative, 0.00001));
assertThat(docCountDeriv.normalizedValue(), closeTo(normalizedDerivative, 0.00001));
}
}
public void testInlineScriptNamedVars() {
Map<String, String> bucketPathsMap = new HashMap<>();
bucketPathsMap.put("my_value1", "field2Sum");
bucketPathsMap.put("my_value2", "field3Sum");
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
histogram("histo")
.field(FIELD_1_NAME)
.interval(interval)
.subAggregation(sum("field2Sum").field(FIELD_2_NAME))
.subAggregation(sum("field3Sum").field(FIELD_3_NAME))
.subAggregation(
bucketSelector(
"bucketSelector",
bucketPathsMap,
new Script(
"Double.isNaN(my_value1) ? false : (my_value1 + my_value2 > 100)",
ScriptType.INLINE,
null,
null))))
.execute()
.actionGet();
assertSearchResponse(response);
InternalHistogram<Bucket> histo = response.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(histo.getName(), equalTo("histo"));
List<? extends Bucket> buckets = histo.getBuckets();
for (int i = 0; i < buckets.size(); ++i) {
Histogram.Bucket bucket = buckets.get(i);
Sum field2Sum = bucket.getAggregations().get("field2Sum");
assertThat(field2Sum, notNullValue());
double field2SumValue = field2Sum.getValue();
Sum field3Sum = bucket.getAggregations().get("field3Sum");
assertThat(field3Sum, notNullValue());
double field3SumValue = field3Sum.getValue();
assertThat(field2SumValue + field3SumValue, greaterThan(100.0));
}
}
@Test
public void testDocCount_asSubAgg() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
terms("terms")
.field("tag")
.order(Order.term(true))
.subAggregation(
histogram("histo")
.field(SINGLE_VALUED_FIELD_NAME)
.interval(interval)
.extendedBounds((long) minRandomValue, (long) maxRandomValue))
.subAggregation(maxBucket("max_bucket").setBucketsPaths("histo>_count")))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
List<Terms.Bucket> termsBuckets = terms.getBuckets();
assertThat(termsBuckets.size(), equalTo(interval));
for (int i = 0; i < interval; ++i) {
Terms.Bucket termsBucket = termsBuckets.get(i);
assertThat(termsBucket, notNullValue());
assertThat((String) termsBucket.getKey(), equalTo("tag" + (i % interval)));
Histogram histo = termsBucket.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(histo.getName(), equalTo("histo"));
List<? extends Bucket> buckets = histo.getBuckets();
List<String> maxKeys = new ArrayList<>();
double maxValue = Double.NEGATIVE_INFINITY;
for (int j = 0; j < numValueBuckets; ++j) {
Histogram.Bucket bucket = buckets.get(j);
assertThat(bucket, notNullValue());
assertThat(((Number) bucket.getKey()).longValue(), equalTo((long) j * interval));
if (bucket.getDocCount() > maxValue) {
maxValue = bucket.getDocCount();
maxKeys = new ArrayList<>();
maxKeys.add(bucket.getKeyAsString());
} else if (bucket.getDocCount() == maxValue) {
maxKeys.add(bucket.getKeyAsString());
}
}
InternalBucketMetricValue maxBucketValue = termsBucket.getAggregations().get("max_bucket");
assertThat(maxBucketValue, notNullValue());
assertThat(maxBucketValue.getName(), equalTo("max_bucket"));
assertThat(maxBucketValue.value(), equalTo(maxValue));
assertThat(maxBucketValue.keys(), equalTo(maxKeys.toArray(new String[maxKeys.size()])));
}
}
public void testPartiallyUnmapped() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx", "idx_unmapped")
.addAggregation(
histogram("histo")
.field(FIELD_1_NAME)
.interval(interval)
.subAggregation(sum("field2Sum").field(FIELD_2_NAME))
.subAggregation(sum("field3Sum").field(FIELD_3_NAME))
.subAggregation(
bucketSelector(
"bucketSelector",
new Script(
"Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)",
ScriptType.INLINE,
null,
null),
"field2Sum",
"field3Sum")))
.execute()
.actionGet();
assertSearchResponse(response);
InternalHistogram<Bucket> histo = response.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(histo.getName(), equalTo("histo"));
List<? extends Bucket> buckets = histo.getBuckets();
for (int i = 0; i < buckets.size(); ++i) {
Histogram.Bucket bucket = buckets.get(i);
Sum field2Sum = bucket.getAggregations().get("field2Sum");
assertThat(field2Sum, notNullValue());
double field2SumValue = field2Sum.getValue();
Sum field3Sum = bucket.getAggregations().get("field3Sum");
assertThat(field3Sum, notNullValue());
double field3SumValue = field3Sum.getValue();
assertThat(field2SumValue + field3SumValue, greaterThan(100.0));
}
}
@Test
public void emptyAggregation() throws Exception {
prepareCreate("empty_bucket_idx")
.addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=integer")
.execute()
.actionGet();
List<IndexRequestBuilder> builders = new ArrayList<IndexRequestBuilder>();
for (int i = 0; i < 2; i++) {
builders.add(
client()
.prepareIndex("empty_bucket_idx", "type", "" + i)
.setSource(
jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, i * 2).endObject()));
}
indexRandom(true, builders.toArray(new IndexRequestBuilder[builders.size()]));
SearchResponse searchResponse =
client()
.prepareSearch("empty_bucket_idx")
.setQuery(matchAllQuery())
.addAggregation(
histogram("histo")
.field(SINGLE_VALUED_FIELD_NAME)
.interval(1l)
.minDocCount(0)
.subAggregation(terms("terms")))
.execute()
.actionGet();
assertThat(searchResponse.getHits().getTotalHits(), equalTo(2l));
Histogram histo = searchResponse.getAggregations().get("histo");
assertThat(histo, Matchers.notNullValue());
Histogram.Bucket bucket = histo.getBucketByKey(1l);
assertThat(bucket, Matchers.notNullValue());
Terms terms = bucket.getAggregations().get("terms");
assertThat(terms, Matchers.notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().isEmpty(), is(true));
}
public void testInlineScriptInsertZeros() {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
histogram("histo")
.field(FIELD_1_NAME)
.interval(interval)
.subAggregation(sum("field2Sum").field(FIELD_2_NAME))
.subAggregation(sum("field3Sum").field(FIELD_3_NAME))
.subAggregation(
bucketSelector(
"bucketSelector",
new Script(
"_value0 + _value1 > 100", ScriptType.INLINE, null, null),
"field2Sum",
"field3Sum")
.gapPolicy(GapPolicy.INSERT_ZEROS)))
.execute()
.actionGet();
assertSearchResponse(response);
InternalHistogram<Bucket> histo = response.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(histo.getName(), equalTo("histo"));
List<? extends Bucket> buckets = histo.getBuckets();
for (int i = 0; i < buckets.size(); ++i) {
Histogram.Bucket bucket = buckets.get(i);
Sum field2Sum = bucket.getAggregations().get("field2Sum");
assertThat(field2Sum, notNullValue());
double field2SumValue = field2Sum.getValue();
Sum field3Sum = bucket.getAggregations().get("field3Sum");
assertThat(field3Sum, notNullValue());
double field3SumValue = field3Sum.getValue();
assertThat(field2SumValue + field3SumValue, greaterThan(100.0));
}
}
@Test
public void testMetric_asSubAggOfSingleBucketAgg() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
filter("filter")
.filter(termQuery("tag", "tag0"))
.subAggregation(
histogram("histo")
.field(SINGLE_VALUED_FIELD_NAME)
.interval(interval)
.extendedBounds((long) minRandomValue, (long) maxRandomValue)
.subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME)))
.subAggregation(maxBucket("max_bucket").setBucketsPaths("histo>sum")))
.execute()
.actionGet();
assertSearchResponse(response);
Filter filter = response.getAggregations().get("filter");
assertThat(filter, notNullValue());
assertThat(filter.getName(), equalTo("filter"));
Histogram histo = filter.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(histo.getName(), equalTo("histo"));
List<? extends Bucket> buckets = histo.getBuckets();
List<String> maxKeys = new ArrayList<>();
double maxValue = Double.NEGATIVE_INFINITY;
for (int j = 0; j < numValueBuckets; ++j) {
Histogram.Bucket bucket = buckets.get(j);
assertThat(bucket, notNullValue());
assertThat(((Number) bucket.getKey()).longValue(), equalTo((long) j * interval));
if (bucket.getDocCount() != 0) {
Sum sum = bucket.getAggregations().get("sum");
assertThat(sum, notNullValue());
if (sum.value() > maxValue) {
maxValue = sum.value();
maxKeys = new ArrayList<>();
maxKeys.add(bucket.getKeyAsString());
} else if (sum.value() == maxValue) {
maxKeys.add(bucket.getKeyAsString());
}
}
}
InternalBucketMetricValue maxBucketValue = filter.getAggregations().get("max_bucket");
assertThat(maxBucketValue, notNullValue());
assertThat(maxBucketValue.getName(), equalTo("max_bucket"));
assertThat(maxBucketValue.value(), equalTo(maxValue));
assertThat(maxBucketValue.keys(), equalTo(maxKeys.toArray(new String[maxKeys.size()])));
}
@Test
public void histogramOnNestedField() throws Exception {
Aggregations result =
query(
String.format(
"select count(*) from %s/nestedType group by histogram('field'='message.dayOfWeek','nested'='message','interval'='2' , 'alias' = 'someAlias' )",
TEST_INDEX));
InternalNested nested = result.get("message@NESTED");
Histogram histogram = nested.getAggregations().get("someAlias");
for (Histogram.Bucket bucket : histogram.getBuckets()) {
long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue();
if (bucket.getKey().equals("0") || bucket.getKey().equals("4")) {
Assert.assertEquals(2, count);
} else if (bucket.getKey().equals("2")) {
Assert.assertEquals(1, count);
} else {
Assert.assertTrue("only 0 2 4 keys are allowed got:" + bucket.getKey(), false);
}
}
}
public void testPartiallyUnmapped() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx", "idx_unmapped")
.addAggregation(
dateHistogram("histo")
.field("date")
.dateHistogramInterval(DateHistogramInterval.MONTH)
.minDocCount(0)
.subAggregation(derivative("deriv", "_count")))
.execute()
.actionGet();
assertSearchResponse(response);
Histogram deriv = response.getAggregations().get("histo");
assertThat(deriv, notNullValue());
assertThat(deriv.getName(), equalTo("histo"));
List<? extends Bucket> buckets = deriv.getBuckets();
assertThat(buckets.size(), equalTo(3));
DateTime key = new DateTime(2012, 1, 1, 0, 0, DateTimeZone.UTC);
Histogram.Bucket bucket = buckets.get(0);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(1L));
assertThat(bucket.getAggregations().asList().isEmpty(), is(true));
SimpleValue docCountDeriv = bucket.getAggregations().get("deriv");
assertThat(docCountDeriv, nullValue());
key = new DateTime(2012, 2, 1, 0, 0, DateTimeZone.UTC);
bucket = buckets.get(1);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(2L));
assertThat(bucket.getAggregations().asList().isEmpty(), is(false));
docCountDeriv = bucket.getAggregations().get("deriv");
assertThat(docCountDeriv, notNullValue());
assertThat(docCountDeriv.value(), equalTo(1.0));
key = new DateTime(2012, 3, 1, 0, 0, DateTimeZone.UTC);
bucket = buckets.get(2);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(3L));
assertThat(bucket.getAggregations().asList().isEmpty(), is(false));
docCountDeriv = bucket.getAggregations().get("deriv");
assertThat(docCountDeriv, notNullValue());
assertThat(docCountDeriv.value(), equalTo(1.0));
}
public void testSingleValuedFieldWithSubAggregation() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
dateHistogram("histo")
.field("date")
.dateHistogramInterval(DateHistogramInterval.MONTH)
.minDocCount(0)
.subAggregation(sum("sum").field("value"))
.subAggregation(derivative("deriv", "sum")))
.execute()
.actionGet();
assertSearchResponse(response);
Histogram histo = response.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(histo.getName(), equalTo("histo"));
List<? extends Bucket> buckets = histo.getBuckets();
assertThat(buckets.size(), equalTo(3));
Object[] propertiesKeys = (Object[]) histo.getProperty("_key");
Object[] propertiesDocCounts = (Object[]) histo.getProperty("_count");
Object[] propertiesCounts = (Object[]) histo.getProperty("sum.value");
DateTime key = new DateTime(2012, 1, 1, 0, 0, DateTimeZone.UTC);
Histogram.Bucket bucket = buckets.get(0);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(1L));
assertThat(bucket.getAggregations().asList().isEmpty(), is(false));
Sum sum = bucket.getAggregations().get("sum");
assertThat(sum, notNullValue());
assertThat(sum.getValue(), equalTo(1.0));
SimpleValue deriv = bucket.getAggregations().get("deriv");
assertThat(deriv, nullValue());
assertThat((DateTime) propertiesKeys[0], equalTo(key));
assertThat((long) propertiesDocCounts[0], equalTo(1L));
assertThat((double) propertiesCounts[0], equalTo(1.0));
key = new DateTime(2012, 2, 1, 0, 0, DateTimeZone.UTC);
bucket = buckets.get(1);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(2L));
assertThat(bucket.getAggregations().asList().isEmpty(), is(false));
sum = bucket.getAggregations().get("sum");
assertThat(sum, notNullValue());
assertThat(sum.getValue(), equalTo(5.0));
deriv = bucket.getAggregations().get("deriv");
assertThat(deriv, notNullValue());
assertThat(deriv.value(), equalTo(4.0));
assertThat(
(double)
bucket.getProperty(
"histo", AggregationPath.parse("deriv.value").getPathElementsAsStringList()),
equalTo(4.0));
assertThat((DateTime) propertiesKeys[1], equalTo(key));
assertThat((long) propertiesDocCounts[1], equalTo(2L));
assertThat((double) propertiesCounts[1], equalTo(5.0));
key = new DateTime(2012, 3, 1, 0, 0, DateTimeZone.UTC);
bucket = buckets.get(2);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(3L));
assertThat(bucket.getAggregations().asList().isEmpty(), is(false));
sum = bucket.getAggregations().get("sum");
assertThat(sum, notNullValue());
assertThat(sum.getValue(), equalTo(15.0));
deriv = bucket.getAggregations().get("deriv");
assertThat(deriv, notNullValue());
assertThat(deriv.value(), equalTo(10.0));
assertThat(
(double)
bucket.getProperty(
"histo", AggregationPath.parse("deriv.value").getPathElementsAsStringList()),
equalTo(10.0));
assertThat((DateTime) propertiesKeys[2], equalTo(key));
assertThat((long) propertiesDocCounts[2], equalTo(3L));
assertThat((double) propertiesCounts[2], equalTo(15.0));
}
public void testEmptyBuckets() {
SearchResponse response =
client()
.prepareSearch("idx_with_gaps")
.addAggregation(
histogram("histo")
.field(FIELD_1_NAME)
.interval(1)
.subAggregation(
histogram("inner_histo")
.field(FIELD_1_NAME)
.interval(1)
.extendedBounds(new ExtendedBounds(1L, 4L))
.minDocCount(0)
.subAggregation(
derivative("derivative", "_count")
.gapPolicy(GapPolicy.INSERT_ZEROS))))
.execute()
.actionGet();
assertSearchResponse(response);
InternalHistogram<Bucket> histo = response.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(histo.getName(), equalTo("histo"));
List<? extends Bucket> buckets = histo.getBuckets();
assertThat(buckets.size(), equalTo(3));
Histogram.Bucket bucket = buckets.get(0);
assertThat(bucket, notNullValue());
assertThat(bucket.getKeyAsString(), equalTo("1"));
Histogram innerHisto = bucket.getAggregations().get("inner_histo");
assertThat(innerHisto, notNullValue());
List<? extends Histogram.Bucket> innerBuckets = innerHisto.getBuckets();
assertThat(innerBuckets, notNullValue());
assertThat(innerBuckets.size(), equalTo(4));
for (int i = 0; i < innerBuckets.size(); i++) {
Histogram.Bucket innerBucket = innerBuckets.get(i);
if (i == 0) {
assertThat(innerBucket.getAggregations().get("derivative"), nullValue());
} else {
assertThat(innerBucket.getAggregations().get("derivative"), notNullValue());
}
}
bucket = buckets.get(1);
assertThat(bucket, notNullValue());
assertThat(bucket.getKeyAsString(), equalTo("2"));
innerHisto = bucket.getAggregations().get("inner_histo");
assertThat(innerHisto, notNullValue());
innerBuckets = innerHisto.getBuckets();
assertThat(innerBuckets, notNullValue());
assertThat(innerBuckets.size(), equalTo(4));
for (int i = 0; i < innerBuckets.size(); i++) {
Histogram.Bucket innerBucket = innerBuckets.get(i);
if (i == 0) {
assertThat(innerBucket.getAggregations().get("derivative"), nullValue());
} else {
assertThat(innerBucket.getAggregations().get("derivative"), notNullValue());
}
}
bucket = buckets.get(2);
assertThat(bucket, notNullValue());
assertThat(bucket.getKeyAsString(), equalTo("3"));
innerHisto = bucket.getAggregations().get("inner_histo");
assertThat(innerHisto, notNullValue());
innerBuckets = innerHisto.getBuckets();
assertThat(innerBuckets, notNullValue());
assertThat(innerBuckets.size(), equalTo(4));
for (int i = 0; i < innerBuckets.size(); i++) {
Histogram.Bucket innerBucket = innerBuckets.get(i);
if (i == 0) {
assertThat(innerBucket.getAggregations().get("derivative"), nullValue());
} else {
assertThat(innerBucket.getAggregations().get("derivative"), notNullValue());
}
}
}
/**
* @param bucket the bucket to check asssertions for
* @param key the expected key
* @param expectedSize the expected size of the bucket
*/
private static void checkBucketFor(Histogram.Bucket bucket, DateTime key, long expectedSize) {
assertThat(bucket, notNullValue());
assertThat(bucket.getKeyAsString(), equalTo(key.toString(DATE_FORMAT)));
assertThat(((DateTime) bucket.getKey()), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(expectedSize));
}
// test to make sure expressions are allowed to be used for reduce in pipeline aggregations
public void testPipelineAggregationScript() throws Exception {
createIndex("agg_index");
ensureGreen("agg_index");
indexRandom(
true,
client()
.prepareIndex("agg_index", "doc", "1")
.setSource("one", 1.0, "two", 2.0, "three", 3.0, "four", 4.0),
client()
.prepareIndex("agg_index", "doc", "2")
.setSource("one", 2.0, "two", 2.0, "three", 3.0, "four", 4.0),
client()
.prepareIndex("agg_index", "doc", "3")
.setSource("one", 3.0, "two", 2.0, "three", 3.0, "four", 4.0),
client()
.prepareIndex("agg_index", "doc", "4")
.setSource("one", 4.0, "two", 2.0, "three", 3.0, "four", 4.0),
client()
.prepareIndex("agg_index", "doc", "5")
.setSource("one", 5.0, "two", 2.0, "three", 3.0, "four", 4.0));
SearchResponse response =
client()
.prepareSearch("agg_index")
.addAggregation(
histogram("histogram")
.field("one")
.interval(2)
.subAggregation(sum("twoSum").field("two"))
.subAggregation(sum("threeSum").field("three"))
.subAggregation(sum("fourSum").field("four"))
.subAggregation(
bucketScript("totalSum")
.setBucketsPaths("twoSum", "threeSum", "fourSum")
.script(
new Script(
"_value0 + _value1 + _value2",
ScriptType.INLINE,
ExpressionScriptEngineService.NAME,
null))))
.execute()
.actionGet();
InternalHistogram<Bucket> histogram = response.getAggregations().get("histogram");
assertThat(histogram, notNullValue());
assertThat(histogram.getName(), equalTo("histogram"));
List<Bucket> buckets = histogram.getBuckets();
for (int bucketCount = 0; bucketCount < buckets.size(); ++bucketCount) {
Histogram.Bucket bucket = buckets.get(bucketCount);
if (bucket.getDocCount() == 1) {
SimpleValue seriesArithmetic = bucket.getAggregations().get("totalSum");
assertThat(seriesArithmetic, notNullValue());
double seriesArithmeticValue = seriesArithmetic.value();
assertEquals(9.0, seriesArithmeticValue, 0.001);
} else if (bucket.getDocCount() == 2) {
SimpleValue seriesArithmetic = bucket.getAggregations().get("totalSum");
assertThat(seriesArithmetic, notNullValue());
double seriesArithmeticValue = seriesArithmetic.value();
assertEquals(18.0, seriesArithmeticValue, 0.001);
} else {
fail("Incorrect number of documents in a bucket in the histogram.");
}
}
}