public void testMultiValuedField() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
dateHistogram("histo")
.field("dates")
.dateHistogramInterval(DateHistogramInterval.MONTH)
.minDocCount(0)
.subAggregation(derivative("deriv", "_count")))
.execute()
.actionGet();
assertSearchResponse(response);
Histogram deriv = response.getAggregations().get("histo");
assertThat(deriv, notNullValue());
assertThat(deriv.getName(), equalTo("histo"));
List<? extends Bucket> buckets = deriv.getBuckets();
assertThat(buckets.size(), equalTo(4));
DateTime key = new DateTime(2012, 1, 1, 0, 0, DateTimeZone.UTC);
Histogram.Bucket bucket = buckets.get(0);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(1L));
assertThat(bucket.getAggregations().asList().isEmpty(), is(true));
SimpleValue docCountDeriv = bucket.getAggregations().get("deriv");
assertThat(docCountDeriv, nullValue());
key = new DateTime(2012, 2, 1, 0, 0, DateTimeZone.UTC);
bucket = buckets.get(1);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(3L));
assertThat(bucket.getAggregations().asList().isEmpty(), is(false));
docCountDeriv = bucket.getAggregations().get("deriv");
assertThat(docCountDeriv, notNullValue());
assertThat(docCountDeriv.value(), equalTo(2.0));
key = new DateTime(2012, 3, 1, 0, 0, DateTimeZone.UTC);
bucket = buckets.get(2);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(5L));
assertThat(bucket.getAggregations().asList().isEmpty(), is(false));
docCountDeriv = bucket.getAggregations().get("deriv");
assertThat(docCountDeriv, notNullValue());
assertThat(docCountDeriv.value(), equalTo(2.0));
key = new DateTime(2012, 4, 1, 0, 0, DateTimeZone.UTC);
bucket = buckets.get(3);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(3L));
assertThat(bucket.getAggregations().asList().isEmpty(), is(false));
docCountDeriv = bucket.getAggregations().get("deriv");
assertThat(docCountDeriv, notNullValue());
assertThat(docCountDeriv.value(), equalTo(-2.0));
}
@Test
public void testDocCount_asSubAgg() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
terms("terms")
.field("tag")
.order(Order.term(true))
.subAggregation(
histogram("histo")
.field(SINGLE_VALUED_FIELD_NAME)
.interval(interval)
.extendedBounds((long) minRandomValue, (long) maxRandomValue))
.subAggregation(maxBucket("max_bucket").setBucketsPaths("histo>_count")))
.execute()
.actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
List<Terms.Bucket> termsBuckets = terms.getBuckets();
assertThat(termsBuckets.size(), equalTo(interval));
for (int i = 0; i < interval; ++i) {
Terms.Bucket termsBucket = termsBuckets.get(i);
assertThat(termsBucket, notNullValue());
assertThat((String) termsBucket.getKey(), equalTo("tag" + (i % interval)));
Histogram histo = termsBucket.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(histo.getName(), equalTo("histo"));
List<? extends Bucket> buckets = histo.getBuckets();
List<String> maxKeys = new ArrayList<>();
double maxValue = Double.NEGATIVE_INFINITY;
for (int j = 0; j < numValueBuckets; ++j) {
Histogram.Bucket bucket = buckets.get(j);
assertThat(bucket, notNullValue());
assertThat(((Number) bucket.getKey()).longValue(), equalTo((long) j * interval));
if (bucket.getDocCount() > maxValue) {
maxValue = bucket.getDocCount();
maxKeys = new ArrayList<>();
maxKeys.add(bucket.getKeyAsString());
} else if (bucket.getDocCount() == maxValue) {
maxKeys.add(bucket.getKeyAsString());
}
}
InternalBucketMetricValue maxBucketValue = termsBucket.getAggregations().get("max_bucket");
assertThat(maxBucketValue, notNullValue());
assertThat(maxBucketValue.getName(), equalTo("max_bucket"));
assertThat(maxBucketValue.value(), equalTo(maxValue));
assertThat(maxBucketValue.keys(), equalTo(maxKeys.toArray(new String[maxKeys.size()])));
}
}
@Test
public void testMetric_asSubAggOfSingleBucketAgg() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
filter("filter")
.filter(termQuery("tag", "tag0"))
.subAggregation(
histogram("histo")
.field(SINGLE_VALUED_FIELD_NAME)
.interval(interval)
.extendedBounds((long) minRandomValue, (long) maxRandomValue)
.subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME)))
.subAggregation(maxBucket("max_bucket").setBucketsPaths("histo>sum")))
.execute()
.actionGet();
assertSearchResponse(response);
Filter filter = response.getAggregations().get("filter");
assertThat(filter, notNullValue());
assertThat(filter.getName(), equalTo("filter"));
Histogram histo = filter.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(histo.getName(), equalTo("histo"));
List<? extends Bucket> buckets = histo.getBuckets();
List<String> maxKeys = new ArrayList<>();
double maxValue = Double.NEGATIVE_INFINITY;
for (int j = 0; j < numValueBuckets; ++j) {
Histogram.Bucket bucket = buckets.get(j);
assertThat(bucket, notNullValue());
assertThat(((Number) bucket.getKey()).longValue(), equalTo((long) j * interval));
if (bucket.getDocCount() != 0) {
Sum sum = bucket.getAggregations().get("sum");
assertThat(sum, notNullValue());
if (sum.value() > maxValue) {
maxValue = sum.value();
maxKeys = new ArrayList<>();
maxKeys.add(bucket.getKeyAsString());
} else if (sum.value() == maxValue) {
maxKeys.add(bucket.getKeyAsString());
}
}
}
InternalBucketMetricValue maxBucketValue = filter.getAggregations().get("max_bucket");
assertThat(maxBucketValue, notNullValue());
assertThat(maxBucketValue.getName(), equalTo("max_bucket"));
assertThat(maxBucketValue.value(), equalTo(maxValue));
assertThat(maxBucketValue.keys(), equalTo(maxKeys.toArray(new String[maxKeys.size()])));
}
public void testSingleValuedFieldNormalised() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
dateHistogram("histo")
.field("date")
.dateHistogramInterval(DateHistogramInterval.MONTH)
.minDocCount(0)
.subAggregation(derivative("deriv", "_count").unit(DateHistogramInterval.DAY)))
.execute()
.actionGet();
assertSearchResponse(response);
Histogram deriv = response.getAggregations().get("histo");
assertThat(deriv, notNullValue());
assertThat(deriv.getName(), equalTo("histo"));
List<? extends Bucket> buckets = deriv.getBuckets();
assertThat(buckets.size(), equalTo(3));
DateTime key = new DateTime(2012, 1, 1, 0, 0, DateTimeZone.UTC);
Histogram.Bucket bucket = buckets.get(0);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(1L));
Derivative docCountDeriv = bucket.getAggregations().get("deriv");
assertThat(docCountDeriv, nullValue());
key = new DateTime(2012, 2, 1, 0, 0, DateTimeZone.UTC);
bucket = buckets.get(1);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(2L));
docCountDeriv = bucket.getAggregations().get("deriv");
assertThat(docCountDeriv, notNullValue());
assertThat(docCountDeriv.value(), closeTo(1d, 0.00001));
assertThat(docCountDeriv.normalizedValue(), closeTo(1d / 31d, 0.00001));
key = new DateTime(2012, 3, 1, 0, 0, DateTimeZone.UTC);
bucket = buckets.get(2);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(3L));
docCountDeriv = bucket.getAggregations().get("deriv");
assertThat(docCountDeriv, notNullValue());
assertThat(docCountDeriv.value(), closeTo(1d, 0.00001));
assertThat(docCountDeriv.normalizedValue(), closeTo(1d / 29d, 0.00001));
}
public void testUnmapped() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx_unmapped")
.addAggregation(
dateHistogram("histo")
.field("date")
.dateHistogramInterval(DateHistogramInterval.MONTH)
.minDocCount(0)
.subAggregation(derivative("deriv", "_count")))
.execute()
.actionGet();
assertSearchResponse(response);
Histogram deriv = response.getAggregations().get("histo");
assertThat(deriv, notNullValue());
assertThat(deriv.getName(), equalTo("histo"));
assertThat(deriv.getBuckets().size(), equalTo(0));
}
public void testSingleValuedFieldWithSubAggregation() throws Exception {
SearchResponse response =
client()
.prepareSearch("idx")
.addAggregation(
dateHistogram("histo")
.field("date")
.dateHistogramInterval(DateHistogramInterval.MONTH)
.minDocCount(0)
.subAggregation(sum("sum").field("value"))
.subAggregation(derivative("deriv", "sum")))
.execute()
.actionGet();
assertSearchResponse(response);
Histogram histo = response.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(histo.getName(), equalTo("histo"));
List<? extends Bucket> buckets = histo.getBuckets();
assertThat(buckets.size(), equalTo(3));
Object[] propertiesKeys = (Object[]) histo.getProperty("_key");
Object[] propertiesDocCounts = (Object[]) histo.getProperty("_count");
Object[] propertiesCounts = (Object[]) histo.getProperty("sum.value");
DateTime key = new DateTime(2012, 1, 1, 0, 0, DateTimeZone.UTC);
Histogram.Bucket bucket = buckets.get(0);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(1L));
assertThat(bucket.getAggregations().asList().isEmpty(), is(false));
Sum sum = bucket.getAggregations().get("sum");
assertThat(sum, notNullValue());
assertThat(sum.getValue(), equalTo(1.0));
SimpleValue deriv = bucket.getAggregations().get("deriv");
assertThat(deriv, nullValue());
assertThat((DateTime) propertiesKeys[0], equalTo(key));
assertThat((long) propertiesDocCounts[0], equalTo(1L));
assertThat((double) propertiesCounts[0], equalTo(1.0));
key = new DateTime(2012, 2, 1, 0, 0, DateTimeZone.UTC);
bucket = buckets.get(1);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(2L));
assertThat(bucket.getAggregations().asList().isEmpty(), is(false));
sum = bucket.getAggregations().get("sum");
assertThat(sum, notNullValue());
assertThat(sum.getValue(), equalTo(5.0));
deriv = bucket.getAggregations().get("deriv");
assertThat(deriv, notNullValue());
assertThat(deriv.value(), equalTo(4.0));
assertThat(
(double)
bucket.getProperty(
"histo", AggregationPath.parse("deriv.value").getPathElementsAsStringList()),
equalTo(4.0));
assertThat((DateTime) propertiesKeys[1], equalTo(key));
assertThat((long) propertiesDocCounts[1], equalTo(2L));
assertThat((double) propertiesCounts[1], equalTo(5.0));
key = new DateTime(2012, 3, 1, 0, 0, DateTimeZone.UTC);
bucket = buckets.get(2);
assertThat(bucket, notNullValue());
assertThat((DateTime) bucket.getKey(), equalTo(key));
assertThat(bucket.getDocCount(), equalTo(3L));
assertThat(bucket.getAggregations().asList().isEmpty(), is(false));
sum = bucket.getAggregations().get("sum");
assertThat(sum, notNullValue());
assertThat(sum.getValue(), equalTo(15.0));
deriv = bucket.getAggregations().get("deriv");
assertThat(deriv, notNullValue());
assertThat(deriv.value(), equalTo(10.0));
assertThat(
(double)
bucket.getProperty(
"histo", AggregationPath.parse("deriv.value").getPathElementsAsStringList()),
equalTo(10.0));
assertThat((DateTime) propertiesKeys[2], equalTo(key));
assertThat((long) propertiesDocCounts[2], equalTo(3L));
assertThat((double) propertiesCounts[2], equalTo(15.0));
}
/**
* also check for time zone shifts that are not one hour, e.g. "Asia/Kathmandu, 1 Jan 1986 - Time
* Zone Change (IST → NPT), at 00:00:00 clocks were turned forward 00:15 minutes
*/
public void testSingleValuedFieldNormalised_timeZone_AsiaKathmandu() throws Exception {
createIndex(IDX_DST_KATHMANDU);
DateTimeZone timezone = DateTimeZone.forID("Asia/Kathmandu");
List<IndexRequestBuilder> builders = new ArrayList<>();
addNTimes(1, IDX_DST_KATHMANDU, new DateTime("1985-12-31T22:30:00", timezone), builders);
// the shift happens during the next bucket, which includes the 45min that do not start on the
// full hour
addNTimes(2, IDX_DST_KATHMANDU, new DateTime("1985-12-31T23:30:00", timezone), builders);
addNTimes(3, IDX_DST_KATHMANDU, new DateTime("1986-01-01T01:30:00", timezone), builders);
addNTimes(4, IDX_DST_KATHMANDU, new DateTime("1986-01-01T02:30:00", timezone), builders);
indexRandom(true, builders);
ensureSearchable();
SearchResponse response =
client()
.prepareSearch(IDX_DST_KATHMANDU)
.addAggregation(
dateHistogram("histo")
.field("date")
.dateHistogramInterval(DateHistogramInterval.HOUR)
.timeZone(timezone)
.minDocCount(0)
.subAggregation(
derivative("deriv", "_count").unit(DateHistogramInterval.MINUTE)))
.execute()
.actionGet();
assertSearchResponse(response);
Histogram deriv = response.getAggregations().get("histo");
assertThat(deriv, notNullValue());
assertThat(deriv.getName(), equalTo("histo"));
List<? extends Bucket> buckets = deriv.getBuckets();
assertThat(buckets.size(), equalTo(4));
assertBucket(
buckets.get(0),
new DateTime("1985-12-31T22:00:00", timezone).toDateTime(DateTimeZone.UTC),
1L,
nullValue(),
null,
null);
assertBucket(
buckets.get(1),
new DateTime("1985-12-31T23:00:00", timezone).toDateTime(DateTimeZone.UTC),
2L,
notNullValue(),
1d,
1d / 60d);
// the following is normalized using a 105min bucket width
assertBucket(
buckets.get(2),
new DateTime("1986-01-01T01:00:00", timezone).toDateTime(DateTimeZone.UTC),
3L,
notNullValue(),
1d,
1d / 105d);
assertBucket(
buckets.get(3),
new DateTime("1986-01-01T02:00:00", timezone).toDateTime(DateTimeZone.UTC),
4L,
notNullValue(),
1d,
1d / 60d);
}
/** Do a derivative on a date histogram with time zone CET at DST end */
public void testSingleValuedFieldNormalised_timeZone_CET_DstEnd() throws Exception {
createIndex(IDX_DST_END);
DateTimeZone timezone = DateTimeZone.forID("CET");
List<IndexRequestBuilder> builders = new ArrayList<>();
addNTimes(1, IDX_DST_END, new DateTime("2012-10-27T01:00:00", timezone), builders);
addNTimes(
2,
IDX_DST_END,
new DateTime("2012-10-28T01:00:00", timezone),
builders); // day with dst shift -1h, 25h long
addNTimes(3, IDX_DST_END, new DateTime("2012-10-29T01:00:00", timezone), builders);
addNTimes(4, IDX_DST_END, new DateTime("2012-10-30T01:00:00", timezone), builders);
indexRandom(true, builders);
ensureSearchable();
SearchResponse response =
client()
.prepareSearch(IDX_DST_END)
.addAggregation(
dateHistogram("histo")
.field("date")
.dateHistogramInterval(DateHistogramInterval.DAY)
.timeZone(timezone)
.minDocCount(0)
.subAggregation(derivative("deriv", "_count").unit(DateHistogramInterval.HOUR)))
.execute()
.actionGet();
assertSearchResponse(response);
Histogram deriv = response.getAggregations().get("histo");
assertThat(deriv, notNullValue());
assertThat(deriv.getName(), equalTo("histo"));
List<? extends Bucket> buckets = deriv.getBuckets();
assertThat(buckets.size(), equalTo(4));
assertBucket(
buckets.get(0),
new DateTime("2012-10-27", timezone).toDateTime(DateTimeZone.UTC),
1L,
nullValue(),
null,
null);
assertBucket(
buckets.get(1),
new DateTime("2012-10-28", timezone).toDateTime(DateTimeZone.UTC),
2L,
notNullValue(),
1d,
1d / 24d);
// the following is normalized using a 25h bucket width
assertBucket(
buckets.get(2),
new DateTime("2012-10-29", timezone).toDateTime(DateTimeZone.UTC),
3L,
notNullValue(),
1d,
1d / 25d);
assertBucket(
buckets.get(3),
new DateTime("2012-10-30", timezone).toDateTime(DateTimeZone.UTC),
4L,
notNullValue(),
1d,
1d / 24d);
}
