Beispiel #1
0
  public void testLargeNumbersOfPercentileBuckets() throws Exception {
    // test high numbers of percentile buckets to make sure paging and release work correctly
    createIndex("idx");

    final int numDocs = scaledRandomIntBetween(2500, 5000);
    logger.info("Indexing [" + numDocs + "] docs");
    List<IndexRequestBuilder> indexingRequests = Lists.newArrayList();
    for (int i = 0; i < numDocs; ++i) {
      indexingRequests.add(
          client()
              .prepareIndex("idx", "type", Integer.toString(i))
              .setSource("double_value", randomDouble()));
    }
    indexRandom(true, indexingRequests);

    SearchResponse response =
        client()
            .prepareSearch("idx")
            .addAggregation(
                terms("terms")
                    .field("double_value")
                    .collectMode(randomFrom(SubAggCollectionMode.values()))
                    .subAggregation(percentiles("pcts").field("double_value")))
            .execute()
            .actionGet();
    assertAllSuccessful(response);
    assertEquals(numDocs, response.getHits().getTotalHits());
  }
  @Before
  public void init() throws Exception {
    createIndex("idx");
    IndexRequestBuilder[] lowCardBuilders = new IndexRequestBuilder[5]; // TODO randomize the size?
    for (int i = 0; i < lowCardBuilders.length; i++) {
      lowCardBuilders[i] =
          client()
              .prepareIndex("idx", "type")
              .setSource(
                  jsonBuilder()
                      .startObject()
                      .field(SINGLE_VALUED_FIELD_NAME, "val" + i)
                      .field("i", i)
                      .field("tag", i < lowCardBuilders.length / 2 + 1 ? "more" : "less")
                      .startArray(MULTI_VALUED_FIELD_NAME)
                      .value("val" + i)
                      .value("val" + (i + 1))
                      .endArray()
                      .endObject());
    }
    indexRandom(true, lowCardBuilders);
    IndexRequestBuilder[] highCardBuilders =
        new IndexRequestBuilder[100]; // TODO randomize the size?

    for (int i = 0; i < highCardBuilders.length; i++) {
      highCardBuilders[i] =
          client()
              .prepareIndex("idx", "high_card_type")
              .setSource(
                  jsonBuilder()
                      .startObject()
                      .field(SINGLE_VALUED_FIELD_NAME, "val" + Strings.padStart(i + "", 3, '0'))
                      .startArray(MULTI_VALUED_FIELD_NAME)
                      .value("val" + Strings.padStart(i + "", 3, '0'))
                      .value("val" + Strings.padStart((i + 1) + "", 3, '0'))
                      .endArray()
                      .endObject());
    }
    indexRandom(true, highCardBuilders);
    createIndex("idx_unmapped");
    ensureSearchable();
  }
Beispiel #3
0
  // https://github.com/elasticsearch/elasticsearch/issues/6435
  public void testReduce() throws Exception {
    createIndex("idx");
    final int value = randomIntBetween(0, 10);
    indexRandom(true, client().prepareIndex("idx", "type").setSource("f", value));
    ensureYellow("idx"); // only one document let's make sure all shards have an active primary
    SearchResponse response =
        client()
            .prepareSearch("idx")
            .addAggregation(
                filter("filter")
                    .filter(FilterBuilders.matchAllFilter())
                    .subAggregation(
                        range("range")
                            .field("f")
                            .addUnboundedTo(6)
                            .addUnboundedFrom(6)
                            .subAggregation(sum("sum").field("f"))))
            .execute()
            .actionGet();

    assertSearchResponse(response);

    Filter filter = response.getAggregations().get("filter");
    assertNotNull(filter);
    assertEquals(1, filter.getDocCount());

    Range range = filter.getAggregations().get("range");
    assertThat(range, notNullValue());
    assertThat(range.getName(), equalTo("range"));
    assertThat(range.getBuckets().size(), equalTo(2));

    Range.Bucket bucket = range.getBucketByKey("*-6.0");
    assertThat(bucket, notNullValue());
    assertThat(bucket.getKey(), equalTo("*-6.0"));
    assertThat(bucket.getFrom().doubleValue(), equalTo(Double.NEGATIVE_INFINITY));
    assertThat(bucket.getTo().doubleValue(), equalTo(6.0));
    assertThat(bucket.getDocCount(), equalTo(value < 6 ? 1L : 0L));
    Sum sum = bucket.getAggregations().get("sum");
    assertEquals(value < 6 ? value : 0, sum.getValue(), 0d);

    bucket = range.getBucketByKey("6.0-*");
    assertThat(bucket, notNullValue());
    assertThat(bucket.getKey(), equalTo("6.0-*"));
    assertThat(bucket.getFrom().doubleValue(), equalTo(6.0));
    assertThat(bucket.getTo().doubleValue(), equalTo(Double.POSITIVE_INFINITY));
    assertThat(bucket.getDocCount(), equalTo(value >= 6 ? 1L : 0L));
    sum = bucket.getAggregations().get("sum");
    assertEquals(value >= 6 ? value : 0, sum.getValue(), 0d);
  }
  @Test
  public void emptyAggregation() throws Exception {
    prepareCreate("empty_bucket_idx")
        .addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=integer")
        .execute()
        .actionGet();
    List<IndexRequestBuilder> builders = new ArrayList<IndexRequestBuilder>();
    for (int i = 0; i < 2; i++) {
      builders.add(
          client()
              .prepareIndex("empty_bucket_idx", "type", "" + i)
              .setSource(
                  jsonBuilder().startObject().field(SINGLE_VALUED_FIELD_NAME, i * 2).endObject()));
    }
    indexRandom(true, builders.toArray(new IndexRequestBuilder[builders.size()]));

    SearchResponse searchResponse =
        client()
            .prepareSearch("empty_bucket_idx")
            .setQuery(matchAllQuery())
            .addAggregation(
                histogram("histo")
                    .field(SINGLE_VALUED_FIELD_NAME)
                    .interval(1l)
                    .minDocCount(0)
                    .subAggregation(terms("terms")))
            .execute()
            .actionGet();

    assertThat(searchResponse.getHits().getTotalHits(), equalTo(2l));
    Histogram histo = searchResponse.getAggregations().get("histo");
    assertThat(histo, Matchers.notNullValue());
    Histogram.Bucket bucket = histo.getBucketByKey(1l);
    assertThat(bucket, Matchers.notNullValue());

    Terms terms = bucket.getAggregations().get("terms");
    assertThat(terms, Matchers.notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    assertThat(terms.getBuckets().isEmpty(), is(true));
  }
Beispiel #5
0
  // test long/double/string terms aggs with high number of buckets that require array growth
  public void testDuelTerms() throws Exception {
    final int numDocs = scaledRandomIntBetween(1000, 2000);
    final int maxNumTerms = randomIntBetween(10, 5000);

    final IntOpenHashSet valuesSet = new IntOpenHashSet();
    cluster().wipeIndices("idx");
    prepareCreate("idx")
        .addMapping(
            "type",
            jsonBuilder()
                .startObject()
                .startObject("type")
                .startObject("properties")
                .startObject("string_values")
                .field("type", "string")
                .field("index", "not_analyzed")
                .startObject("fields")
                .startObject("doc_values")
                .field("type", "string")
                .field("index", "no")
                .startObject("fielddata")
                .field("format", "doc_values")
                .endObject()
                .endObject()
                .endObject()
                .endObject()
                .startObject("long_values")
                .field("type", "long")
                .endObject()
                .startObject("double_values")
                .field("type", "double")
                .endObject()
                .endObject()
                .endObject())
        .execute()
        .actionGet();

    List<IndexRequestBuilder> indexingRequests = Lists.newArrayList();
    for (int i = 0; i < numDocs; ++i) {
      final int[] values = new int[randomInt(4)];
      for (int j = 0; j < values.length; ++j) {
        values[j] = randomInt(maxNumTerms - 1) - 1000;
        valuesSet.add(values[j]);
      }
      XContentBuilder source =
          jsonBuilder().startObject().field("num", randomDouble()).startArray("long_values");
      for (int j = 0; j < values.length; ++j) {
        source = source.value(values[j]);
      }
      source = source.endArray().startArray("double_values");
      for (int j = 0; j < values.length; ++j) {
        source = source.value((double) values[j]);
      }
      source = source.endArray().startArray("string_values");
      for (int j = 0; j < values.length; ++j) {
        source = source.value(Integer.toString(values[j]));
      }
      source = source.endArray().endObject();
      indexingRequests.add(client().prepareIndex("idx", "type").setSource(source));
    }
    indexRandom(true, indexingRequests);

    assertNoFailures(
        client()
            .admin()
            .indices()
            .prepareRefresh("idx")
            .setIndicesOptions(IndicesOptions.lenientExpandOpen())
            .execute()
            .get());

    TermsAggregatorFactory.ExecutionMode[] globalOrdinalModes =
        new TermsAggregatorFactory.ExecutionMode[] {
          TermsAggregatorFactory.ExecutionMode.GLOBAL_ORDINALS_HASH,
          TermsAggregatorFactory.ExecutionMode.GLOBAL_ORDINALS
        };

    SearchResponse resp =
        client()
            .prepareSearch("idx")
            .addAggregation(
                terms("long")
                    .field("long_values")
                    .size(maxNumTerms)
                    .collectMode(randomFrom(SubAggCollectionMode.values()))
                    .subAggregation(min("min").field("num")))
            .addAggregation(
                terms("double")
                    .field("double_values")
                    .size(maxNumTerms)
                    .collectMode(randomFrom(SubAggCollectionMode.values()))
                    .subAggregation(max("max").field("num")))
            .addAggregation(
                terms("string_map")
                    .field("string_values")
                    .collectMode(randomFrom(SubAggCollectionMode.values()))
                    .executionHint(TermsAggregatorFactory.ExecutionMode.MAP.toString())
                    .size(maxNumTerms)
                    .subAggregation(stats("stats").field("num")))
            .addAggregation(
                terms("string_global_ordinals")
                    .field("string_values")
                    .collectMode(randomFrom(SubAggCollectionMode.values()))
                    .executionHint(
                        globalOrdinalModes[randomInt(globalOrdinalModes.length - 1)].toString())
                    .size(maxNumTerms)
                    .subAggregation(extendedStats("stats").field("num")))
            .addAggregation(
                terms("string_global_ordinals_doc_values")
                    .field("string_values.doc_values")
                    .collectMode(randomFrom(SubAggCollectionMode.values()))
                    .executionHint(
                        globalOrdinalModes[randomInt(globalOrdinalModes.length - 1)].toString())
                    .size(maxNumTerms)
                    .subAggregation(extendedStats("stats").field("num")))
            .execute()
            .actionGet();
    assertAllSuccessful(resp);
    assertEquals(numDocs, resp.getHits().getTotalHits());

    final Terms longTerms = resp.getAggregations().get("long");
    final Terms doubleTerms = resp.getAggregations().get("double");
    final Terms stringMapTerms = resp.getAggregations().get("string_map");
    final Terms stringGlobalOrdinalsTerms = resp.getAggregations().get("string_global_ordinals");
    final Terms stringGlobalOrdinalsDVTerms =
        resp.getAggregations().get("string_global_ordinals_doc_values");

    assertEquals(valuesSet.size(), longTerms.getBuckets().size());
    assertEquals(valuesSet.size(), doubleTerms.getBuckets().size());
    assertEquals(valuesSet.size(), stringMapTerms.getBuckets().size());
    assertEquals(valuesSet.size(), stringGlobalOrdinalsTerms.getBuckets().size());
    assertEquals(valuesSet.size(), stringGlobalOrdinalsDVTerms.getBuckets().size());
    for (Terms.Bucket bucket : longTerms.getBuckets()) {
      final Terms.Bucket doubleBucket =
          doubleTerms.getBucketByKey(
              Double.toString(Long.parseLong(bucket.getKeyAsText().string())));
      final Terms.Bucket stringMapBucket =
          stringMapTerms.getBucketByKey(bucket.getKeyAsText().string());
      final Terms.Bucket stringGlobalOrdinalsBucket =
          stringGlobalOrdinalsTerms.getBucketByKey(bucket.getKeyAsText().string());
      final Terms.Bucket stringGlobalOrdinalsDVBucket =
          stringGlobalOrdinalsDVTerms.getBucketByKey(bucket.getKeyAsText().string());
      assertNotNull(doubleBucket);
      assertNotNull(stringMapBucket);
      assertNotNull(stringGlobalOrdinalsBucket);
      assertNotNull(stringGlobalOrdinalsDVBucket);
      assertEquals(bucket.getDocCount(), doubleBucket.getDocCount());
      assertEquals(bucket.getDocCount(), stringMapBucket.getDocCount());
      assertEquals(bucket.getDocCount(), stringGlobalOrdinalsBucket.getDocCount());
      assertEquals(bucket.getDocCount(), stringGlobalOrdinalsDVBucket.getDocCount());
    }
  }