Exemplo n.º 1
0
  @Test
  public void testChildrenAggs() throws Exception {
    SearchResponse searchResponse =
        client()
            .prepareSearch("test")
            .setQuery(matchQuery("randomized", true))
            .addAggregation(
                terms("category")
                    .field("category")
                    .size(0)
                    .subAggregation(
                        children("to_comment")
                            .childType("comment")
                            .subAggregation(
                                terms("commenters")
                                    .field("commenter")
                                    .size(0)
                                    .subAggregation(topHits("top_comments")))))
            .get();
    assertSearchResponse(searchResponse);

    Terms categoryTerms = searchResponse.getAggregations().get("category");
    assertThat(categoryTerms.getBuckets().size(), equalTo(categoryToControl.size()));
    for (Map.Entry<String, Control> entry1 : categoryToControl.entrySet()) {
      Terms.Bucket categoryBucket = categoryTerms.getBucketByKey(entry1.getKey());
      assertThat(categoryBucket.getKey(), equalTo(entry1.getKey()));
      assertThat(categoryBucket.getDocCount(), equalTo((long) entry1.getValue().articleIds.size()));

      Children childrenBucket = categoryBucket.getAggregations().get("to_comment");
      assertThat(childrenBucket.getName(), equalTo("to_comment"));
      assertThat(childrenBucket.getDocCount(), equalTo((long) entry1.getValue().commentIds.size()));
      assertThat(
          (long) childrenBucket.getProperty("_count"),
          equalTo((long) entry1.getValue().commentIds.size()));

      Terms commentersTerms = childrenBucket.getAggregations().get("commenters");
      assertThat((Terms) childrenBucket.getProperty("commenters"), sameInstance(commentersTerms));
      assertThat(
          commentersTerms.getBuckets().size(),
          equalTo(entry1.getValue().commenterToCommentId.size()));
      for (Map.Entry<String, Set<String>> entry2 :
          entry1.getValue().commenterToCommentId.entrySet()) {
        Terms.Bucket commentBucket = commentersTerms.getBucketByKey(entry2.getKey());
        assertThat(commentBucket.getKey(), equalTo(entry2.getKey()));
        assertThat(commentBucket.getDocCount(), equalTo((long) entry2.getValue().size()));

        TopHits topHits = commentBucket.getAggregations().get("top_comments");
        for (SearchHit searchHit : topHits.getHits().getHits()) {
          assertThat(entry2.getValue().contains(searchHit.getId()), is(true));
        }
      }
    }
  }
Exemplo n.º 2
0
  @Test
  public void multiValuedField_WithValueScript_NotUnique() throws Exception {
    SearchResponse response =
        client()
            .prepareSearch("idx")
            .setTypes("type")
            .addAggregation(
                terms("terms")
                    .executionHint(randomExecutionHint())
                    .field(MULTI_VALUED_FIELD_NAME)
                    .script("_value.substring(0,3)"))
            .execute()
            .actionGet();

    assertSearchResponse(response);

    Terms terms = response.getAggregations().get("terms");
    assertThat(terms, notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    assertThat(terms.getBuckets().size(), equalTo(1));

    Terms.Bucket bucket = terms.getBucketByKey("val");
    assertThat(bucket, notNullValue());
    assertThat(key(bucket), equalTo("val"));
    assertThat(bucket.getDocCount(), equalTo(5l));
  }
Exemplo n.º 3
0
  @Test
  public void script_SingleValue_WithSubAggregator_Inherited() throws Exception {
    SearchResponse response =
        client()
            .prepareSearch("idx")
            .setTypes("type")
            .addAggregation(
                terms("terms")
                    .executionHint(randomExecutionHint())
                    .script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value")
                    .subAggregation(count("count")))
            .execute()
            .actionGet();

    assertSearchResponse(response);

    Terms terms = response.getAggregations().get("terms");
    assertThat(terms, notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    assertThat(terms.getBuckets().size(), equalTo(5));

    for (int i = 0; i < 5; i++) {
      Terms.Bucket bucket = terms.getBucketByKey("val" + i);
      assertThat(bucket, notNullValue());
      assertThat(key(bucket), equalTo("val" + i));
      assertThat(bucket.getDocCount(), equalTo(1l));
      ValueCount valueCount = bucket.getAggregations().get("count");
      assertThat(valueCount, notNullValue());
      assertThat(valueCount.getValue(), equalTo(1l));
    }
  }
Exemplo n.º 4
0
  @Test
  public void script_MultiValued() throws Exception {
    SearchResponse response =
        client()
            .prepareSearch("idx")
            .setTypes("type")
            .addAggregation(
                terms("terms")
                    .executionHint(randomExecutionHint())
                    .script("doc['" + MULTI_VALUED_FIELD_NAME + "'].values"))
            .execute()
            .actionGet();

    assertSearchResponse(response);

    Terms terms = response.getAggregations().get("terms");
    assertThat(terms, notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    assertThat(terms.getBuckets().size(), equalTo(6));

    for (int i = 0; i < 6; i++) {
      Terms.Bucket bucket = terms.getBucketByKey("val" + i);
      assertThat(bucket, notNullValue());
      assertThat(key(bucket), equalTo("val" + i));
      if (i == 0 || i == 5) {
        assertThat(bucket.getDocCount(), equalTo(1l));
      } else {
        assertThat(bucket.getDocCount(), equalTo(2l));
      }
    }
  }
Exemplo n.º 5
0
  @Test
  public void partiallyUnmapped() throws Exception {
    SearchResponse response =
        client()
            .prepareSearch("idx", "idx_unmapped")
            .setTypes("type")
            .addAggregation(
                terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME))
            .execute()
            .actionGet();

    assertSearchResponse(response);

    Terms terms = response.getAggregations().get("terms");
    assertThat(terms, notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    assertThat(terms.getBuckets().size(), equalTo(5));

    for (int i = 0; i < 5; i++) {
      Terms.Bucket bucket = terms.getBucketByKey("val" + i);
      assertThat(bucket, notNullValue());
      assertThat(key(bucket), equalTo("val" + i));
      assertThat(bucket.getDocCount(), equalTo(1l));
    }
  }
Exemplo n.º 6
0
  @Test
  public void stringTermsNestedIntoPerBucketAggregator() throws Exception {
    // no execution hint so that the logic that decides whether or not to use ordinals is executed
    SearchResponse response =
        client()
            .prepareSearch("idx")
            .setTypes("type")
            .addAggregation(
                filter("filter")
                    .filter(termFilter(MULTI_VALUED_FIELD_NAME, "val3"))
                    .subAggregation(terms("terms").field(MULTI_VALUED_FIELD_NAME)))
            .execute()
            .actionGet();

    assertThat(response.getFailedShards(), equalTo(0));

    Filter filter = response.getAggregations().get("filter");

    Terms terms = filter.getAggregations().get("terms");
    assertThat(terms, notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    assertThat(terms.getBuckets().size(), equalTo(3));

    for (int i = 2; i <= 4; i++) {
      Terms.Bucket bucket = terms.getBucketByKey("val" + i);
      assertThat(bucket, notNullValue());
      assertThat(key(bucket), equalTo("val" + i));
      assertThat(bucket.getDocCount(), equalTo(i == 3 ? 2L : 1L));
    }
  }
Exemplo n.º 7
0
  @Test
  public void singleValueField_WithMaxSize() throws Exception {
    SearchResponse response =
        client()
            .prepareSearch("idx")
            .setTypes("high_card_type")
            .addAggregation(
                terms("terms")
                    .executionHint(randomExecutionHint())
                    .field(SINGLE_VALUED_FIELD_NAME)
                    .size(20)
                    .order(
                        Terms.Order.term(
                            true))) // we need to sort by terms cause we're checking the first 20
                                    // values
            .execute()
            .actionGet();

    assertSearchResponse(response);

    Terms terms = response.getAggregations().get("terms");
    assertThat(terms, notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    assertThat(terms.getBuckets().size(), equalTo(20));

    for (int i = 0; i < 20; i++) {
      Terms.Bucket bucket = terms.getBucketByKey("val" + Strings.padStart(i + "", 3, '0'));
      assertThat(bucket, notNullValue());
      assertThat(key(bucket), equalTo("val" + Strings.padStart(i + "", 3, '0')));
      assertThat(bucket.getDocCount(), equalTo(1l));
    }
  }
Exemplo n.º 8
0
 public void testUnmappedTerms() {
   SearchResponse response =
       client()
           .prepareSearch("idx")
           .addAggregation(terms("my_terms").field("non_existing_field").missing("bar"))
           .get();
   assertSearchResponse(response);
   Terms terms = response.getAggregations().get("my_terms");
   assertEquals(1, terms.getBuckets().size());
   assertEquals(2, terms.getBucketByKey("bar").getDocCount());
 }
Exemplo n.º 9
0
  public void testDoubleTerms() {
    SearchResponse response =
        client()
            .prepareSearch("idx")
            .addAggregation(terms("my_terms").field("double").missing(4.5))
            .get();
    assertSearchResponse(response);
    Terms terms = response.getAggregations().get("my_terms");
    assertEquals(2, terms.getBuckets().size());
    assertEquals(1, terms.getBucketByKey("4.5").getDocCount());
    assertEquals(1, terms.getBucketByKey("5.5").getDocCount());

    response =
        client()
            .prepareSearch("idx")
            .addAggregation(terms("my_terms").field("double").missing(5.5))
            .get();
    assertSearchResponse(response);
    terms = response.getAggregations().get("my_terms");
    assertEquals(1, terms.getBuckets().size());
    assertEquals(2, terms.getBucketByKey("5.5").getDocCount());
  }
Exemplo n.º 10
0
  private void assertDocCountErrorWithinBounds(
      int size, SearchResponse accurateResponse, SearchResponse testResponse) {
    Terms accurateTerms = accurateResponse.getAggregations().get("terms");
    assertThat(accurateTerms, notNullValue());
    assertThat(accurateTerms.getName(), equalTo("terms"));
    assertThat(accurateTerms.getDocCountError(), equalTo(0L));

    Terms testTerms = testResponse.getAggregations().get("terms");
    assertThat(testTerms, notNullValue());
    assertThat(testTerms.getName(), equalTo("terms"));
    assertThat(testTerms.getDocCountError(), greaterThanOrEqualTo(0L));
    Collection<Bucket> testBuckets = testTerms.getBuckets();
    assertThat(testBuckets.size(), lessThanOrEqualTo(size));
    assertThat(accurateTerms.getBuckets().size(), greaterThanOrEqualTo(testBuckets.size()));

    for (Terms.Bucket testBucket : testBuckets) {
      assertThat(testBucket, notNullValue());
      Terms.Bucket accurateBucket = accurateTerms.getBucketByKey(testBucket.getKeyAsString());
      assertThat(accurateBucket, notNullValue());
      assertThat(accurateBucket.getDocCountError(), equalTo(0L));
      assertThat(testBucket.getDocCountError(), lessThanOrEqualTo(testTerms.getDocCountError()));
      assertThat(
          testBucket.getDocCount() + testBucket.getDocCountError(),
          greaterThanOrEqualTo(accurateBucket.getDocCount()));
      assertThat(
          testBucket.getDocCount() - testBucket.getDocCountError(),
          lessThanOrEqualTo(accurateBucket.getDocCount()));
    }

    for (Terms.Bucket accurateBucket : accurateTerms.getBuckets()) {
      assertThat(accurateBucket, notNullValue());
      Terms.Bucket testBucket = accurateTerms.getBucketByKey(accurateBucket.getKeyAsString());
      if (testBucket == null) {
        assertThat(accurateBucket.getDocCount(), lessThanOrEqualTo(testTerms.getDocCountError()));
      }
    }
  }
Exemplo n.º 11
0
  @Test
  public void multiValuedField_WithValueScript_WithInheritedSubAggregator() throws Exception {
    SearchResponse response =
        client()
            .prepareSearch("idx")
            .setTypes("type")
            .addAggregation(
                terms("terms")
                    .executionHint(randomExecutionHint())
                    .field(MULTI_VALUED_FIELD_NAME)
                    .script("'foo_' + _value")
                    .subAggregation(count("count")))
            .execute()
            .actionGet();

    assertSearchResponse(response);

    Terms terms = response.getAggregations().get("terms");
    assertThat(terms, notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    assertThat(terms.getBuckets().size(), equalTo(6));

    for (int i = 0; i < 6; i++) {
      Terms.Bucket bucket = terms.getBucketByKey("foo_val" + i);
      assertThat(bucket, notNullValue());
      assertThat(key(bucket), equalTo("foo_val" + i));
      if (i == 0 | i == 5) {
        assertThat(bucket.getDocCount(), equalTo(1l));
        ValueCount valueCount = bucket.getAggregations().get("count");
        assertThat(valueCount, notNullValue());
        assertThat(valueCount.getValue(), equalTo(2l));
      } else {
        assertThat(bucket.getDocCount(), equalTo(2l));
        ValueCount valueCount = bucket.getAggregations().get("count");
        assertThat(valueCount, notNullValue());
        assertThat("term[" + key(bucket) + "]", valueCount.getValue(), equalTo(4l));
      }
    }
  }
Exemplo n.º 12
0
  public void testNestedSameDocIdProcessedMultipleTime() throws Exception {
    assertAcked(
        prepareCreate("idx4")
            .setSettings(
                Settings.builder()
                    .put(SETTING_NUMBER_OF_SHARDS, 1)
                    .put(SETTING_NUMBER_OF_REPLICAS, 0))
            .addMapping(
                "product",
                "categories",
                "type=text",
                "name",
                "type=text",
                "property",
                "type=nested"));
    ensureGreen("idx4");

    client()
        .prepareIndex("idx4", "product", "1")
        .setSource(
            jsonBuilder()
                .startObject()
                .field("name", "product1")
                .field("categories", "1", "2", "3", "4")
                .startArray("property")
                .startObject()
                .field("id", 1)
                .endObject()
                .startObject()
                .field("id", 2)
                .endObject()
                .startObject()
                .field("id", 3)
                .endObject()
                .endArray()
                .endObject())
        .get();
    client()
        .prepareIndex("idx4", "product", "2")
        .setSource(
            jsonBuilder()
                .startObject()
                .field("name", "product2")
                .field("categories", "1", "2")
                .startArray("property")
                .startObject()
                .field("id", 1)
                .endObject()
                .startObject()
                .field("id", 5)
                .endObject()
                .startObject()
                .field("id", 4)
                .endObject()
                .endArray()
                .endObject())
        .get();
    refresh();

    SearchResponse response =
        client()
            .prepareSearch("idx4")
            .setTypes("product")
            .addAggregation(
                terms("category")
                    .field("categories")
                    .subAggregation(
                        nested("property", "property")
                            .subAggregation(terms("property_id").field("property.id"))))
            .get();
    assertNoFailures(response);
    assertHitCount(response, 2);

    Terms category = response.getAggregations().get("category");
    assertThat(category.getBuckets().size(), equalTo(4));

    Terms.Bucket bucket = category.getBucketByKey("1");
    assertThat(bucket.getDocCount(), equalTo(2L));
    Nested property = bucket.getAggregations().get("property");
    assertThat(property.getDocCount(), equalTo(6L));
    Terms propertyId = property.getAggregations().get("property_id");
    assertThat(propertyId.getBuckets().size(), equalTo(5));
    assertThat(propertyId.getBucketByKey("1").getDocCount(), equalTo(2L));
    assertThat(propertyId.getBucketByKey("2").getDocCount(), equalTo(1L));
    assertThat(propertyId.getBucketByKey("3").getDocCount(), equalTo(1L));
    assertThat(propertyId.getBucketByKey("4").getDocCount(), equalTo(1L));
    assertThat(propertyId.getBucketByKey("5").getDocCount(), equalTo(1L));

    bucket = category.getBucketByKey("2");
    assertThat(bucket.getDocCount(), equalTo(2L));
    property = bucket.getAggregations().get("property");
    assertThat(property.getDocCount(), equalTo(6L));
    propertyId = property.getAggregations().get("property_id");
    assertThat(propertyId.getBuckets().size(), equalTo(5));
    assertThat(propertyId.getBucketByKey("1").getDocCount(), equalTo(2L));
    assertThat(propertyId.getBucketByKey("2").getDocCount(), equalTo(1L));
    assertThat(propertyId.getBucketByKey("3").getDocCount(), equalTo(1L));
    assertThat(propertyId.getBucketByKey("4").getDocCount(), equalTo(1L));
    assertThat(propertyId.getBucketByKey("5").getDocCount(), equalTo(1L));

    bucket = category.getBucketByKey("3");
    assertThat(bucket.getDocCount(), equalTo(1L));
    property = bucket.getAggregations().get("property");
    assertThat(property.getDocCount(), equalTo(3L));
    propertyId = property.getAggregations().get("property_id");
    assertThat(propertyId.getBuckets().size(), equalTo(3));
    assertThat(propertyId.getBucketByKey("1").getDocCount(), equalTo(1L));
    assertThat(propertyId.getBucketByKey("2").getDocCount(), equalTo(1L));
    assertThat(propertyId.getBucketByKey("3").getDocCount(), equalTo(1L));

    bucket = category.getBucketByKey("4");
    assertThat(bucket.getDocCount(), equalTo(1L));
    property = bucket.getAggregations().get("property");
    assertThat(property.getDocCount(), equalTo(3L));
    propertyId = property.getAggregations().get("property_id");
    assertThat(propertyId.getBuckets().size(), equalTo(3));
    assertThat(propertyId.getBucketByKey("1").getDocCount(), equalTo(1L));
    assertThat(propertyId.getBucketByKey("2").getDocCount(), equalTo(1L));
    assertThat(propertyId.getBucketByKey("3").getDocCount(), equalTo(1L));
  }
Exemplo n.º 13
0
  // Test based on: https://github.com/elastic/elasticsearch/issues/9280
  public void testParentFilterResolvedCorrectly() throws Exception {
    XContentBuilder mapping =
        jsonBuilder()
            .startObject()
            .startObject("provider")
            .startObject("properties")
            .startObject("comments")
            .field("type", "nested")
            .startObject("properties")
            .startObject("cid")
            .field("type", "long")
            .endObject()
            .startObject("identifier")
            .field("type", "keyword")
            .endObject()
            .startObject("tags")
            .field("type", "nested")
            .startObject("properties")
            .startObject("tid")
            .field("type", "long")
            .endObject()
            .startObject("name")
            .field("type", "keyword")
            .endObject()
            .endObject()
            .endObject()
            .endObject()
            .endObject()
            .startObject("dates")
            .field("type", "object")
            .startObject("properties")
            .startObject("day")
            .field("type", "date")
            .field("format", "dateOptionalTime")
            .endObject()
            .startObject("month")
            .field("type", "object")
            .startObject("properties")
            .startObject("end")
            .field("type", "date")
            .field("format", "dateOptionalTime")
            .endObject()
            .startObject("start")
            .field("type", "date")
            .field("format", "dateOptionalTime")
            .endObject()
            .startObject("label")
            .field("type", "keyword")
            .endObject()
            .endObject()
            .endObject()
            .endObject()
            .endObject()
            .endObject()
            .endObject()
            .endObject();
    assertAcked(
        prepareCreate("idx2")
            .setSettings(
                Settings.builder()
                    .put(SETTING_NUMBER_OF_SHARDS, 1)
                    .put(SETTING_NUMBER_OF_REPLICAS, 0))
            .addMapping("provider", mapping));
    ensureGreen("idx2");

    List<IndexRequestBuilder> indexRequests = new ArrayList<>(2);
    indexRequests.add(
        client()
            .prepareIndex("idx2", "provider", "1")
            .setSource(
                "{\"dates\": {\"month\": {\"label\": \"2014-11\", \"end\": \"2014-11-30\", \"start\": \"2014-11-01\"}, \"day\": \"2014-11-30\"}, \"comments\": [{\"cid\": 3,\"identifier\": \"29111\"}, {\"cid\": 4,\"tags\": [{\"tid\" :44,\"name\": \"Roles\"}], \"identifier\": \"29101\"}]}"));
    indexRequests.add(
        client()
            .prepareIndex("idx2", "provider", "2")
            .setSource(
                "{\"dates\": {\"month\": {\"label\": \"2014-12\", \"end\": \"2014-12-31\", \"start\": \"2014-12-01\"}, \"day\": \"2014-12-03\"}, \"comments\": [{\"cid\": 1, \"identifier\": \"29111\"}, {\"cid\": 2,\"tags\": [{\"tid\" : 22, \"name\": \"DataChannels\"}], \"identifier\": \"29101\"}]}"));
    indexRandom(true, indexRequests);

    SearchResponse response =
        client()
            .prepareSearch("idx2")
            .setTypes("provider")
            .addAggregation(
                terms("startDate")
                    .field("dates.month.start")
                    .subAggregation(
                        terms("endDate")
                            .field("dates.month.end")
                            .subAggregation(
                                terms("period")
                                    .field("dates.month.label")
                                    .subAggregation(
                                        nested("ctxt_idfier_nested", "comments")
                                            .subAggregation(
                                                filter(
                                                        "comment_filter",
                                                        termQuery("comments.identifier", "29111"))
                                                    .subAggregation(
                                                        nested("nested_tags", "comments.tags")
                                                            .subAggregation(
                                                                terms("tag")
                                                                    .field(
                                                                        "comments.tags.name"))))))))
            .get();
    assertNoFailures(response);
    assertHitCount(response, 2);

    Terms startDate = response.getAggregations().get("startDate");
    assertThat(startDate.getBuckets().size(), equalTo(2));
    Terms.Bucket bucket = startDate.getBucketByKey("2014-11-01T00:00:00.000Z");
    assertThat(bucket.getDocCount(), equalTo(1L));
    Terms endDate = bucket.getAggregations().get("endDate");
    bucket = endDate.getBucketByKey("2014-11-30T00:00:00.000Z");
    assertThat(bucket.getDocCount(), equalTo(1L));
    Terms period = bucket.getAggregations().get("period");
    bucket = period.getBucketByKey("2014-11");
    assertThat(bucket.getDocCount(), equalTo(1L));
    Nested comments = bucket.getAggregations().get("ctxt_idfier_nested");
    assertThat(comments.getDocCount(), equalTo(2L));
    Filter filter = comments.getAggregations().get("comment_filter");
    assertThat(filter.getDocCount(), equalTo(1L));
    Nested nestedTags = filter.getAggregations().get("nested_tags");
    assertThat(nestedTags.getDocCount(), equalTo(0L)); // This must be 0
    Terms tags = nestedTags.getAggregations().get("tag");
    assertThat(tags.getBuckets().size(), equalTo(0)); // and this must be empty

    bucket = startDate.getBucketByKey("2014-12-01T00:00:00.000Z");
    assertThat(bucket.getDocCount(), equalTo(1L));
    endDate = bucket.getAggregations().get("endDate");
    bucket = endDate.getBucketByKey("2014-12-31T00:00:00.000Z");
    assertThat(bucket.getDocCount(), equalTo(1L));
    period = bucket.getAggregations().get("period");
    bucket = period.getBucketByKey("2014-12");
    assertThat(bucket.getDocCount(), equalTo(1L));
    comments = bucket.getAggregations().get("ctxt_idfier_nested");
    assertThat(comments.getDocCount(), equalTo(2L));
    filter = comments.getAggregations().get("comment_filter");
    assertThat(filter.getDocCount(), equalTo(1L));
    nestedTags = filter.getAggregations().get("nested_tags");
    assertThat(nestedTags.getDocCount(), equalTo(0L)); // This must be 0
    tags = nestedTags.getAggregations().get("tag");
    assertThat(tags.getBuckets().size(), equalTo(0)); // and this must be empty
  }
Exemplo n.º 14
0
  // test long/double/string terms aggs with high number of buckets that require array growth
  public void testDuelTerms() throws Exception {
    final int numDocs = scaledRandomIntBetween(1000, 2000);
    final int maxNumTerms = randomIntBetween(10, 5000);

    final IntOpenHashSet valuesSet = new IntOpenHashSet();
    cluster().wipeIndices("idx");
    prepareCreate("idx")
        .addMapping(
            "type",
            jsonBuilder()
                .startObject()
                .startObject("type")
                .startObject("properties")
                .startObject("string_values")
                .field("type", "string")
                .field("index", "not_analyzed")
                .startObject("fields")
                .startObject("doc_values")
                .field("type", "string")
                .field("index", "no")
                .startObject("fielddata")
                .field("format", "doc_values")
                .endObject()
                .endObject()
                .endObject()
                .endObject()
                .startObject("long_values")
                .field("type", "long")
                .endObject()
                .startObject("double_values")
                .field("type", "double")
                .endObject()
                .endObject()
                .endObject())
        .execute()
        .actionGet();

    List<IndexRequestBuilder> indexingRequests = Lists.newArrayList();
    for (int i = 0; i < numDocs; ++i) {
      final int[] values = new int[randomInt(4)];
      for (int j = 0; j < values.length; ++j) {
        values[j] = randomInt(maxNumTerms - 1) - 1000;
        valuesSet.add(values[j]);
      }
      XContentBuilder source =
          jsonBuilder().startObject().field("num", randomDouble()).startArray("long_values");
      for (int j = 0; j < values.length; ++j) {
        source = source.value(values[j]);
      }
      source = source.endArray().startArray("double_values");
      for (int j = 0; j < values.length; ++j) {
        source = source.value((double) values[j]);
      }
      source = source.endArray().startArray("string_values");
      for (int j = 0; j < values.length; ++j) {
        source = source.value(Integer.toString(values[j]));
      }
      source = source.endArray().endObject();
      indexingRequests.add(client().prepareIndex("idx", "type").setSource(source));
    }
    indexRandom(true, indexingRequests);

    assertNoFailures(
        client()
            .admin()
            .indices()
            .prepareRefresh("idx")
            .setIndicesOptions(IndicesOptions.lenientExpandOpen())
            .execute()
            .get());

    TermsAggregatorFactory.ExecutionMode[] globalOrdinalModes =
        new TermsAggregatorFactory.ExecutionMode[] {
          TermsAggregatorFactory.ExecutionMode.GLOBAL_ORDINALS_HASH,
          TermsAggregatorFactory.ExecutionMode.GLOBAL_ORDINALS
        };

    SearchResponse resp =
        client()
            .prepareSearch("idx")
            .addAggregation(
                terms("long")
                    .field("long_values")
                    .size(maxNumTerms)
                    .collectMode(randomFrom(SubAggCollectionMode.values()))
                    .subAggregation(min("min").field("num")))
            .addAggregation(
                terms("double")
                    .field("double_values")
                    .size(maxNumTerms)
                    .collectMode(randomFrom(SubAggCollectionMode.values()))
                    .subAggregation(max("max").field("num")))
            .addAggregation(
                terms("string_map")
                    .field("string_values")
                    .collectMode(randomFrom(SubAggCollectionMode.values()))
                    .executionHint(TermsAggregatorFactory.ExecutionMode.MAP.toString())
                    .size(maxNumTerms)
                    .subAggregation(stats("stats").field("num")))
            .addAggregation(
                terms("string_global_ordinals")
                    .field("string_values")
                    .collectMode(randomFrom(SubAggCollectionMode.values()))
                    .executionHint(
                        globalOrdinalModes[randomInt(globalOrdinalModes.length - 1)].toString())
                    .size(maxNumTerms)
                    .subAggregation(extendedStats("stats").field("num")))
            .addAggregation(
                terms("string_global_ordinals_doc_values")
                    .field("string_values.doc_values")
                    .collectMode(randomFrom(SubAggCollectionMode.values()))
                    .executionHint(
                        globalOrdinalModes[randomInt(globalOrdinalModes.length - 1)].toString())
                    .size(maxNumTerms)
                    .subAggregation(extendedStats("stats").field("num")))
            .execute()
            .actionGet();
    assertAllSuccessful(resp);
    assertEquals(numDocs, resp.getHits().getTotalHits());

    final Terms longTerms = resp.getAggregations().get("long");
    final Terms doubleTerms = resp.getAggregations().get("double");
    final Terms stringMapTerms = resp.getAggregations().get("string_map");
    final Terms stringGlobalOrdinalsTerms = resp.getAggregations().get("string_global_ordinals");
    final Terms stringGlobalOrdinalsDVTerms =
        resp.getAggregations().get("string_global_ordinals_doc_values");

    assertEquals(valuesSet.size(), longTerms.getBuckets().size());
    assertEquals(valuesSet.size(), doubleTerms.getBuckets().size());
    assertEquals(valuesSet.size(), stringMapTerms.getBuckets().size());
    assertEquals(valuesSet.size(), stringGlobalOrdinalsTerms.getBuckets().size());
    assertEquals(valuesSet.size(), stringGlobalOrdinalsDVTerms.getBuckets().size());
    for (Terms.Bucket bucket : longTerms.getBuckets()) {
      final Terms.Bucket doubleBucket =
          doubleTerms.getBucketByKey(
              Double.toString(Long.parseLong(bucket.getKeyAsText().string())));
      final Terms.Bucket stringMapBucket =
          stringMapTerms.getBucketByKey(bucket.getKeyAsText().string());
      final Terms.Bucket stringGlobalOrdinalsBucket =
          stringGlobalOrdinalsTerms.getBucketByKey(bucket.getKeyAsText().string());
      final Terms.Bucket stringGlobalOrdinalsDVBucket =
          stringGlobalOrdinalsDVTerms.getBucketByKey(bucket.getKeyAsText().string());
      assertNotNull(doubleBucket);
      assertNotNull(stringMapBucket);
      assertNotNull(stringGlobalOrdinalsBucket);
      assertNotNull(stringGlobalOrdinalsDVBucket);
      assertEquals(bucket.getDocCount(), doubleBucket.getDocCount());
      assertEquals(bucket.getDocCount(), stringMapBucket.getDocCount());
      assertEquals(bucket.getDocCount(), stringGlobalOrdinalsBucket.getDocCount());
      assertEquals(bucket.getDocCount(), stringGlobalOrdinalsDVBucket.getDocCount());
    }
  }
Exemplo n.º 15
0
  @Test
  public void testParentWithMultipleBuckets() throws Exception {
    SearchResponse searchResponse =
        client()
            .prepareSearch("test")
            .setQuery(matchQuery("randomized", false))
            .addAggregation(
                terms("category")
                    .field("category")
                    .size(0)
                    .subAggregation(
                        children("to_comment")
                            .childType("comment")
                            .subAggregation(topHits("top_comments").addSort("_id", SortOrder.ASC))))
            .get();
    assertSearchResponse(searchResponse);

    Terms categoryTerms = searchResponse.getAggregations().get("category");
    assertThat(categoryTerms.getBuckets().size(), equalTo(3));

    for (Terms.Bucket bucket : categoryTerms.getBuckets()) {
      logger.info("bucket=" + bucket.getKey());
      Children childrenBucket = bucket.getAggregations().get("to_comment");
      TopHits topHits = childrenBucket.getAggregations().get("top_comments");
      logger.info("total_hits={}", topHits.getHits().getTotalHits());
      for (SearchHit searchHit : topHits.getHits()) {
        logger.info(
            "hit= {} {} {}", searchHit.sortValues()[0], searchHit.getType(), searchHit.getId());
      }
    }

    Terms.Bucket categoryBucket = categoryTerms.getBucketByKey("a");
    assertThat(categoryBucket.getKey(), equalTo("a"));
    assertThat(categoryBucket.getDocCount(), equalTo(3l));

    Children childrenBucket = categoryBucket.getAggregations().get("to_comment");
    assertThat(childrenBucket.getName(), equalTo("to_comment"));
    assertThat(childrenBucket.getDocCount(), equalTo(2l));
    TopHits topHits = childrenBucket.getAggregations().get("top_comments");
    assertThat(topHits.getHits().totalHits(), equalTo(2l));
    assertThat(topHits.getHits().getAt(0).sortValues()[0].toString(), equalTo("a"));
    assertThat(topHits.getHits().getAt(0).getId(), equalTo("a"));
    assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
    assertThat(topHits.getHits().getAt(1).sortValues()[0].toString(), equalTo("c"));
    assertThat(topHits.getHits().getAt(1).getId(), equalTo("c"));
    assertThat(topHits.getHits().getAt(1).getType(), equalTo("comment"));

    categoryBucket = categoryTerms.getBucketByKey("b");
    assertThat(categoryBucket.getKey(), equalTo("b"));
    assertThat(categoryBucket.getDocCount(), equalTo(2l));

    childrenBucket = categoryBucket.getAggregations().get("to_comment");
    assertThat(childrenBucket.getName(), equalTo("to_comment"));
    assertThat(childrenBucket.getDocCount(), equalTo(1l));
    topHits = childrenBucket.getAggregations().get("top_comments");
    assertThat(topHits.getHits().totalHits(), equalTo(1l));
    assertThat(topHits.getHits().getAt(0).getId(), equalTo("c"));
    assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));

    categoryBucket = categoryTerms.getBucketByKey("c");
    assertThat(categoryBucket.getKey(), equalTo("c"));
    assertThat(categoryBucket.getDocCount(), equalTo(2l));

    childrenBucket = categoryBucket.getAggregations().get("to_comment");
    assertThat(childrenBucket.getName(), equalTo("to_comment"));
    assertThat(childrenBucket.getDocCount(), equalTo(1l));
    topHits = childrenBucket.getAggregations().get("top_comments");
    assertThat(topHits.getHits().totalHits(), equalTo(1l));
    assertThat(topHits.getHits().getAt(0).getId(), equalTo("c"));
    assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
  }
Exemplo n.º 16
0
  @Test
  public void singleValueField_WithRegexFiltering() throws Exception {

    // include without exclude
    // we should be left with: val000, val001, val002, val003, val004, val005, val006, val007,
    // val008, val009

    SearchResponse response =
        client()
            .prepareSearch("idx")
            .setTypes("high_card_type")
            .addAggregation(terms("terms").field(SINGLE_VALUED_FIELD_NAME).include("val00.+"))
            .execute()
            .actionGet();

    assertSearchResponse(response);

    Terms terms = response.getAggregations().get("terms");
    assertThat(terms, notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    assertThat(terms.getBuckets().size(), equalTo(10));

    for (int i = 0; i < 10; i++) {
      Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
      assertThat(bucket, notNullValue());
      assertThat(key(bucket), equalTo("val00" + i));
      assertThat(bucket.getDocCount(), equalTo(1l));
    }

    // include and exclude
    // we should be left with: val002, val003, val004, val005, val006, val007, val008, val009

    response =
        client()
            .prepareSearch("idx")
            .setTypes("high_card_type")
            .addAggregation(
                terms("terms")
                    .field(SINGLE_VALUED_FIELD_NAME)
                    .include("val00.+")
                    .exclude("(val000|val001)"))
            .execute()
            .actionGet();

    assertSearchResponse(response);

    terms = response.getAggregations().get("terms");
    assertThat(terms, notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    assertThat(terms.getBuckets().size(), equalTo(8));

    for (int i = 2; i < 10; i++) {
      Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
      assertThat(bucket, notNullValue());
      assertThat(key(bucket), equalTo("val00" + i));
      assertThat(bucket.getDocCount(), equalTo(1l));
    }

    // exclude without include
    // we should be left with: val000, val001, val002, val003, val004, val005, val006, val007,
    // val008, val009

    response =
        client()
            .prepareSearch("idx")
            .setTypes("high_card_type")
            .addAggregation(terms("terms").field(SINGLE_VALUED_FIELD_NAME).exclude("val0[1-9]+.+"))
            .execute()
            .actionGet();

    assertSearchResponse(response);

    terms = response.getAggregations().get("terms");
    assertThat(terms, notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    assertThat(terms.getBuckets().size(), equalTo(10));

    for (int i = 0; i < 10; i++) {
      Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
      assertThat(bucket, notNullValue());
      assertThat(key(bucket), equalTo("val00" + i));
      assertThat(bucket.getDocCount(), equalTo(1l));
    }
  }
Exemplo n.º 17
0
  @Test
  public void singleValueField_WithRegexFiltering_WithFlags() throws Exception {

    // include without exclude
    // we should be left with: val000, val001, val002, val003, val004, val005, val006, val007,
    // val008, val009
    // with case insensitive flag on the include regex

    SearchResponse response =
        client()
            .prepareSearch("idx")
            .setTypes("high_card_type")
            .addAggregation(
                terms("terms")
                    .field(SINGLE_VALUED_FIELD_NAME)
                    .include("VAL00.+", Pattern.CASE_INSENSITIVE))
            .execute()
            .actionGet();

    assertSearchResponse(response);

    Terms terms = response.getAggregations().get("terms");
    assertThat(terms, notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    assertThat(terms.getBuckets().size(), equalTo(10));

    for (int i = 0; i < 10; i++) {
      Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
      assertThat(bucket, notNullValue());
      assertThat(key(bucket), equalTo("val00" + i));
      assertThat(bucket.getDocCount(), equalTo(1l));
    }

    // include and exclude
    // we should be left with: val002, val003, val004, val005, val006, val007, val008, val009
    // with multi-flag masking on the exclude regex

    response =
        client()
            .prepareSearch("idx")
            .setTypes("high_card_type")
            .addAggregation(
                terms("terms")
                    .field(SINGLE_VALUED_FIELD_NAME)
                    .include("val00.+")
                    .exclude(
                        "( val000 | VAL001 )#this is a comment",
                        Pattern.CASE_INSENSITIVE | Pattern.COMMENTS))
            .execute()
            .actionGet();

    assertSearchResponse(response);

    terms = response.getAggregations().get("terms");
    assertThat(terms, notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    assertThat(terms.getBuckets().size(), equalTo(8));

    for (int i = 2; i < 10; i++) {
      Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
      assertThat(bucket, notNullValue());
      assertThat(key(bucket), equalTo("val00" + i));
      assertThat(bucket.getDocCount(), equalTo(1l));
    }

    // exclude without include
    // we should be left with: val000, val001, val002, val003, val004, val005, val006, val007,
    // val008, val009
    // with a "no flag" flag

    response =
        client()
            .prepareSearch("idx")
            .setTypes("high_card_type")
            .addAggregation(
                terms("terms").field(SINGLE_VALUED_FIELD_NAME).exclude("val0[1-9]+.+", 0))
            .execute()
            .actionGet();

    assertSearchResponse(response);

    terms = response.getAggregations().get("terms");
    assertThat(terms, notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    assertThat(terms.getBuckets().size(), equalTo(10));

    for (int i = 0; i < 10; i++) {
      Terms.Bucket bucket = terms.getBucketByKey("val00" + i);
      assertThat(bucket, notNullValue());
      assertThat(key(bucket), equalTo("val00" + i));
      assertThat(bucket.getDocCount(), equalTo(1l));
    }
  }