@Test
  public void testSimpleSubAggregations() throws Exception {
    final String query =
        String.format(
            "SELECT /*! DOCS_WITH_AGGREGATION(10) */ * FROM %s/account GROUP BY (gender), (state) ",
            TEST_INDEX);

    SqlElasticSearchRequestBuilder select = getSearchRequestBuilder(query);
    SearchResponse response = (SearchResponse) select.get();
    Aggregations result = response.getAggregations();

    Terms gender = result.get("gender");
    for (Terms.Bucket genderBucket : gender.getBuckets()) {
      String genderKey = genderBucket.getKey();
      Assert.assertTrue("Gender should be m or f", genderKey.equals("m") || genderKey.equals("f"));
    }

    Assert.assertEquals(2, gender.getBuckets().size());

    Terms state = result.get("state");
    for (Terms.Bucket stateBucket : state.getBuckets()) {
      if (stateBucket.getKey().equalsIgnoreCase("ak")) {
        Assert.assertTrue("There are 22 entries for state ak", stateBucket.getDocCount() == 22);
      }
    }

    Assert.assertEquals(response.getHits().totalHits(), 1000);
    Assert.assertEquals(response.getHits().hits().length, 10);
  }
Example #2
0
  @Test
  public void testChildrenAggs() throws Exception {
    SearchResponse searchResponse =
        client()
            .prepareSearch("test")
            .setQuery(matchQuery("randomized", true))
            .addAggregation(
                terms("category")
                    .field("category")
                    .size(0)
                    .subAggregation(
                        children("to_comment")
                            .childType("comment")
                            .subAggregation(
                                terms("commenters")
                                    .field("commenter")
                                    .size(0)
                                    .subAggregation(topHits("top_comments")))))
            .get();
    assertSearchResponse(searchResponse);

    Terms categoryTerms = searchResponse.getAggregations().get("category");
    assertThat(categoryTerms.getBuckets().size(), equalTo(categoryToControl.size()));
    for (Map.Entry<String, Control> entry1 : categoryToControl.entrySet()) {
      Terms.Bucket categoryBucket = categoryTerms.getBucketByKey(entry1.getKey());
      assertThat(categoryBucket.getKey(), equalTo(entry1.getKey()));
      assertThat(categoryBucket.getDocCount(), equalTo((long) entry1.getValue().articleIds.size()));

      Children childrenBucket = categoryBucket.getAggregations().get("to_comment");
      assertThat(childrenBucket.getName(), equalTo("to_comment"));
      assertThat(childrenBucket.getDocCount(), equalTo((long) entry1.getValue().commentIds.size()));
      assertThat(
          (long) childrenBucket.getProperty("_count"),
          equalTo((long) entry1.getValue().commentIds.size()));

      Terms commentersTerms = childrenBucket.getAggregations().get("commenters");
      assertThat((Terms) childrenBucket.getProperty("commenters"), sameInstance(commentersTerms));
      assertThat(
          commentersTerms.getBuckets().size(),
          equalTo(entry1.getValue().commenterToCommentId.size()));
      for (Map.Entry<String, Set<String>> entry2 :
          entry1.getValue().commenterToCommentId.entrySet()) {
        Terms.Bucket commentBucket = commentersTerms.getBucketByKey(entry2.getKey());
        assertThat(commentBucket.getKey(), equalTo(entry2.getKey()));
        assertThat(commentBucket.getDocCount(), equalTo((long) entry2.getValue().size()));

        TopHits topHits = commentBucket.getAggregations().get("top_comments");
        for (SearchHit searchHit : topHits.getHits().getHits()) {
          assertThat(entry2.getValue().contains(searchHit.getId()), is(true));
        }
      }
    }
  }
  @Test
  public void reverseAnotherNestedGroupByOnNestedFieldWithFilterTestWithReverseNestedNoPath()
      throws Exception {
    Aggregations result =
        query(
            String.format(
                "SELECT COUNT(*) FROM %s/nestedType GROUP BY  nested(message.info),filter('myFilter',message.info = 'a'),reverse_nested(comment.data,'~comment')",
                TEST_INDEX));
    InternalNested nested = result.get("message.info@NESTED");
    InternalFilter filter = nested.getAggregations().get("myFilter@FILTER");
    Terms infos = filter.getAggregations().get("message.info");
    Assert.assertEquals(1, infos.getBuckets().size());
    for (Terms.Bucket bucket : infos.getBuckets()) {
      InternalReverseNested reverseNested =
          bucket.getAggregations().get("comment.data@NESTED_REVERSED");
      InternalNested innerNested = reverseNested.getAggregations().get("comment.data@NESTED");
      Terms terms = innerNested.getAggregations().get("comment.data");
      Terms.Bucket internalBucket = terms.getBuckets().get(0);

      long count = ((ValueCount) internalBucket.getAggregations().get("COUNT(*)")).getValue();
      String key = internalBucket.getKey();
      if (key.equalsIgnoreCase("ab")) {
        Assert.assertEquals(2, count);
      } else {
        throw new Exception(String.format("Unexpected key. expected: only a . found: %s", key));
      }
    }
  }
Example #4
0
  public Map<String, Long> countByField(IndexField indexField, FilterBuilder filter) {
    Map<String, Long> counts = new HashMap<>();

    SearchRequestBuilder request =
        client
            .prepareSearch(this.getIndexName())
            .setTypes(this.getIndexType())
            .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), filter))
            .setSize(0)
            .addAggregation(
                AggregationBuilders.terms(indexField.field())
                    .field(indexField.field())
                    .order(Terms.Order.count(false))
                    .size(Integer.MAX_VALUE)
                    .minDocCount(0));

    SearchResponse response = request.get();

    Terms values = response.getAggregations().get(indexField.field());

    for (Terms.Bucket value : values.getBuckets()) {
      counts.put(value.getKey(), value.getDocCount());
    }
    return counts;
  }
  @Test
  public void testDocCount_asSubAgg() throws Exception {
    SearchResponse response =
        client()
            .prepareSearch("idx")
            .addAggregation(
                terms("terms")
                    .field("tag")
                    .order(Order.term(true))
                    .subAggregation(
                        histogram("histo")
                            .field(SINGLE_VALUED_FIELD_NAME)
                            .interval(interval)
                            .extendedBounds((long) minRandomValue, (long) maxRandomValue))
                    .subAggregation(maxBucket("max_bucket").setBucketsPaths("histo>_count")))
            .execute()
            .actionGet();

    assertSearchResponse(response);

    Terms terms = response.getAggregations().get("terms");
    assertThat(terms, notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    List<Terms.Bucket> termsBuckets = terms.getBuckets();
    assertThat(termsBuckets.size(), equalTo(interval));

    for (int i = 0; i < interval; ++i) {
      Terms.Bucket termsBucket = termsBuckets.get(i);
      assertThat(termsBucket, notNullValue());
      assertThat((String) termsBucket.getKey(), equalTo("tag" + (i % interval)));

      Histogram histo = termsBucket.getAggregations().get("histo");
      assertThat(histo, notNullValue());
      assertThat(histo.getName(), equalTo("histo"));
      List<? extends Bucket> buckets = histo.getBuckets();

      List<String> maxKeys = new ArrayList<>();
      double maxValue = Double.NEGATIVE_INFINITY;
      for (int j = 0; j < numValueBuckets; ++j) {
        Histogram.Bucket bucket = buckets.get(j);
        assertThat(bucket, notNullValue());
        assertThat(((Number) bucket.getKey()).longValue(), equalTo((long) j * interval));
        if (bucket.getDocCount() > maxValue) {
          maxValue = bucket.getDocCount();
          maxKeys = new ArrayList<>();
          maxKeys.add(bucket.getKeyAsString());
        } else if (bucket.getDocCount() == maxValue) {
          maxKeys.add(bucket.getKeyAsString());
        }
      }

      InternalBucketMetricValue maxBucketValue = termsBucket.getAggregations().get("max_bucket");
      assertThat(maxBucketValue, notNullValue());
      assertThat(maxBucketValue.getName(), equalTo("max_bucket"));
      assertThat(maxBucketValue.value(), equalTo(maxValue));
      assertThat(maxBucketValue.keys(), equalTo(maxKeys.toArray(new String[maxKeys.size()])));
    }
  }
  @Test
  public void testSubAggregations() throws Exception {
    Set expectedAges =
        new HashSet<>(ContiguousSet.create(Range.closed(20, 40), DiscreteDomain.integers()));
    final String query =
        String.format(
            "SELECT /*! DOCS_WITH_AGGREGATION(10) */"
                + " * FROM %s/account GROUP BY (gender, age), (state) LIMIT 0,10",
            TEST_INDEX);

    Map<String, Set<Integer>> buckets = new HashMap<>();

    SqlElasticSearchRequestBuilder select = getSearchRequestBuilder(query);
    SearchResponse response = (SearchResponse) select.get();
    Aggregations result = response.getAggregations();

    Terms gender = result.get("gender");
    for (Terms.Bucket genderBucket : gender.getBuckets()) {
      String genderKey = genderBucket.getKey();
      buckets.put(genderKey, new HashSet<Integer>());
      Terms ageBuckets = (Terms) genderBucket.getAggregations().get("age");
      for (Terms.Bucket ageBucket : ageBuckets.getBuckets()) {
        buckets.get(genderKey).add(Integer.parseInt(ageBucket.getKey()));
      }
    }

    Assert.assertEquals(2, buckets.keySet().size());
    Assert.assertEquals(expectedAges, buckets.get("m"));
    Assert.assertEquals(expectedAges, buckets.get("f"));

    Terms state = result.get("state");
    for (Terms.Bucket stateBucket : state.getBuckets()) {
      if (stateBucket.getKey().equalsIgnoreCase("ak")) {
        Assert.assertTrue("There are 22 entries for state ak", stateBucket.getDocCount() == 22);
      }
    }

    Assert.assertEquals(response.getHits().totalHits(), 1000);
    Assert.assertEquals(response.getHits().hits().length, 10);
  }
  @Test
  public void multipleGroupByTest() throws Exception {
    Set expectedAges =
        new HashSet<Integer>(ContiguousSet.create(Range.closed(20, 40), DiscreteDomain.integers()));

    Map<String, Set<Integer>> buckets = new HashMap<>();

    Aggregations result =
        query(String.format("SELECT COUNT(*) FROM %s/account GROUP BY gender, age", TEST_INDEX));
    Terms gender = result.get("gender");
    for (Terms.Bucket genderBucket : gender.getBuckets()) {
      String genderKey = genderBucket.getKey();
      buckets.put(genderKey, new HashSet<Integer>());
      Terms ageBuckets = (Terms) genderBucket.getAggregations().get("age");
      for (Terms.Bucket ageBucket : ageBuckets.getBuckets()) {
        buckets.get(genderKey).add(Integer.parseInt(ageBucket.getKey()));
      }
    }

    Assert.assertEquals(2, buckets.keySet().size());
    Assert.assertEquals(expectedAges, buckets.get("m"));
    Assert.assertEquals(expectedAges, buckets.get("f"));
  }
 /*
  * 所有姓氏为"Smith"的员工的最大的共同点(兴趣爱好)是什么
  * 聚合变成只包含和查询语句想匹配的文档
  */
 @Test
 public void queryAggregationTest() {
   Client client = ClientTemplate.getClient();
   SearchResponse response =
       client
           .prepareSearch("fccs")
           .setTypes("employee")
           .setQuery(QueryBuilders.matchQuery("lastName", "Smith"))
           .addAggregation(AggregationBuilders.terms("by_interests").field("interests"))
           .get();
   Terms terms = response.getAggregations().get("by_interests");
   for (Terms.Bucket bucket : terms.getBuckets()) {
     System.out.println("key:" + bucket.getKey() + " doc_count:" + bucket.getDocCount());
   }
   client.close();
 }
 @Test
 public void groupByTest() throws Exception {
   Aggregations result =
       query(String.format("SELECT COUNT(*) FROM %s/account GROUP BY gender", TEST_INDEX));
   Terms gender = result.get("gender");
   for (Terms.Bucket bucket : gender.getBuckets()) {
     String key = bucket.getKey();
     long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue();
     if (key.equalsIgnoreCase("m")) {
       Assert.assertEquals(507, count);
     } else if (key.equalsIgnoreCase("f")) {
       Assert.assertEquals(493, count);
     } else {
       throw new Exception(String.format("Unexpected key. expected: m OR f. found: %s", key));
     }
   }
 }
  @Test
  public void testMetric_topLevel() throws Exception {
    SearchResponse response =
        client()
            .prepareSearch("idx")
            .addAggregation(
                terms("terms")
                    .field("tag")
                    .subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME)))
            .addAggregation(maxBucket("max_bucket").setBucketsPaths("terms>sum"))
            .execute()
            .actionGet();

    assertSearchResponse(response);

    Terms terms = response.getAggregations().get("terms");
    assertThat(terms, notNullValue());
    assertThat(terms.getName(), equalTo("terms"));
    List<Terms.Bucket> buckets = terms.getBuckets();
    assertThat(buckets.size(), equalTo(interval));

    List<String> maxKeys = new ArrayList<>();
    double maxValue = Double.NEGATIVE_INFINITY;
    for (int i = 0; i < interval; ++i) {
      Terms.Bucket bucket = buckets.get(i);
      assertThat(bucket, notNullValue());
      assertThat((String) bucket.getKey(), equalTo("tag" + (i % interval)));
      assertThat(bucket.getDocCount(), greaterThan(0l));
      Sum sum = bucket.getAggregations().get("sum");
      assertThat(sum, notNullValue());
      if (sum.value() > maxValue) {
        maxValue = sum.value();
        maxKeys = new ArrayList<>();
        maxKeys.add(bucket.getKeyAsString());
      } else if (sum.value() == maxValue) {
        maxKeys.add(bucket.getKeyAsString());
      }
    }

    InternalBucketMetricValue maxBucketValue = response.getAggregations().get("max_bucket");
    assertThat(maxBucketValue, notNullValue());
    assertThat(maxBucketValue.getName(), equalTo("max_bucket"));
    assertThat(maxBucketValue.value(), equalTo(maxValue));
    assertThat(maxBucketValue.keys(), equalTo(maxKeys.toArray(new String[maxKeys.size()])));
  }
Example #11
0
 // Response helpers
 protected Multimap<String, FacetValue> processAggregations(Aggregations aggregations) {
   Multimap<String, FacetValue> stats = ArrayListMultimap.create();
   if (aggregations != null) {
     for (Aggregation aggregation : aggregations.asList()) {
       if (aggregation instanceof StringTerms) {
         for (Terms.Bucket value : ((Terms) aggregation).getBuckets()) {
           FacetValue facetValue = new FacetValue(value.getKey(), value.getDocCount());
           stats.put(aggregation.getName(), facetValue);
         }
       } else if (aggregation instanceof InternalValueCount) {
         InternalValueCount count = (InternalValueCount) aggregation;
         FacetValue facetValue = new FacetValue(count.getName(), count.getValue());
         stats.put(count.getName(), facetValue);
       }
     }
   }
   return stats;
 }
  public Map<QualityProfileKey, Multimap<String, FacetValue>> getStatsByProfileKey(
      List<QualityProfileKey> keys) {

    String[] stringKeys = new String[keys.size()];
    for (int i = 0; i < keys.size(); i++) {
      stringKeys[i] = keys.get(i).toString();
    }

    SearchResponse response =
        getClient()
            .prepareSearch(this.getIndexName())
            .setQuery(
                QueryBuilders.filteredQuery(
                    QueryBuilders.matchAllQuery(),
                    FilterBuilders.termsFilter(
                        ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field(), stringKeys)))
            .addAggregation(
                AggregationBuilders.terms(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field())
                    .field(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field())
                    .subAggregation(
                        AggregationBuilders.terms(
                                ActiveRuleNormalizer.ActiveRuleField.INHERITANCE.field())
                            .field(ActiveRuleNormalizer.ActiveRuleField.INHERITANCE.field()))
                    .subAggregation(
                        AggregationBuilders.terms(
                                ActiveRuleNormalizer.ActiveRuleField.SEVERITY.field())
                            .field(ActiveRuleNormalizer.ActiveRuleField.SEVERITY.field())))
            .setSize(0)
            .setTypes(this.getIndexType())
            .get();

    Map<QualityProfileKey, Multimap<String, FacetValue>> stats =
        new HashMap<QualityProfileKey, Multimap<String, FacetValue>>();
    Aggregation aggregation =
        response.getAggregations().get(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field());
    for (Terms.Bucket value : ((Terms) aggregation).getBuckets()) {
      stats.put(
          QualityProfileKey.parse(value.getKey()),
          this.processAggregations(value.getAggregations()));
    }

    return stats;
  }
 @Test
 public void groupByOnNestedFieldWithFilterTest() throws Exception {
   Aggregations result =
       query(
           String.format(
               "SELECT COUNT(*) FROM %s/nestedType GROUP BY  nested(message.info),filter('myFilter',message.info = 'a')",
               TEST_INDEX));
   InternalNested nested = result.get("message.info@NESTED");
   InternalFilter filter = nested.getAggregations().get("myFilter@FILTER");
   Terms infos = filter.getAggregations().get("message.info");
   Assert.assertEquals(1, infos.getBuckets().size());
   for (Terms.Bucket bucket : infos.getBuckets()) {
     String key = bucket.getKey();
     long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue();
     if (key.equalsIgnoreCase("a")) {
       Assert.assertEquals(2, count);
     } else {
       throw new Exception(String.format("Unexpected key. expected: only a . found: %s", key));
     }
   }
 }
  @Test
  public void multipleGroupBysWithSize() throws Exception {
    Set expectedAges =
        new HashSet<Integer>(ContiguousSet.create(Range.closed(20, 40), DiscreteDomain.integers()));

    Map<String, Set<Integer>> buckets = new HashMap<>();

    Aggregations result =
        query(
            String.format(
                "SELECT COUNT(*) FROM %s/account GROUP BY gender, terms('alias'='ageAgg','field'='age','size'=3)",
                TEST_INDEX));
    Terms gender = result.get("gender");
    Assert.assertEquals(2, gender.getBuckets().size());
    for (Terms.Bucket genderBucket : gender.getBuckets()) {

      String genderKey = genderBucket.getKey();
      buckets.put(genderKey, new HashSet<Integer>());
      Terms ageBuckets = genderBucket.getAggregations().get("ageAgg");
      Assert.assertEquals(3, ageBuckets.getBuckets().size());
    }
  }
 @Override
 public Map<String, Integer> countTermsByField(SearchCriteria searchCriteria, String field) {
   ElasticSearchCriteria criteria = (ElasticSearchCriteria) searchCriteria;
   if (criteria == null) return null;
   SearchRequestBuilder srb = criteria2builder(criteria);
   srb.setFrom(0);
   srb.setSize(0);
   TermsBuilder tb = AggregationBuilders.terms(field);
   tb.field(field);
   srb.addAggregation(tb);
   try {
     SearchResponse response = srb.execute().get();
     StringTerms aggr = response.getAggregations().get(field);
     Map<String, Integer> result = new LinkedHashMap<>();
     for (Terms.Bucket bucket : aggr.getBuckets()) {
       result.put(bucket.getKey(), (int) bucket.getDocCount());
     }
     return result;
   } catch (Exception e) {
     logger.error(e.getMessage(), e);
   }
   return Collections.emptyMap();
 }
 private String key(Terms.Bucket bucket) {
   return randomBoolean() ? bucket.getKey() : bucket.getKeyAsText().string();
 }
Example #17
0
  @Test
  public void testParentWithMultipleBuckets() throws Exception {
    SearchResponse searchResponse =
        client()
            .prepareSearch("test")
            .setQuery(matchQuery("randomized", false))
            .addAggregation(
                terms("category")
                    .field("category")
                    .size(0)
                    .subAggregation(
                        children("to_comment")
                            .childType("comment")
                            .subAggregation(topHits("top_comments").addSort("_id", SortOrder.ASC))))
            .get();
    assertSearchResponse(searchResponse);

    Terms categoryTerms = searchResponse.getAggregations().get("category");
    assertThat(categoryTerms.getBuckets().size(), equalTo(3));

    for (Terms.Bucket bucket : categoryTerms.getBuckets()) {
      logger.info("bucket=" + bucket.getKey());
      Children childrenBucket = bucket.getAggregations().get("to_comment");
      TopHits topHits = childrenBucket.getAggregations().get("top_comments");
      logger.info("total_hits={}", topHits.getHits().getTotalHits());
      for (SearchHit searchHit : topHits.getHits()) {
        logger.info(
            "hit= {} {} {}", searchHit.sortValues()[0], searchHit.getType(), searchHit.getId());
      }
    }

    Terms.Bucket categoryBucket = categoryTerms.getBucketByKey("a");
    assertThat(categoryBucket.getKey(), equalTo("a"));
    assertThat(categoryBucket.getDocCount(), equalTo(3l));

    Children childrenBucket = categoryBucket.getAggregations().get("to_comment");
    assertThat(childrenBucket.getName(), equalTo("to_comment"));
    assertThat(childrenBucket.getDocCount(), equalTo(2l));
    TopHits topHits = childrenBucket.getAggregations().get("top_comments");
    assertThat(topHits.getHits().totalHits(), equalTo(2l));
    assertThat(topHits.getHits().getAt(0).sortValues()[0].toString(), equalTo("a"));
    assertThat(topHits.getHits().getAt(0).getId(), equalTo("a"));
    assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
    assertThat(topHits.getHits().getAt(1).sortValues()[0].toString(), equalTo("c"));
    assertThat(topHits.getHits().getAt(1).getId(), equalTo("c"));
    assertThat(topHits.getHits().getAt(1).getType(), equalTo("comment"));

    categoryBucket = categoryTerms.getBucketByKey("b");
    assertThat(categoryBucket.getKey(), equalTo("b"));
    assertThat(categoryBucket.getDocCount(), equalTo(2l));

    childrenBucket = categoryBucket.getAggregations().get("to_comment");
    assertThat(childrenBucket.getName(), equalTo("to_comment"));
    assertThat(childrenBucket.getDocCount(), equalTo(1l));
    topHits = childrenBucket.getAggregations().get("top_comments");
    assertThat(topHits.getHits().totalHits(), equalTo(1l));
    assertThat(topHits.getHits().getAt(0).getId(), equalTo("c"));
    assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));

    categoryBucket = categoryTerms.getBucketByKey("c");
    assertThat(categoryBucket.getKey(), equalTo("c"));
    assertThat(categoryBucket.getDocCount(), equalTo(2l));

    childrenBucket = categoryBucket.getAggregations().get("to_comment");
    assertThat(childrenBucket.getName(), equalTo("to_comment"));
    assertThat(childrenBucket.getDocCount(), equalTo(1l));
    topHits = childrenBucket.getAggregations().get("top_comments");
    assertThat(topHits.getHits().totalHits(), equalTo(1l));
    assertThat(topHits.getHits().getAt(0).getId(), equalTo("c"));
    assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment"));
  }