/** * Generic test that creates new AggregatorFactory from the test AggregatorFactory and checks both * for equality and asserts equality on the two queries. */ public void testFromXContent() throws IOException { AF testAgg = createTestAggregatorFactory(); AggregatorFactories.Builder factoriesBuilder = AggregatorFactories.builder().skipResolveOrder().addPipelineAggregator(testAgg); logger.info("Content string: {}", factoriesBuilder); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } factoriesBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); XContentBuilder shuffled = shuffleXContent(builder); XContentParser parser = XContentFactory.xContent(shuffled.bytes()).createParser(shuffled.bytes()); QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher); String contentString = factoriesBuilder.toString(); logger.info("Content string: {}", contentString); assertSame(XContentParser.Token.START_OBJECT, parser.nextToken()); assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken()); assertEquals(testAgg.getName(), parser.currentName()); assertSame(XContentParser.Token.START_OBJECT, parser.nextToken()); assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken()); assertEquals(testAgg.type(), parser.currentName()); assertSame(XContentParser.Token.START_OBJECT, parser.nextToken()); PipelineAggregationBuilder newAgg = aggParsers .pipelineParser(testAgg.getWriteableName(), ParseFieldMatcher.STRICT) .parse(testAgg.getName(), parseContext); assertSame(XContentParser.Token.END_OBJECT, parser.currentToken()); assertSame(XContentParser.Token.END_OBJECT, parser.nextToken()); assertSame(XContentParser.Token.END_OBJECT, parser.nextToken()); assertNull(parser.nextToken()); assertNotNull(newAgg); assertNotSame(newAgg, testAgg); assertEquals(testAgg, newAgg); assertEquals(testAgg.hashCode(), newAgg.hashCode()); }
private AggregatorFactories parseAggregators( XContentParser parser, SearchContext context, int level) throws IOException { Matcher validAggMatcher = VALID_AGG_NAME.matcher(""); AggregatorFactories.Builder factories = new AggregatorFactories.Builder(); XContentParser.Token token = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token != XContentParser.Token.FIELD_NAME) { throw new SearchParseException( context, "Unexpected token " + token + " in [aggs]: aggregations definitions must start with the name of the aggregation."); } final String aggregationName = parser.currentName(); if (!validAggMatcher.reset(aggregationName).matches()) { throw new SearchParseException( context, "Invalid aggregation name [" + aggregationName + "]. Aggregation names must be alpha-numeric and can only contain '_' and '-'"); } token = parser.nextToken(); if (token != XContentParser.Token.START_OBJECT) { throw new SearchParseException( context, "Aggregation definition for [" + aggregationName + " starts with a [" + token + "], expected a [" + XContentParser.Token.START_OBJECT + "]."); } AggregatorFactory factory = null; AggregatorFactories subFactories = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token != XContentParser.Token.FIELD_NAME) { throw new SearchParseException( context, "Expected [" + XContentParser.Token.FIELD_NAME + "] under a [" + XContentParser.Token.START_OBJECT + "], but got a [" + token + "] in [" + aggregationName + "]"); } final String fieldName = parser.currentName(); token = parser.nextToken(); if ("aggregations_binary".equals(fieldName)) { if (subFactories != null) { throw new SearchParseException( context, "Found two sub aggregation definitions under [" + aggregationName + "]"); } XContentParser binaryParser = null; if (token == XContentParser.Token.VALUE_STRING || token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { byte[] source = parser.binaryValue(); binaryParser = XContentFactory.xContent(source).createParser(source); } else { throw new SearchParseException( context, "Expected [" + XContentParser.Token.VALUE_STRING + " or " + XContentParser.Token.VALUE_EMBEDDED_OBJECT + "] for [" + fieldName + "], but got a [" + token + "] in [" + aggregationName + "]"); } XContentParser.Token binaryToken = binaryParser.nextToken(); if (binaryToken != XContentParser.Token.START_OBJECT) { throw new SearchParseException( context, "Expected [" + XContentParser.Token.START_OBJECT + "] as first token when parsing [" + fieldName + "], but got a [" + binaryToken + "] in [" + aggregationName + "]"); } subFactories = parseAggregators(binaryParser, context, level + 1); } else if (token == XContentParser.Token.START_OBJECT) { switch (fieldName) { case "aggregations": case "aggs": if (subFactories != null) { throw new SearchParseException( context, "Found two sub aggregation definitions under [" + aggregationName + "]"); } subFactories = parseAggregators(parser, context, level + 1); break; default: if (factory != null) { throw new SearchParseException( context, "Found two aggregation type definitions in [" + aggregationName + "]: [" + factory.type + "] and [" + fieldName + "]"); } Aggregator.Parser aggregatorParser = parser(fieldName); if (aggregatorParser == null) { throw new SearchParseException( context, "Could not find aggregator type [" + fieldName + "] in [" + aggregationName + "]"); } factory = aggregatorParser.parse(aggregationName, parser, context); } } else { throw new SearchParseException( context, "Expected [" + XContentParser.Token.START_OBJECT + "] under [" + fieldName + "], but got a [" + token + "] in [" + aggregationName + "]"); } } if (factory == null) { throw new SearchParseException( context, "Missing definition for aggregation [" + aggregationName + "]"); } if (subFactories != null) { factory.subFactories(subFactories); } if (level == 0) { factory.validate(); } factories.add(factory); } return factories.build(); }
public void testResetRootDocId() throws Exception { Directory directory = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(null); iwc.setMergePolicy(NoMergePolicy.INSTANCE); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory, iwc); List<Document> documents = new ArrayList<>(); // 1 segment with, 1 root document, with 3 nested sub docs Document document = new Document(); document.add( new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add( new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); document.add( new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add( new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); document.add( new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add( new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); document.add(new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); indexWriter.addDocuments(documents); indexWriter.commit(); documents.clear(); // 1 segment with: // 1 document, with 1 nested subdoc document = new Document(); document.add( new Field(UidFieldMapper.NAME, "type#2", UidFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add( new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); document.add(new Field(UidFieldMapper.NAME, "type#2", UidFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); indexWriter.addDocuments(documents); documents.clear(); // and 1 document, with 1 nested subdoc document = new Document(); document.add( new Field(UidFieldMapper.NAME, "type#3", UidFieldMapper.Defaults.NESTED_FIELD_TYPE)); document.add( new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); document = new Document(); document.add(new Field(UidFieldMapper.NAME, "type#3", UidFieldMapper.Defaults.FIELD_TYPE)); document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE)); documents.add(document); indexWriter.addDocuments(documents); indexWriter.commit(); indexWriter.close(); IndexService indexService = createIndex("test"); DirectoryReader directoryReader = DirectoryReader.open(directory); directoryReader = ElasticsearchDirectoryReader.wrap(directoryReader, new ShardId(indexService.index(), 0)); IndexSearcher searcher = new IndexSearcher(directoryReader); indexService .mapperService() .merge( "test", new CompressedXContent( PutMappingRequest.buildFromSimplifiedDef("test", "nested_field", "type=nested") .string()), MapperService.MergeReason.MAPPING_UPDATE, false); SearchContext searchContext = createSearchContext(indexService); AggregationContext context = new AggregationContext(searchContext); AggregatorFactories.Builder builder = AggregatorFactories.builder(); NestedAggregatorBuilder factory = new NestedAggregatorBuilder("test", "nested_field"); builder.addAggregator(factory); AggregatorFactories factories = builder.build(context, null); searchContext.aggregations(new SearchContextAggregations(factories)); Aggregator[] aggs = factories.createTopLevelAggregators(); BucketCollector collector = BucketCollector.wrap(Arrays.asList(aggs)); collector.preCollection(); // A regular search always exclude nested docs, so we use NonNestedDocsFilter.INSTANCE here // (otherwise MatchAllDocsQuery would be sufficient) // We exclude root doc with uid type#2, this will trigger the bug if we don't reset the root doc // when we process a new segment, because // root doc type#3 and root doc type#1 have the same segment docid BooleanQuery.Builder bq = new BooleanQuery.Builder(); bq.add(Queries.newNonNestedFilter(), Occur.MUST); bq.add(new TermQuery(new Term(UidFieldMapper.NAME, "type#2")), Occur.MUST_NOT); searcher.search(new ConstantScoreQuery(bq.build()), collector); collector.postCollection(); Nested nested = (Nested) aggs[0].buildAggregation(0); // The bug manifests if 6 docs are returned, because currentRootDoc isn't reset the previous // child docs from the first segment are emitted as hits. assertThat(nested.getDocCount(), equalTo(4L)); directoryReader.close(); directory.close(); }