private Mapper parse(DocumentMapper mapper, DocumentMapperParser parser, XContentBuilder builder) throws Exception { Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); ParseContext.InternalParseContext ctx = new ParseContext.InternalParseContext("test", settings, parser, mapper, new ContentPath(0)); SourceToParse source = SourceToParse.source(builder.bytes()); ctx.reset(XContentHelper.createParser(source.source()), new ParseContext.Document(), source); assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken()); ctx.parser().nextToken(); return DocumentParser.parseObject(ctx, mapper.root()); }
public void testJoinFieldSet() throws Exception { String parentMapping = XContentFactory.jsonBuilder() .startObject() .startObject("parent_type") .endObject() .endObject() .string(); String childMapping = XContentFactory.jsonBuilder() .startObject() .startObject("child_type") .startObject("_parent") .field("type", "parent_type") .endObject() .endObject() .endObject() .string(); IndexService indexService = createIndex("test"); indexService .mapperService() .merge( "parent_type", new CompressedXContent(parentMapping), MergeReason.MAPPING_UPDATE, false); indexService .mapperService() .merge( "child_type", new CompressedXContent(childMapping), MergeReason.MAPPING_UPDATE, false); // Indexing parent doc: DocumentMapper parentDocMapper = indexService.mapperService().documentMapper("parent_type"); ParsedDocument doc = parentDocMapper.parse( SourceToParse.source("test", "parent_type", "1122", new BytesArray("{}"))); assertEquals(1, getNumberOfFieldWithParentPrefix(doc.rootDoc())); assertEquals("1122", doc.rootDoc().getBinaryValue("_parent#parent_type").utf8ToString()); // Indexing child doc: DocumentMapper childDocMapper = indexService.mapperService().documentMapper("child_type"); doc = childDocMapper.parse( SourceToParse.source("test", "child_type", "1", new BytesArray("{}")).parent("1122")); assertEquals(1, getNumberOfFieldWithParentPrefix(doc.rootDoc())); assertEquals("1122", doc.rootDoc().getBinaryValue("_parent#parent_type").utf8ToString()); }
public void testParentSetInDocNotAllowed() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .endObject() .endObject() .string(); DocumentMapper docMapper = createIndex("test") .mapperService() .documentMapperParser() .parse("type", new CompressedXContent(mapping)); try { docMapper.parse( SourceToParse.source( "test", "type", "1", XContentFactory.jsonBuilder() .startObject() .field("_parent", "1122") .endObject() .bytes())); fail("Expected failure to parse metadata field"); } catch (MapperParsingException e) { assertTrue( e.getMessage(), e.getMessage() .contains( "Field [_parent] is a metadata field and cannot be added inside a document")); } }
public void testJoinFieldNotSet() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .endObject() .endObject() .string(); DocumentMapper docMapper = createIndex("test") .mapperService() .documentMapperParser() .parse("type", new CompressedXContent(mapping)); ParsedDocument doc = docMapper.parse( SourceToParse.source( "test", "type", "1", XContentFactory.jsonBuilder() .startObject() .field("x_field", "x_value") .endObject() .bytes())); assertEquals(0, getNumberOfFieldWithParentPrefix(doc.rootDoc())); }
public ParsedDocument parse(SourceToParse source, @Nullable ParseListener listener) throws MapperParsingException { ParseContext context = cache.get(); if (source.type() != null && !source.type().equals(this.type)) { throw new MapperParsingException( "Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + this.type + "]"); } source.type(this.type); XContentParser parser = source.parser(); try { if (parser == null) { parser = XContentHelper.createParser(source.source()); } context.reset(parser, new Document(), source, listener); // on a newly created instance of document mapper, we always consider it as new mappers that // have been added if (initMappersAdded) { context.setMappingsModified(); initMappersAdded = false; } // will result in START_OBJECT int countDownTokens = 0; XContentParser.Token token = parser.nextToken(); if (token != XContentParser.Token.START_OBJECT) { throw new MapperParsingException("Malformed content, must start with an object"); } boolean emptyDoc = false; token = parser.nextToken(); if (token == XContentParser.Token.END_OBJECT) { // empty doc, we can handle it... emptyDoc = true; } else if (token != XContentParser.Token.FIELD_NAME) { throw new MapperParsingException( "Malformed content, after first object, either the type field or the actual properties should exist"); } if (type.equals(parser.currentName())) { // first field is the same as the type, this might be because the type is provided, and the // object exists within it // or because there is a valid field that by chance is named as the type // Note, in this case, we only handle plain value types, an object type will be analyzed as // if it was the type itself // and other same level fields will be ignored token = parser.nextToken(); countDownTokens++; // commented out, allow for same type with START_OBJECT, we do our best to handle it except // for the above corner case // if (token != XContentParser.Token.START_OBJECT) { // throw new MapperException("Malformed content, a field with the same // name as the type must be an object with the properties/fields within it"); // } } for (RootMapper rootMapper : rootMappersOrdered) { rootMapper.preParse(context); } if (!emptyDoc) { rootObjectMapper.parse(context); } for (int i = 0; i < countDownTokens; i++) { parser.nextToken(); } // fire up any new mappers if exists if (!context.newFieldMappers().mappers.isEmpty()) { addFieldMappers(context.newFieldMappers().mappers); context.newFieldMappers().mappers.clear(); } if (!context.newObjectMappers().mappers.isEmpty()) { addObjectMappers(context.newObjectMappers().mappers); context.newObjectMappers().mappers.clear(); } for (RootMapper rootMapper : rootMappersOrdered) { rootMapper.postParse(context); } for (RootMapper rootMapper : rootMappersOrdered) { rootMapper.validate(context); } } catch (Throwable e) { // we have to fire up any new mappers even on a failure, because they // have been added internally to each compound mapper... // ... we have no option to "rollback" a change, which is very tricky in our copy on change // system... if (!context.newFieldMappers().mappers.isEmpty()) { addFieldMappers(context.newFieldMappers().mappers); context.newFieldMappers().mappers.clear(); } if (!context.newObjectMappers().mappers.isEmpty()) { addObjectMappers(context.newObjectMappers().mappers); context.newObjectMappers().mappers.clear(); } // if its already a mapper parsing exception, no need to wrap it... if (e instanceof MapperParsingException) { throw (MapperParsingException) e; } throw new MapperParsingException("failed to parse", e); } finally { // only close the parser when its not provided externally if (source.parser() == null && parser != null) { parser.close(); } } // reverse the order of docs for nested docs support, parent should be last if (context.docs().size() > 1) { Collections.reverse(context.docs()); } // apply doc boost if (context.docBoost() != 1.0f) { Set<String> encounteredFields = Sets.newHashSet(); for (Document doc : context.docs()) { encounteredFields.clear(); for (IndexableField field : doc) { if (field.fieldType().indexed() && !field.fieldType().omitNorms()) { if (!encounteredFields.contains(field.name())) { ((Field) field).setBoost(context.docBoost() * field.boost()); encounteredFields.add(field.name()); } } } } } ParsedDocument doc = new ParsedDocument( context.uid(), context.id(), context.type(), source.routing(), source.timestamp(), source.ttl(), context.docs(), context.analyzer(), context.source(), context.mappingsModified()) .parent(source.parent()); // reset the context to free up memory context.reset(null, null, null, null); return doc; }
public ParsedDocument parse(String type, String id, BytesReference source) throws MapperParsingException { return parse(SourceToParse.source(source).type(type).id(id)); }
public ParsedDocument parse(BytesReference source) throws MapperParsingException { return parse(SourceToParse.source(source)); }
private ParsedDocument innerParseDocument(SourceToParse source) throws MapperParsingException { if (docMapper.type().equals(MapperService.DEFAULT_MAPPING)) { throw new IllegalArgumentException( "It is forbidden to index into the default mapping [" + MapperService.DEFAULT_MAPPING + "]"); } ParseContext.InternalParseContext context = cache.get(); final Mapping mapping = docMapper.mapping(); if (source.type() != null && !source.type().equals(docMapper.type())) { throw new MapperParsingException( "Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + docMapper.type() + "]"); } source.type(docMapper.type()); XContentParser parser = source.parser(); try { if (parser == null) { parser = XContentHelper.createParser(source.source()); } context.reset(parser, new ParseContext.Document(), source); // will result in START_OBJECT XContentParser.Token token = parser.nextToken(); if (token != XContentParser.Token.START_OBJECT) { throw new MapperParsingException("Malformed content, must start with an object"); } boolean emptyDoc = false; if (mapping.root.isEnabled()) { token = parser.nextToken(); if (token == XContentParser.Token.END_OBJECT) { // empty doc, we can handle it... emptyDoc = true; } else if (token != XContentParser.Token.FIELD_NAME) { throw new MapperParsingException( "Malformed content, after first object, either the type field or the actual properties should exist"); } } for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) { metadataMapper.preParse(context); } if (mapping.root.isEnabled() == false) { // entire type is disabled parser.skipChildren(); } else if (emptyDoc == false) { Mapper update = parseObject(context, mapping.root, true); if (update != null) { context.addDynamicMappingsUpdate(update); } } for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) { metadataMapper.postParse(context); } // try to parse the next token, this should be null if the object is ended properly // but will throw a JSON exception if the extra tokens is not valid JSON (this will be handled // by the catch) if (Version.indexCreated(indexSettings).onOrAfter(Version.V_2_0_0_beta1) && source.parser() == null && parser != null) { // only check for end of tokens if we created the parser here token = parser.nextToken(); if (token != null) { throw new IllegalArgumentException( "Malformed content, found extra data after parsing: " + token); } } } catch (Throwable e) { // if its already a mapper parsing exception, no need to wrap it... if (e instanceof MapperParsingException) { throw (MapperParsingException) e; } // Throw a more meaningful message if the document is empty. if (source.source() != null && source.source().length() == 0) { throw new MapperParsingException("failed to parse, document is empty"); } throw new MapperParsingException("failed to parse", e); } finally { // only close the parser when its not provided externally if (source.parser() == null && parser != null) { parser.close(); } } // reverse the order of docs for nested docs support, parent should be last if (context.docs().size() > 1) { Collections.reverse(context.docs()); } // apply doc boost if (context.docBoost() != 1.0f) { Set<String> encounteredFields = new HashSet<>(); for (ParseContext.Document doc : context.docs()) { encounteredFields.clear(); for (IndexableField field : doc) { if (field.fieldType().indexOptions() != IndexOptions.NONE && !field.fieldType().omitNorms()) { if (!encounteredFields.contains(field.name())) { ((Field) field).setBoost(context.docBoost() * field.boost()); encounteredFields.add(field.name()); } } } } } Mapper rootDynamicUpdate = context.dynamicMappingsUpdate(); Mapping update = null; if (rootDynamicUpdate != null) { update = mapping.mappingUpdate(rootDynamicUpdate); } ParsedDocument doc = new ParsedDocument( context.uid(), context.version(), context.id(), context.type(), source.routing(), source.timestamp(), source.ttl(), context.docs(), context.source(), update) .parent(source.parent()); // reset the context to free up memory context.reset(null, null, null); return doc; }
public void testMixTemplateMultiFieldAndMappingReuse() throws Exception { IndexService indexService = createIndex("test"); XContentBuilder mappings1 = jsonBuilder() .startObject() .startObject("type1") .startArray("dynamic_templates") .startObject() .startObject("template1") .field("match_mapping_type", "string") .startObject("mapping") .field("type", "text") .startObject("fields") .startObject("raw") .field("type", "keyword") .endObject() .endObject() .endObject() .endObject() .endObject() .endArray() .endObject() .endObject(); indexService .mapperService() .merge( "type1", new CompressedXContent(mappings1.bytes()), MapperService.MergeReason.MAPPING_UPDATE, false); XContentBuilder mappings2 = jsonBuilder() .startObject() .startObject("type2") .startObject("properties") .startObject("field") .field("type", "text") .endObject() .endObject() .endObject() .endObject(); indexService .mapperService() .merge( "type2", new CompressedXContent(mappings2.bytes()), MapperService.MergeReason.MAPPING_UPDATE, false); XContentBuilder json = XContentFactory.jsonBuilder().startObject().field("field", "foo").endObject(); SourceToParse source = SourceToParse.source(json.bytes()).id("1"); DocumentMapper mapper = indexService.mapperService().documentMapper("type1"); assertNull(mapper.mappers().getMapper("field.raw")); ParsedDocument parsed = mapper.parse(source); assertNotNull(parsed.dynamicMappingsUpdate()); indexService .mapperService() .merge( "type1", new CompressedXContent(parsed.dynamicMappingsUpdate().toString()), MapperService.MergeReason.MAPPING_UPDATE, false); mapper = indexService.mapperService().documentMapper("type1"); assertNotNull(mapper.mappers().getMapper("field.raw")); parsed = mapper.parse(source); assertNull(parsed.dynamicMappingsUpdate()); }