private Mapper parse(DocumentMapper mapper, DocumentMapperParser parser, XContentBuilder builder)
throws Exception {
Settings settings =
Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
ParseContext.InternalParseContext ctx =
new ParseContext.InternalParseContext("test", settings, parser, mapper, new ContentPath(0));
SourceToParse source = SourceToParse.source(builder.bytes());
ctx.reset(XContentHelper.createParser(source.source()), new ParseContext.Document(), source);
assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken());
ctx.parser().nextToken();
return DocumentParser.parseObject(ctx, mapper.root());
}
public void testJoinFieldSet() throws Exception {
String parentMapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("parent_type")
.endObject()
.endObject()
.string();
String childMapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("child_type")
.startObject("_parent")
.field("type", "parent_type")
.endObject()
.endObject()
.endObject()
.string();
IndexService indexService = createIndex("test");
indexService
.mapperService()
.merge(
"parent_type",
new CompressedXContent(parentMapping),
MergeReason.MAPPING_UPDATE,
false);
indexService
.mapperService()
.merge(
"child_type", new CompressedXContent(childMapping), MergeReason.MAPPING_UPDATE, false);
// Indexing parent doc:
DocumentMapper parentDocMapper = indexService.mapperService().documentMapper("parent_type");
ParsedDocument doc =
parentDocMapper.parse(
SourceToParse.source("test", "parent_type", "1122", new BytesArray("{}")));
assertEquals(1, getNumberOfFieldWithParentPrefix(doc.rootDoc()));
assertEquals("1122", doc.rootDoc().getBinaryValue("_parent#parent_type").utf8ToString());
// Indexing child doc:
DocumentMapper childDocMapper = indexService.mapperService().documentMapper("child_type");
doc =
childDocMapper.parse(
SourceToParse.source("test", "child_type", "1", new BytesArray("{}")).parent("1122"));
assertEquals(1, getNumberOfFieldWithParentPrefix(doc.rootDoc()));
assertEquals("1122", doc.rootDoc().getBinaryValue("_parent#parent_type").utf8ToString());
}
public void testParentSetInDocNotAllowed() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.endObject()
.endObject()
.string();
DocumentMapper docMapper =
createIndex("test")
.mapperService()
.documentMapperParser()
.parse("type", new CompressedXContent(mapping));
try {
docMapper.parse(
SourceToParse.source(
"test",
"type",
"1",
XContentFactory.jsonBuilder()
.startObject()
.field("_parent", "1122")
.endObject()
.bytes()));
fail("Expected failure to parse metadata field");
} catch (MapperParsingException e) {
assertTrue(
e.getMessage(),
e.getMessage()
.contains(
"Field [_parent] is a metadata field and cannot be added inside a document"));
}
}
public void testJoinFieldNotSet() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.endObject()
.endObject()
.string();
DocumentMapper docMapper =
createIndex("test")
.mapperService()
.documentMapperParser()
.parse("type", new CompressedXContent(mapping));
ParsedDocument doc =
docMapper.parse(
SourceToParse.source(
"test",
"type",
"1",
XContentFactory.jsonBuilder()
.startObject()
.field("x_field", "x_value")
.endObject()
.bytes()));
assertEquals(0, getNumberOfFieldWithParentPrefix(doc.rootDoc()));
}
public ParsedDocument parse(SourceToParse source, @Nullable ParseListener listener)
throws MapperParsingException {
ParseContext context = cache.get();
if (source.type() != null && !source.type().equals(this.type)) {
throw new MapperParsingException(
"Type mismatch, provide type ["
+ source.type()
+ "] but mapper is of type ["
+ this.type
+ "]");
}
source.type(this.type);
XContentParser parser = source.parser();
try {
if (parser == null) {
parser = XContentHelper.createParser(source.source());
}
context.reset(parser, new Document(), source, listener);
// on a newly created instance of document mapper, we always consider it as new mappers that
// have been added
if (initMappersAdded) {
context.setMappingsModified();
initMappersAdded = false;
}
// will result in START_OBJECT
int countDownTokens = 0;
XContentParser.Token token = parser.nextToken();
if (token != XContentParser.Token.START_OBJECT) {
throw new MapperParsingException("Malformed content, must start with an object");
}
boolean emptyDoc = false;
token = parser.nextToken();
if (token == XContentParser.Token.END_OBJECT) {
// empty doc, we can handle it...
emptyDoc = true;
} else if (token != XContentParser.Token.FIELD_NAME) {
throw new MapperParsingException(
"Malformed content, after first object, either the type field or the actual properties should exist");
}
if (type.equals(parser.currentName())) {
// first field is the same as the type, this might be because the type is provided, and the
// object exists within it
// or because there is a valid field that by chance is named as the type
// Note, in this case, we only handle plain value types, an object type will be analyzed as
// if it was the type itself
// and other same level fields will be ignored
token = parser.nextToken();
countDownTokens++;
// commented out, allow for same type with START_OBJECT, we do our best to handle it except
// for the above corner case
// if (token != XContentParser.Token.START_OBJECT) {
// throw new MapperException("Malformed content, a field with the same
// name as the type must be an object with the properties/fields within it");
// }
}
for (RootMapper rootMapper : rootMappersOrdered) {
rootMapper.preParse(context);
}
if (!emptyDoc) {
rootObjectMapper.parse(context);
}
for (int i = 0; i < countDownTokens; i++) {
parser.nextToken();
}
// fire up any new mappers if exists
if (!context.newFieldMappers().mappers.isEmpty()) {
addFieldMappers(context.newFieldMappers().mappers);
context.newFieldMappers().mappers.clear();
}
if (!context.newObjectMappers().mappers.isEmpty()) {
addObjectMappers(context.newObjectMappers().mappers);
context.newObjectMappers().mappers.clear();
}
for (RootMapper rootMapper : rootMappersOrdered) {
rootMapper.postParse(context);
}
for (RootMapper rootMapper : rootMappersOrdered) {
rootMapper.validate(context);
}
} catch (Throwable e) {
// we have to fire up any new mappers even on a failure, because they
// have been added internally to each compound mapper...
// ... we have no option to "rollback" a change, which is very tricky in our copy on change
// system...
if (!context.newFieldMappers().mappers.isEmpty()) {
addFieldMappers(context.newFieldMappers().mappers);
context.newFieldMappers().mappers.clear();
}
if (!context.newObjectMappers().mappers.isEmpty()) {
addObjectMappers(context.newObjectMappers().mappers);
context.newObjectMappers().mappers.clear();
}
// if its already a mapper parsing exception, no need to wrap it...
if (e instanceof MapperParsingException) {
throw (MapperParsingException) e;
}
throw new MapperParsingException("failed to parse", e);
} finally {
// only close the parser when its not provided externally
if (source.parser() == null && parser != null) {
parser.close();
}
}
// reverse the order of docs for nested docs support, parent should be last
if (context.docs().size() > 1) {
Collections.reverse(context.docs());
}
// apply doc boost
if (context.docBoost() != 1.0f) {
Set<String> encounteredFields = Sets.newHashSet();
for (Document doc : context.docs()) {
encounteredFields.clear();
for (IndexableField field : doc) {
if (field.fieldType().indexed() && !field.fieldType().omitNorms()) {
if (!encounteredFields.contains(field.name())) {
((Field) field).setBoost(context.docBoost() * field.boost());
encounteredFields.add(field.name());
}
}
}
}
}
ParsedDocument doc =
new ParsedDocument(
context.uid(),
context.id(),
context.type(),
source.routing(),
source.timestamp(),
source.ttl(),
context.docs(),
context.analyzer(),
context.source(),
context.mappingsModified())
.parent(source.parent());
// reset the context to free up memory
context.reset(null, null, null, null);
return doc;
}
public ParsedDocument parse(String type, String id, BytesReference source)
throws MapperParsingException {
return parse(SourceToParse.source(source).type(type).id(id));
}
public ParsedDocument parse(BytesReference source) throws MapperParsingException {
return parse(SourceToParse.source(source));
}
private ParsedDocument innerParseDocument(SourceToParse source) throws MapperParsingException {
if (docMapper.type().equals(MapperService.DEFAULT_MAPPING)) {
throw new IllegalArgumentException(
"It is forbidden to index into the default mapping ["
+ MapperService.DEFAULT_MAPPING
+ "]");
}
ParseContext.InternalParseContext context = cache.get();
final Mapping mapping = docMapper.mapping();
if (source.type() != null && !source.type().equals(docMapper.type())) {
throw new MapperParsingException(
"Type mismatch, provide type ["
+ source.type()
+ "] but mapper is of type ["
+ docMapper.type()
+ "]");
}
source.type(docMapper.type());
XContentParser parser = source.parser();
try {
if (parser == null) {
parser = XContentHelper.createParser(source.source());
}
context.reset(parser, new ParseContext.Document(), source);
// will result in START_OBJECT
XContentParser.Token token = parser.nextToken();
if (token != XContentParser.Token.START_OBJECT) {
throw new MapperParsingException("Malformed content, must start with an object");
}
boolean emptyDoc = false;
if (mapping.root.isEnabled()) {
token = parser.nextToken();
if (token == XContentParser.Token.END_OBJECT) {
// empty doc, we can handle it...
emptyDoc = true;
} else if (token != XContentParser.Token.FIELD_NAME) {
throw new MapperParsingException(
"Malformed content, after first object, either the type field or the actual properties should exist");
}
}
for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) {
metadataMapper.preParse(context);
}
if (mapping.root.isEnabled() == false) {
// entire type is disabled
parser.skipChildren();
} else if (emptyDoc == false) {
Mapper update = parseObject(context, mapping.root, true);
if (update != null) {
context.addDynamicMappingsUpdate(update);
}
}
for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) {
metadataMapper.postParse(context);
}
// try to parse the next token, this should be null if the object is ended properly
// but will throw a JSON exception if the extra tokens is not valid JSON (this will be handled
// by the catch)
if (Version.indexCreated(indexSettings).onOrAfter(Version.V_2_0_0_beta1)
&& source.parser() == null
&& parser != null) {
// only check for end of tokens if we created the parser here
token = parser.nextToken();
if (token != null) {
throw new IllegalArgumentException(
"Malformed content, found extra data after parsing: " + token);
}
}
} catch (Throwable e) {
// if its already a mapper parsing exception, no need to wrap it...
if (e instanceof MapperParsingException) {
throw (MapperParsingException) e;
}
// Throw a more meaningful message if the document is empty.
if (source.source() != null && source.source().length() == 0) {
throw new MapperParsingException("failed to parse, document is empty");
}
throw new MapperParsingException("failed to parse", e);
} finally {
// only close the parser when its not provided externally
if (source.parser() == null && parser != null) {
parser.close();
}
}
// reverse the order of docs for nested docs support, parent should be last
if (context.docs().size() > 1) {
Collections.reverse(context.docs());
}
// apply doc boost
if (context.docBoost() != 1.0f) {
Set<String> encounteredFields = new HashSet<>();
for (ParseContext.Document doc : context.docs()) {
encounteredFields.clear();
for (IndexableField field : doc) {
if (field.fieldType().indexOptions() != IndexOptions.NONE
&& !field.fieldType().omitNorms()) {
if (!encounteredFields.contains(field.name())) {
((Field) field).setBoost(context.docBoost() * field.boost());
encounteredFields.add(field.name());
}
}
}
}
}
Mapper rootDynamicUpdate = context.dynamicMappingsUpdate();
Mapping update = null;
if (rootDynamicUpdate != null) {
update = mapping.mappingUpdate(rootDynamicUpdate);
}
ParsedDocument doc =
new ParsedDocument(
context.uid(),
context.version(),
context.id(),
context.type(),
source.routing(),
source.timestamp(),
source.ttl(),
context.docs(),
context.source(),
update)
.parent(source.parent());
// reset the context to free up memory
context.reset(null, null, null);
return doc;
}
public void testMixTemplateMultiFieldAndMappingReuse() throws Exception {
IndexService indexService = createIndex("test");
XContentBuilder mappings1 =
jsonBuilder()
.startObject()
.startObject("type1")
.startArray("dynamic_templates")
.startObject()
.startObject("template1")
.field("match_mapping_type", "string")
.startObject("mapping")
.field("type", "text")
.startObject("fields")
.startObject("raw")
.field("type", "keyword")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endArray()
.endObject()
.endObject();
indexService
.mapperService()
.merge(
"type1",
new CompressedXContent(mappings1.bytes()),
MapperService.MergeReason.MAPPING_UPDATE,
false);
XContentBuilder mappings2 =
jsonBuilder()
.startObject()
.startObject("type2")
.startObject("properties")
.startObject("field")
.field("type", "text")
.endObject()
.endObject()
.endObject()
.endObject();
indexService
.mapperService()
.merge(
"type2",
new CompressedXContent(mappings2.bytes()),
MapperService.MergeReason.MAPPING_UPDATE,
false);
XContentBuilder json =
XContentFactory.jsonBuilder().startObject().field("field", "foo").endObject();
SourceToParse source = SourceToParse.source(json.bytes()).id("1");
DocumentMapper mapper = indexService.mapperService().documentMapper("type1");
assertNull(mapper.mappers().getMapper("field.raw"));
ParsedDocument parsed = mapper.parse(source);
assertNotNull(parsed.dynamicMappingsUpdate());
indexService
.mapperService()
.merge(
"type1",
new CompressedXContent(parsed.dynamicMappingsUpdate().toString()),
MapperService.MergeReason.MAPPING_UPDATE,
false);
mapper = indexService.mapperService().documentMapper("type1");
assertNotNull(mapper.mappers().getMapper("field.raw"));
parsed = mapper.parse(source);
assertNull(parsed.dynamicMappingsUpdate());
}
