public void testParentSetInDocNotAllowed() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.endObject()
.endObject()
.string();
DocumentMapper docMapper =
createIndex("test")
.mapperService()
.documentMapperParser()
.parse("type", new CompressedXContent(mapping));
try {
docMapper.parse(
SourceToParse.source(
"test",
"type",
"1",
XContentFactory.jsonBuilder()
.startObject()
.field("_parent", "1122")
.endObject()
.bytes()));
fail("Expected failure to parse metadata field");
} catch (MapperParsingException e) {
assertTrue(
e.getMessage(),
e.getMessage()
.contains(
"Field [_parent] is a metadata field and cannot be added inside a document"));
}
}
public void testRejectNorms() throws IOException {
// not supported as of 5.0
for (String type : Arrays.asList("byte", "short", "integer", "long", "float", "double")) {
DocumentMapperParser parser =
createIndex("index-" + type).mapperService().documentMapperParser();
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("foo")
.field("type", type)
.field("norms", random().nextBoolean())
.endObject()
.endObject()
.endObject()
.endObject()
.string();
MapperParsingException e =
expectThrows(
MapperParsingException.class,
() -> parser.parse("type", new CompressedXContent(mapping)));
assertThat(
e.getMessage(),
containsString("Mapping definition for [foo] has unsupported parameters: [norms"));
}
}
private void doTestIgnoreMalformed(String type) throws IOException {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("field")
.field("type", type)
.endObject()
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, mapper.mappingSource().toString());
ThrowingRunnable runnable =
() ->
mapper.parse(
"test",
"type",
"1",
XContentFactory.jsonBuilder()
.startObject()
.field("field", "a")
.endObject()
.bytes());
MapperParsingException e = expectThrows(MapperParsingException.class, runnable);
assertThat(e.getCause().getMessage(), containsString("For input string: \"a\""));
mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("field")
.field("type", type)
.field("ignore_malformed", true)
.endObject()
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper mapper2 = parser.parse("type", new CompressedXContent(mapping));
ParsedDocument doc =
mapper2.parse(
"test",
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("field", "a").endObject().bytes());
IndexableField[] fields = doc.rootDoc().getFields("field");
assertEquals(0, fields.length);
}
@Test
public void testIgnoreMalformedOption() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("field1")
.field("type", "integer")
.field("ignore_malformed", true)
.endObject()
.startObject("field2")
.field("type", "integer")
.field("ignore_malformed", false)
.endObject()
.startObject("field3")
.field("type", "integer")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(mapping);
ParsedDocument doc =
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder()
.startObject()
.field("field1", "a")
.field("field2", "1")
.endObject()
.bytes());
assertThat(doc.rootDoc().getField("field1"), nullValue());
assertThat(doc.rootDoc().getField("field2"), notNullValue());
try {
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("field2", "a").endObject().bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(NumberFormatException.class));
}
// Verify that the default is false
try {
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("field3", "a").endObject().bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(NumberFormatException.class));
}
// Unless the global ignore_malformed option is set to true
Settings indexSettings = settingsBuilder().put("index.mapping.ignore_malformed", true).build();
defaultMapper = MapperTestUtils.newParser(indexSettings).parse(mapping);
doc =
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("field3", "a").endObject().bytes());
assertThat(doc.rootDoc().getField("field3"), nullValue());
// This should still throw an exception, since field2 is specifically set to
// ignore_malformed=false
try {
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("field2", "a").endObject().bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(NumberFormatException.class));
}
}
@Override
public void doTestCoerce(String type) throws IOException {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("field")
.field("type", type)
.endObject()
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, mapper.mappingSource().toString());
ParsedDocument doc =
mapper.parse(
"test",
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("field", "123").endObject().bytes());
IndexableField[] fields = doc.rootDoc().getFields("field");
assertEquals(2, fields.length);
IndexableField pointField = fields[0];
assertEquals(1, pointField.fieldType().pointDimensionCount());
assertEquals(123, pointField.numericValue().doubleValue(), 0d);
IndexableField dvField = fields[1];
assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("field")
.field("type", type)
.field("coerce", false)
.endObject()
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper mapper2 = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, mapper2.mappingSource().toString());
ThrowingRunnable runnable =
() ->
mapper2.parse(
"test",
"type",
"1",
XContentFactory.jsonBuilder()
.startObject()
.field("field", "123")
.endObject()
.bytes());
MapperParsingException e = expectThrows(MapperParsingException.class, runnable);
assertThat(e.getCause().getMessage(), containsString("passed as String"));
}
@Override
public Mapper parse(ParseContext context) throws IOException {
byte[] content = null;
String contentType = null;
int indexedChars = defaultIndexedChars;
boolean langDetect = defaultLangDetect;
String name = null;
String language = null;
XContentParser parser = context.parser();
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.VALUE_STRING) {
content = parser.binaryValue();
} else {
String currentFieldName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.VALUE_STRING) {
if ("_content".equals(currentFieldName)) {
content = parser.binaryValue();
} else if ("_content_type".equals(currentFieldName)) {
contentType = parser.text();
} else if ("_name".equals(currentFieldName)) {
name = parser.text();
} else if ("_language".equals(currentFieldName)) {
language = parser.text();
}
} else if (token == XContentParser.Token.VALUE_NUMBER) {
if ("_indexed_chars".equals(currentFieldName)
|| "_indexedChars".equals(currentFieldName)) {
indexedChars = parser.intValue();
}
} else if (token == XContentParser.Token.VALUE_BOOLEAN) {
if ("_detect_language".equals(currentFieldName)
|| "_detectLanguage".equals(currentFieldName)) {
langDetect = parser.booleanValue();
}
}
}
}
// Throw clean exception when no content is provided Fix #23
if (content == null) {
throw new MapperParsingException("No content is provided.");
}
Metadata metadata = new Metadata();
if (contentType != null) {
metadata.add(Metadata.CONTENT_TYPE, contentType);
}
if (name != null) {
metadata.add(Metadata.RESOURCE_NAME_KEY, name);
}
String parsedContent;
try {
parsedContent = TikaImpl.parse(content, metadata, indexedChars);
} catch (Throwable e) {
// #18: we could ignore errors when Tika does not parse data
if (!ignoreErrors) {
logger.trace("exception caught", e);
throw new MapperParsingException(
"Failed to extract ["
+ indexedChars
+ "] characters of text for ["
+ name
+ "] : "
+ e.getMessage(),
e);
} else {
logger.debug(
"Failed to extract [{}] characters of text for [{}]: [{}]",
indexedChars,
name,
e.getMessage());
logger.trace("exception caught", e);
}
return null;
}
context = context.createExternalValueContext(parsedContent);
contentMapper.parse(context);
if (langDetect) {
try {
if (language != null) {
metadata.add(Metadata.CONTENT_LANGUAGE, language);
} else {
LanguageIdentifier identifier = new LanguageIdentifier(parsedContent);
language = identifier.getLanguage();
}
context = context.createExternalValueContext(language);
languageMapper.parse(context);
} catch (Throwable t) {
logger.debug("Cannot detect language: [{}]", t.getMessage());
}
}
if (name != null) {
try {
context = context.createExternalValueContext(name);
nameMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing name: [{}]", e.getMessage());
}
}
if (metadata.get(Metadata.DATE) != null) {
try {
context = context.createExternalValueContext(metadata.get(Metadata.DATE));
dateMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing date: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
}
if (metadata.get(Metadata.TITLE) != null) {
try {
context = context.createExternalValueContext(metadata.get(Metadata.TITLE));
titleMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing title: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
}
if (metadata.get(Metadata.AUTHOR) != null) {
try {
context = context.createExternalValueContext(metadata.get(Metadata.AUTHOR));
authorMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing author: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
}
if (metadata.get(Metadata.KEYWORDS) != null) {
try {
context = context.createExternalValueContext(metadata.get(Metadata.KEYWORDS));
keywordsMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing keywords: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
}
if (contentType == null) {
contentType = metadata.get(Metadata.CONTENT_TYPE);
}
if (contentType != null) {
try {
context = context.createExternalValueContext(contentType);
contentTypeMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing content_type: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
}
int length = content.length;
// If we have CONTENT_LENGTH from Tika we use it
if (metadata.get(Metadata.CONTENT_LENGTH) != null) {
length = Integer.parseInt(metadata.get(Metadata.CONTENT_LENGTH));
}
try {
context = context.createExternalValueContext(length);
contentLengthMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing content_length: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
// multiFields.parse(this, context);
return null;
}
