public void testCombineTemplates() throws Exception {
// clean all templates setup by the framework.
client().admin().indices().prepareDeleteTemplate("*").get();
// check get all templates on an empty index.
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
assertThat(response.getIndexTemplates(), empty());
// Now, a complete mapping with two separated templates is error
// base template
client()
.admin()
.indices()
.preparePutTemplate("template_1")
.setTemplate("*")
.setSettings(
" {\n"
+ " \"index\" : {\n"
+ " \"analysis\" : {\n"
+ " \"analyzer\" : {\n"
+ " \"custom_1\" : {\n"
+ " \"tokenizer\" : \"whitespace\"\n"
+ " }\n"
+ " }\n"
+ " }\n"
+ " }\n"
+ " }\n")
.get();
// put template using custom_1 analyzer
MapperParsingException e =
expectThrows(
MapperParsingException.class,
() ->
client()
.admin()
.indices()
.preparePutTemplate("template_2")
.setTemplate("test*")
.setCreate(true)
.setOrder(1)
.addMapping(
"type1",
XContentFactory.jsonBuilder()
.startObject()
.startObject("type1")
.startObject("properties")
.startObject("field2")
.field("type", "string")
.field("analyzer", "custom_1")
.endObject()
.endObject()
.endObject()
.endObject())
.get());
assertThat(e.getMessage(), containsString("analyzer [custom_1] not found for field [field2]"));
response = client().admin().indices().prepareGetTemplates().get();
assertThat(response.getIndexTemplates(), hasSize(1));
}
public void testPercolatorFieldMapper_noQuery() throws Exception {
addQueryMapping();
ParsedDocument doc =
mapperService
.documentMapper(typeName)
.parse(
"test",
typeName,
"1",
XContentFactory.jsonBuilder().startObject().endObject().bytes());
assertThat(doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName()).length, equalTo(0));
try {
mapperService
.documentMapper(typeName)
.parse(
"test",
typeName,
"1",
XContentFactory.jsonBuilder().startObject().nullField(fieldName).endObject().bytes());
} catch (MapperParsingException e) {
assertThat(
e.getDetailedMessage(), containsString("query malformed, must start with start_object"));
}
}
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/8802")
public void testBrokenMapping() throws Exception {
// clean all templates setup by the framework.
client().admin().indices().prepareDeleteTemplate("*").get();
// check get all templates on an empty index.
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
assertThat(response.getIndexTemplates(), empty());
MapperParsingException e =
expectThrows(
MapperParsingException.class,
() ->
client()
.admin()
.indices()
.preparePutTemplate("template_1")
.setTemplate("te*")
.addMapping("type1", "abcde")
.get());
assertThat(e.getMessage(), containsString("Failed to parse mapping "));
response = client().admin().indices().prepareGetTemplates().get();
assertThat(response.getIndexTemplates(), hasSize(0));
}
public void testUnsupportedFeatures() throws IOException {
XContentBuilder mapping =
XContentBuilder.builder(JsonXContent.jsonXContent)
.startObject()
.startObject("type")
.startObject(FieldNamesFieldMapper.NAME)
// by setting randomly index to no we also test the pre-1.3 behavior
.field("index", randomFrom("no", "not_analyzed"))
.field("store", randomFrom("no", "yes"))
.endObject()
.endObject()
.endObject();
try {
assertAcked(
prepareCreate("test")
.setSettings(
Settings.builder()
.put(
"index.routing.allocation.exclude._name",
backwardsCluster().newNodePattern())
.put(indexSettings()))
.addMapping("type", mapping));
} catch (MapperParsingException ex) {
assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class));
assertThat(
ExceptionsHelper.detailedMessage(ex)
.contains(
"type=_field_names is not supported on indices created before version 1.3.0"),
equalTo(true));
}
}
public void testIncludeInObjectNotAllowed() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.endObject()
.endObject()
.string();
DocumentMapper docMapper =
createIndex("test").mapperService().documentMapperParser().parse(mapping);
try {
docMapper.parse(
"test",
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("_all", "foo").endObject().bytes());
fail("Expected failure to parse metadata field");
} catch (MapperParsingException e) {
assertTrue(
e.getMessage(),
e.getMessage()
.contains("Field [_all] is a metadata field and cannot be added inside a document"));
}
}
public void testAllowNoAdditionalSettings() throws Exception {
addQueryMapping();
IndexService indexService = createIndex("test1", Settings.EMPTY);
MapperService mapperService = indexService.mapperService();
String percolatorMapper =
XContentFactory.jsonBuilder()
.startObject()
.startObject(typeName)
.startObject("properties")
.startObject(fieldName)
.field("type", "percolator")
.field("index", "no")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
try {
mapperService.merge(
typeName,
new CompressedXContent(percolatorMapper),
MapperService.MergeReason.MAPPING_UPDATE,
true);
fail("MapperParsingException expected");
} catch (MapperParsingException e) {
assertThat(
e.getMessage(),
equalTo(
"Mapping definition for ["
+ fieldName
+ "] has unsupported parameters: [index : no]"));
}
}
public void testIgnoreMalformed() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("field")
.field("type", "ip")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, mapper.mappingSource().toString());
ThrowingRunnable runnable =
() ->
mapper.parse(
"test",
"type",
"1",
XContentFactory.jsonBuilder()
.startObject()
.field("field", ":1")
.endObject()
.bytes());
MapperParsingException e = expectThrows(MapperParsingException.class, runnable);
assertThat(e.getCause().getMessage(), containsString("':1' is not an IP string literal"));
mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("field")
.field("type", "ip")
.field("ignore_malformed", true)
.endObject()
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper mapper2 = parser.parse("type", new CompressedXContent(mapping));
ParsedDocument doc =
mapper2.parse(
"test",
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("field", ":1").endObject().bytes());
IndexableField[] fields = doc.rootDoc().getFields("field");
assertEquals(0, fields.length);
}
// related to https://github.com/elasticsearch/elasticsearch/issues/5864
public void testMistypedTypeInRoot() throws IOException {
String mapping =
copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mistyped_type_in_root.json");
try {
createIndex("test").mapperService().documentMapperParser().parse("test", mapping);
fail("Expected MapperParsingException");
} catch (MapperParsingException e) {
assertThat(
e.getMessage(), containsString("Root mapping definition has unsupported parameters"));
assertThat(e.getMessage(), containsString("type=string"));
}
}
@Test
public void testPointsOnly() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type1")
.startObject("properties")
.startObject("location")
.field("type", "geo_shape")
.field("tree", randomBoolean() ? "quadtree" : "geohash")
.field("tree_levels", "6")
.field("distance_error_pct", "0.01")
.field("points_only", true)
.endObject()
.endObject()
.endObject()
.endObject()
.string();
assertAcked(prepareCreate("geo_points_only").addMapping("type1", mapping));
ensureGreen();
ShapeBuilder shape = RandomShapeGenerator.createShape(random());
try {
index(
"geo_points_only",
"type1",
"1",
jsonBuilder().startObject().field("location", shape).endObject());
} catch (MapperParsingException e) {
// RandomShapeGenerator created something other than a POINT type, verify the correct
// exception is thrown
assertThat(e.getCause().getMessage(), containsString("is configured for points only"));
return;
}
refresh();
// test that point was inserted
SearchResponse response =
client()
.prepareSearch()
.setQuery(geoIntersectionQuery("location", shape))
.execute()
.actionGet();
assertEquals(1, response.getHits().getTotalHits());
}
public void testDocValuesNotAllowed() throws IOException {
String mapping =
jsonBuilder()
.startObject()
.startObject("type")
.startObject("_all")
.field("doc_values", true)
.endObject()
.endObject()
.endObject()
.string();
try {
createIndex("test").mapperService().documentMapperParser().parse(mapping);
fail();
} catch (MapperParsingException e) {
assertThat(
e.getDetailedMessage(),
containsString("[_all] is always tokenized and cannot have doc values"));
}
mapping =
jsonBuilder()
.startObject()
.startObject("type")
.startObject("_all")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
Settings legacySettings =
Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
try {
createIndex("test_old", legacySettings).mapperService().documentMapperParser().parse(mapping);
fail();
} catch (MapperParsingException e) {
assertThat(
e.getDetailedMessage(),
containsString("[_all] is always tokenized and cannot have doc values"));
}
}
public void testDynamicDateDetectionIn2xDoesNotSupportEpochs() throws Exception {
try {
XContentBuilder mapping =
jsonBuilder()
.startObject()
.startArray("dynamic_date_formats")
.value("dateOptionalTime")
.value("epoch_seconds")
.endArray()
.endObject();
createIndex(Version.CURRENT, mapping);
fail("Expected a MapperParsingException, but did not happen");
} catch (MapperParsingException e) {
assertThat(e.getMessage(), containsString("Failed to parse mapping [" + type + "]"));
assertThat(
e.getMessage(),
containsString("Epoch [epoch_seconds] is not supported as dynamic date format"));
}
}
public void testNoDocumentSent() throws Exception {
IndexService indexService = createIndex("test");
Settings settings = indexService.getIndexSettings().getSettings();
DocumentMapperParser mapperParser = indexService.mapperService().documentMapperParser();
DocumentMapper docMapper =
doc(
settings,
rootObject("person")
.add(object("name").add(stringField("first").store(true).index(false))),
indexService.mapperService())
.build(indexService.mapperService(), mapperParser);
BytesReference json = new BytesArray("".getBytes(StandardCharsets.UTF_8));
try {
docMapper.parse("test", "person", "1", json).rootDoc();
fail("this point is never reached");
} catch (MapperParsingException e) {
assertThat(e.getMessage(), equalTo("failed to parse, document is empty"));
}
}
public void testHazardousFieldNames() throws Exception {
IndexService indexService = createIndex("test");
DocumentMapperParser mapperParser = indexService.mapperService().documentMapperParser();
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("foo.bar")
.field("type", "string")
.endObject()
.endObject()
.endObject()
.string();
try {
mapperParser.parse(mapping);
fail("Mapping parse should have failed");
} catch (MapperParsingException e) {
assertTrue(e.getMessage(), e.getMessage().contains("cannot contain '.'"));
}
}
public void testAcceptDocValuesFormat() throws IOException {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("field")
.field("type", "string")
.field("doc_values_format", Codec.getDefault().docValuesFormat().getName())
.endObject()
.endObject()
.endObject()
.endObject()
.string();
int i = 0;
for (Version v : VersionUtils.allVersions()) {
if (v.onOrAfter(Version.V_2_0_0) == false) {
// no need to test, we don't support upgrading from these versions
continue;
}
IndexService indexService =
createIndex(
"test-" + i++,
Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, v).build());
DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
try {
parser.parse("type", new CompressedXContent(mapping));
if (v.onOrAfter(Version.V_2_0_0_beta1)) {
fail("Elasticsearch 2.0 should not support custom postings formats");
}
} catch (MapperParsingException e) {
if (v.before(Version.V_2_0_0_beta1)) {
// Elasticsearch 1.x should ignore custom postings formats
throw e;
}
Assert.assertThat(
e.getMessage(), containsString("unsupported parameters: [doc_values_format"));
}
}
}
public void testPercolatorFieldMapperUnMappedField() throws Exception {
addQueryMapping();
MapperParsingException exception =
expectThrows(
MapperParsingException.class,
() -> {
mapperService
.documentMapper(typeName)
.parse(
"test",
typeName,
"1",
XContentFactory.jsonBuilder()
.startObject()
.field(fieldName, termQuery("unmapped_field", "value"))
.endObject()
.bytes());
});
assertThat(exception.getCause(), instanceOf(QueryShardException.class));
assertThat(
exception.getCause().getMessage(),
equalTo("No field mapping can be found for the field with name [unmapped_field]"));
}
// percolator field can be nested under an object field, but only one query can be specified per
// document
public void testNestedPercolatorField() throws Exception {
String typeName = "another_type";
String percolatorMapper =
XContentFactory.jsonBuilder()
.startObject()
.startObject(typeName)
.startObject("_field_names")
.field("enabled", false)
.endObject() // makes testing easier
.startObject("properties")
.startObject("object_field")
.field("type", "object")
.startObject("properties")
.startObject("query_field")
.field("type", "percolator")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.string();
mapperService.merge(
typeName,
new CompressedXContent(percolatorMapper),
MapperService.MergeReason.MAPPING_UPDATE,
true);
QueryBuilder queryBuilder = matchQuery("field", "value");
ParsedDocument doc =
mapperService
.documentMapper(typeName)
.parse(
"test",
typeName,
"1",
jsonBuilder()
.startObject()
.startObject("object_field")
.field("query_field", queryBuilder)
.endObject()
.endObject()
.bytes());
assertThat(
doc.rootDoc().getFields().size(),
equalTo(8)); // also includes _uid (1), type (2), source (1)
BytesRef queryBuilderAsBytes =
doc.rootDoc().getField("object_field.query_field.query_builder_field").binaryValue();
assertQueryBuilder(queryBuilderAsBytes, queryBuilder);
doc =
mapperService
.documentMapper(typeName)
.parse(
"test",
typeName,
"1",
jsonBuilder()
.startObject()
.startArray("object_field")
.startObject()
.field("query_field", queryBuilder)
.endObject()
.endArray()
.endObject()
.bytes());
assertThat(
doc.rootDoc().getFields().size(),
equalTo(8)); // also includes _uid (1), type (2), source (1)
queryBuilderAsBytes =
doc.rootDoc().getField("object_field.query_field.query_builder_field").binaryValue();
assertQueryBuilder(queryBuilderAsBytes, queryBuilder);
MapperParsingException e =
expectThrows(
MapperParsingException.class,
() -> {
mapperService
.documentMapper(typeName)
.parse(
"test",
typeName,
"1",
jsonBuilder()
.startObject()
.startArray("object_field")
.startObject()
.field("query_field", queryBuilder)
.endObject()
.startObject()
.field("query_field", queryBuilder)
.endObject()
.endArray()
.endObject()
.bytes());
});
assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
assertThat(
e.getCause().getMessage(), equalTo("a document can only contain one percolator query"));
}
@Test
public void testIgnoreMalformedOption() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("field1")
.field("type", "integer")
.field("ignore_malformed", true)
.endObject()
.startObject("field2")
.field("type", "integer")
.field("ignore_malformed", false)
.endObject()
.startObject("field3")
.field("type", "integer")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(mapping);
ParsedDocument doc =
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder()
.startObject()
.field("field1", "a")
.field("field2", "1")
.endObject()
.bytes());
assertThat(doc.rootDoc().getField("field1"), nullValue());
assertThat(doc.rootDoc().getField("field2"), notNullValue());
try {
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("field2", "a").endObject().bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(NumberFormatException.class));
}
// Verify that the default is false
try {
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("field3", "a").endObject().bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(NumberFormatException.class));
}
// Unless the global ignore_malformed option is set to true
Settings indexSettings = settingsBuilder().put("index.mapping.ignore_malformed", true).build();
defaultMapper = MapperTestUtils.newParser(indexSettings).parse(mapping);
doc =
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("field3", "a").endObject().bytes());
assertThat(doc.rootDoc().getField("field3"), nullValue());
// This should still throw an exception, since field2 is specifically set to
// ignore_malformed=false
try {
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("field2", "a").endObject().bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(NumberFormatException.class));
}
}
@Override
public Mapper parse(ParseContext context) throws IOException {
byte[] content = null;
String contentType = null;
int indexedChars = defaultIndexedChars;
boolean langDetect = defaultLangDetect;
String name = null;
String language = null;
XContentParser parser = context.parser();
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.VALUE_STRING) {
content = parser.binaryValue();
} else {
String currentFieldName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.VALUE_STRING) {
if ("_content".equals(currentFieldName)) {
content = parser.binaryValue();
} else if ("_content_type".equals(currentFieldName)) {
contentType = parser.text();
} else if ("_name".equals(currentFieldName)) {
name = parser.text();
} else if ("_language".equals(currentFieldName)) {
language = parser.text();
}
} else if (token == XContentParser.Token.VALUE_NUMBER) {
if ("_indexed_chars".equals(currentFieldName)
|| "_indexedChars".equals(currentFieldName)) {
indexedChars = parser.intValue();
}
} else if (token == XContentParser.Token.VALUE_BOOLEAN) {
if ("_detect_language".equals(currentFieldName)
|| "_detectLanguage".equals(currentFieldName)) {
langDetect = parser.booleanValue();
}
}
}
}
// Throw clean exception when no content is provided Fix #23
if (content == null) {
throw new MapperParsingException("No content is provided.");
}
Metadata metadata = new Metadata();
if (contentType != null) {
metadata.add(Metadata.CONTENT_TYPE, contentType);
}
if (name != null) {
metadata.add(Metadata.RESOURCE_NAME_KEY, name);
}
String parsedContent;
try {
parsedContent = TikaImpl.parse(content, metadata, indexedChars);
} catch (Throwable e) {
// #18: we could ignore errors when Tika does not parse data
if (!ignoreErrors) {
logger.trace("exception caught", e);
throw new MapperParsingException(
"Failed to extract ["
+ indexedChars
+ "] characters of text for ["
+ name
+ "] : "
+ e.getMessage(),
e);
} else {
logger.debug(
"Failed to extract [{}] characters of text for [{}]: [{}]",
indexedChars,
name,
e.getMessage());
logger.trace("exception caught", e);
}
return null;
}
context = context.createExternalValueContext(parsedContent);
contentMapper.parse(context);
if (langDetect) {
try {
if (language != null) {
metadata.add(Metadata.CONTENT_LANGUAGE, language);
} else {
LanguageIdentifier identifier = new LanguageIdentifier(parsedContent);
language = identifier.getLanguage();
}
context = context.createExternalValueContext(language);
languageMapper.parse(context);
} catch (Throwable t) {
logger.debug("Cannot detect language: [{}]", t.getMessage());
}
}
if (name != null) {
try {
context = context.createExternalValueContext(name);
nameMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing name: [{}]", e.getMessage());
}
}
if (metadata.get(Metadata.DATE) != null) {
try {
context = context.createExternalValueContext(metadata.get(Metadata.DATE));
dateMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing date: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
}
if (metadata.get(Metadata.TITLE) != null) {
try {
context = context.createExternalValueContext(metadata.get(Metadata.TITLE));
titleMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing title: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
}
if (metadata.get(Metadata.AUTHOR) != null) {
try {
context = context.createExternalValueContext(metadata.get(Metadata.AUTHOR));
authorMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing author: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
}
if (metadata.get(Metadata.KEYWORDS) != null) {
try {
context = context.createExternalValueContext(metadata.get(Metadata.KEYWORDS));
keywordsMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing keywords: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
}
if (contentType == null) {
contentType = metadata.get(Metadata.CONTENT_TYPE);
}
if (contentType != null) {
try {
context = context.createExternalValueContext(contentType);
contentTypeMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing content_type: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
}
int length = content.length;
// If we have CONTENT_LENGTH from Tika we use it
if (metadata.get(Metadata.CONTENT_LENGTH) != null) {
length = Integer.parseInt(metadata.get(Metadata.CONTENT_LENGTH));
}
try {
context = context.createExternalValueContext(length);
contentLengthMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing content_length: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
// multiFields.parse(this, context);
return null;
}