public void testParentSetInDocNotAllowed() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .endObject() .endObject() .string(); DocumentMapper docMapper = createIndex("test") .mapperService() .documentMapperParser() .parse("type", new CompressedXContent(mapping)); try { docMapper.parse( SourceToParse.source( "test", "type", "1", XContentFactory.jsonBuilder() .startObject() .field("_parent", "1122") .endObject() .bytes())); fail("Expected failure to parse metadata field"); } catch (MapperParsingException e) { assertTrue( e.getMessage(), e.getMessage() .contains( "Field [_parent] is a metadata field and cannot be added inside a document")); } }
public void testRejectNorms() throws IOException { // not supported as of 5.0 for (String type : Arrays.asList("byte", "short", "integer", "long", "float", "double")) { DocumentMapperParser parser = createIndex("index-" + type).mapperService().documentMapperParser(); String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("foo") .field("type", type) .field("norms", random().nextBoolean()) .endObject() .endObject() .endObject() .endObject() .string(); MapperParsingException e = expectThrows( MapperParsingException.class, () -> parser.parse("type", new CompressedXContent(mapping))); assertThat( e.getMessage(), containsString("Mapping definition for [foo] has unsupported parameters: [norms")); } }
private void doTestIgnoreMalformed(String type) throws IOException { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field") .field("type", type) .endObject() .endObject() .endObject() .endObject() .string(); DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); assertEquals(mapping, mapper.mappingSource().toString()); ThrowingRunnable runnable = () -> mapper.parse( "test", "type", "1", XContentFactory.jsonBuilder() .startObject() .field("field", "a") .endObject() .bytes()); MapperParsingException e = expectThrows(MapperParsingException.class, runnable); assertThat(e.getCause().getMessage(), containsString("For input string: \"a\"")); mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field") .field("type", type) .field("ignore_malformed", true) .endObject() .endObject() .endObject() .endObject() .string(); DocumentMapper mapper2 = parser.parse("type", new CompressedXContent(mapping)); ParsedDocument doc = mapper2.parse( "test", "type", "1", XContentFactory.jsonBuilder().startObject().field("field", "a").endObject().bytes()); IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(0, fields.length); }
@Test public void testIgnoreMalformedOption() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field1") .field("type", "integer") .field("ignore_malformed", true) .endObject() .startObject("field2") .field("type", "integer") .field("ignore_malformed", false) .endObject() .startObject("field3") .field("type", "integer") .endObject() .endObject() .endObject() .endObject() .string(); DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(mapping); ParsedDocument doc = defaultMapper.parse( "type", "1", XContentFactory.jsonBuilder() .startObject() .field("field1", "a") .field("field2", "1") .endObject() .bytes()); assertThat(doc.rootDoc().getField("field1"), nullValue()); assertThat(doc.rootDoc().getField("field2"), notNullValue()); try { defaultMapper.parse( "type", "1", XContentFactory.jsonBuilder().startObject().field("field2", "a").endObject().bytes()); } catch (MapperParsingException e) { assertThat(e.getCause(), instanceOf(NumberFormatException.class)); } // Verify that the default is false try { defaultMapper.parse( "type", "1", XContentFactory.jsonBuilder().startObject().field("field3", "a").endObject().bytes()); } catch (MapperParsingException e) { assertThat(e.getCause(), instanceOf(NumberFormatException.class)); } // Unless the global ignore_malformed option is set to true Settings indexSettings = settingsBuilder().put("index.mapping.ignore_malformed", true).build(); defaultMapper = MapperTestUtils.newParser(indexSettings).parse(mapping); doc = defaultMapper.parse( "type", "1", XContentFactory.jsonBuilder().startObject().field("field3", "a").endObject().bytes()); assertThat(doc.rootDoc().getField("field3"), nullValue()); // This should still throw an exception, since field2 is specifically set to // ignore_malformed=false try { defaultMapper.parse( "type", "1", XContentFactory.jsonBuilder().startObject().field("field2", "a").endObject().bytes()); } catch (MapperParsingException e) { assertThat(e.getCause(), instanceOf(NumberFormatException.class)); } }
@Override public void doTestCoerce(String type) throws IOException { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field") .field("type", type) .endObject() .endObject() .endObject() .endObject() .string(); DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); assertEquals(mapping, mapper.mappingSource().toString()); ParsedDocument doc = mapper.parse( "test", "type", "1", XContentFactory.jsonBuilder().startObject().field("field", "123").endObject().bytes()); IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(2, fields.length); IndexableField pointField = fields[0]; assertEquals(1, pointField.fieldType().pointDimensionCount()); assertEquals(123, pointField.numericValue().doubleValue(), 0d); IndexableField dvField = fields[1]; assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType()); mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field") .field("type", type) .field("coerce", false) .endObject() .endObject() .endObject() .endObject() .string(); DocumentMapper mapper2 = parser.parse("type", new CompressedXContent(mapping)); assertEquals(mapping, mapper2.mappingSource().toString()); ThrowingRunnable runnable = () -> mapper2.parse( "test", "type", "1", XContentFactory.jsonBuilder() .startObject() .field("field", "123") .endObject() .bytes()); MapperParsingException e = expectThrows(MapperParsingException.class, runnable); assertThat(e.getCause().getMessage(), containsString("passed as String")); }
@Override public Mapper parse(ParseContext context) throws IOException { byte[] content = null; String contentType = null; int indexedChars = defaultIndexedChars; boolean langDetect = defaultLangDetect; String name = null; String language = null; XContentParser parser = context.parser(); XContentParser.Token token = parser.currentToken(); if (token == XContentParser.Token.VALUE_STRING) { content = parser.binaryValue(); } else { String currentFieldName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.VALUE_STRING) { if ("_content".equals(currentFieldName)) { content = parser.binaryValue(); } else if ("_content_type".equals(currentFieldName)) { contentType = parser.text(); } else if ("_name".equals(currentFieldName)) { name = parser.text(); } else if ("_language".equals(currentFieldName)) { language = parser.text(); } } else if (token == XContentParser.Token.VALUE_NUMBER) { if ("_indexed_chars".equals(currentFieldName) || "_indexedChars".equals(currentFieldName)) { indexedChars = parser.intValue(); } } else if (token == XContentParser.Token.VALUE_BOOLEAN) { if ("_detect_language".equals(currentFieldName) || "_detectLanguage".equals(currentFieldName)) { langDetect = parser.booleanValue(); } } } } // Throw clean exception when no content is provided Fix #23 if (content == null) { throw new MapperParsingException("No content is provided."); } Metadata metadata = new Metadata(); if (contentType != null) { metadata.add(Metadata.CONTENT_TYPE, contentType); } if (name != null) { metadata.add(Metadata.RESOURCE_NAME_KEY, name); } String parsedContent; try { parsedContent = TikaImpl.parse(content, metadata, indexedChars); } catch (Throwable e) { // #18: we could ignore errors when Tika does not parse data if (!ignoreErrors) { logger.trace("exception caught", e); throw new MapperParsingException( "Failed to extract [" + indexedChars + "] characters of text for [" + name + "] : " + e.getMessage(), e); } else { logger.debug( "Failed to extract [{}] characters of text for [{}]: [{}]", indexedChars, name, e.getMessage()); logger.trace("exception caught", e); } return null; } context = context.createExternalValueContext(parsedContent); contentMapper.parse(context); if (langDetect) { try { if (language != null) { metadata.add(Metadata.CONTENT_LANGUAGE, language); } else { LanguageIdentifier identifier = new LanguageIdentifier(parsedContent); language = identifier.getLanguage(); } context = context.createExternalValueContext(language); languageMapper.parse(context); } catch (Throwable t) { logger.debug("Cannot detect language: [{}]", t.getMessage()); } } if (name != null) { try { context = context.createExternalValueContext(name); nameMapper.parse(context); } catch (MapperParsingException e) { if (!ignoreErrors) throw e; if (logger.isDebugEnabled()) logger.debug( "Ignoring MapperParsingException catch while parsing name: [{}]", e.getMessage()); } } if (metadata.get(Metadata.DATE) != null) { try { context = context.createExternalValueContext(metadata.get(Metadata.DATE)); dateMapper.parse(context); } catch (MapperParsingException e) { if (!ignoreErrors) throw e; if (logger.isDebugEnabled()) logger.debug( "Ignoring MapperParsingException catch while parsing date: [{}]: [{}]", e.getMessage(), context.externalValue()); } } if (metadata.get(Metadata.TITLE) != null) { try { context = context.createExternalValueContext(metadata.get(Metadata.TITLE)); titleMapper.parse(context); } catch (MapperParsingException e) { if (!ignoreErrors) throw e; if (logger.isDebugEnabled()) logger.debug( "Ignoring MapperParsingException catch while parsing title: [{}]: [{}]", e.getMessage(), context.externalValue()); } } if (metadata.get(Metadata.AUTHOR) != null) { try { context = context.createExternalValueContext(metadata.get(Metadata.AUTHOR)); authorMapper.parse(context); } catch (MapperParsingException e) { if (!ignoreErrors) throw e; if (logger.isDebugEnabled()) logger.debug( "Ignoring MapperParsingException catch while parsing author: [{}]: [{}]", e.getMessage(), context.externalValue()); } } if (metadata.get(Metadata.KEYWORDS) != null) { try { context = context.createExternalValueContext(metadata.get(Metadata.KEYWORDS)); keywordsMapper.parse(context); } catch (MapperParsingException e) { if (!ignoreErrors) throw e; if (logger.isDebugEnabled()) logger.debug( "Ignoring MapperParsingException catch while parsing keywords: [{}]: [{}]", e.getMessage(), context.externalValue()); } } if (contentType == null) { contentType = metadata.get(Metadata.CONTENT_TYPE); } if (contentType != null) { try { context = context.createExternalValueContext(contentType); contentTypeMapper.parse(context); } catch (MapperParsingException e) { if (!ignoreErrors) throw e; if (logger.isDebugEnabled()) logger.debug( "Ignoring MapperParsingException catch while parsing content_type: [{}]: [{}]", e.getMessage(), context.externalValue()); } } int length = content.length; // If we have CONTENT_LENGTH from Tika we use it if (metadata.get(Metadata.CONTENT_LENGTH) != null) { length = Integer.parseInt(metadata.get(Metadata.CONTENT_LENGTH)); } try { context = context.createExternalValueContext(length); contentLengthMapper.parse(context); } catch (MapperParsingException e) { if (!ignoreErrors) throw e; if (logger.isDebugEnabled()) logger.debug( "Ignoring MapperParsingException catch while parsing content_length: [{}]: [{}]", e.getMessage(), context.externalValue()); } // multiFields.parse(this, context); return null; }