public void testMergingMappings() throws Exception { String enabledMapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("_field_names") .field("enabled", true) .endObject() .endObject() .endObject() .string(); String disabledMapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("_field_names") .field("enabled", false) .endObject() .endObject() .endObject() .string(); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); DocumentMapper mapperEnabled = parser.parse(enabledMapping); DocumentMapper mapperDisabled = parser.parse(disabledMapping); mapperEnabled.merge(mapperDisabled, DocumentMapper.MergeFlags.mergeFlags().simulate(false)); assertFalse(mapperEnabled.rootMapper(FieldNamesFieldMapper.class).enabled()); mapperEnabled = parser.parse(enabledMapping); mapperDisabled.merge(mapperEnabled, DocumentMapper.MergeFlags.mergeFlags().simulate(false)); assertTrue(mapperEnabled.rootMapper(FieldNamesFieldMapper.class).enabled()); }
public void testEmptyName() throws IOException { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("") .field("type", "ip") .endObject() .endObject() .endObject() .endObject() .string(); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> parser.parse("type", new CompressedXContent(mapping))); assertThat(e.getMessage(), containsString("name cannot be empty string")); // before 5.x Version oldVersion = VersionUtils.randomVersionBetween(getRandom(), Version.V_2_0_0, Version.V_2_3_5); Settings oldIndexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, oldVersion).build(); indexService = createIndex("test_old", oldIndexSettings); parser = indexService.mapperService().documentMapperParser(); DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping)); assertEquals(mapping, defaultMapper.mappingSource().string()); }
public void testThatDisablingWorksWhenMerging() throws Exception { String enabledMapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("_index") .field("enabled", true) .endObject() .endObject() .endObject() .string(); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); DocumentMapper enabledMapper = parser.parse(enabledMapping); String disabledMapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("_index") .field("enabled", false) .endObject() .endObject() .endObject() .string(); DocumentMapper disabledMapper = parser.parse(disabledMapping); enabledMapper.merge(disabledMapper.mapping(), false); assertThat(enabledMapper.indexMapper().enabled(), is(false)); }
public void testThatMergingFieldMappingAllowsDisabling() throws Exception { String mappingWithIndexEnabled = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("_index") .field("enabled", true) .endObject() .endObject() .endObject() .string(); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); DocumentMapper mapperEnabled = parser.parse(mappingWithIndexEnabled); String mappingWithIndexDisabled = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("_index") .field("enabled", false) .endObject() .endObject() .endObject() .string(); DocumentMapper mapperDisabled = parser.parse(mappingWithIndexDisabled); mapperEnabled.merge(mapperDisabled.mapping(), false); assertThat(mapperEnabled.IndexFieldMapper().enabled(), is(false)); }
public void testIgnoreMalformed() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field") .field("type", "ip") .endObject() .endObject() .endObject() .endObject() .string(); DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); assertEquals(mapping, mapper.mappingSource().toString()); ThrowingRunnable runnable = () -> mapper.parse( "test", "type", "1", XContentFactory.jsonBuilder() .startObject() .field("field", ":1") .endObject() .bytes()); MapperParsingException e = expectThrows(MapperParsingException.class, runnable); assertThat(e.getCause().getMessage(), containsString("':1' is not an IP string literal")); mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field") .field("type", "ip") .field("ignore_malformed", true) .endObject() .endObject() .endObject() .endObject() .string(); DocumentMapper mapper2 = parser.parse("type", new CompressedXContent(mapping)); ParsedDocument doc = mapper2.parse( "test", "type", "1", XContentFactory.jsonBuilder().startObject().field("field", ":1").endObject().bytes()); IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(0, fields.length); }
public void testIncludeInAll() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field") .field("type", "ip") .field("include_in_all", true) .endObject() .endObject() .endObject() .endObject() .string(); DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); assertEquals(mapping, mapper.mappingSource().toString()); ParsedDocument doc = mapper.parse( "test", "type", "1", XContentFactory.jsonBuilder().startObject().field("field", "::1").endObject().bytes()); IndexableField[] fields = doc.rootDoc().getFields("_all"); assertEquals(1, fields.length); assertEquals("::1", fields[0].stringValue()); mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field") .field("type", "ip") .endObject() .endObject() .endObject() .endObject() .string(); mapper = parser.parse("type", new CompressedXContent(mapping)); assertEquals(mapping, mapper.mappingSource().toString()); doc = mapper.parse( "test", "type", "1", XContentFactory.jsonBuilder().startObject().field("field", "::1").endObject().bytes()); fields = doc.rootDoc().getFields("_all"); assertEquals(0, fields.length); }
public void testAttributes() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/simple/test-mapping.json"); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); DocumentMapper docMapper = parser.parse(mapping); assertThat((String) docMapper.meta().get("param1"), equalTo("value1")); String builtMapping = docMapper.mappingSource().string(); DocumentMapper builtDocMapper = parser.parse(builtMapping); assertThat((String) builtDocMapper.meta().get("param1"), equalTo("value1")); }
@Before public void createMapper() throws IOException { DocumentMapperParser mapperParser = MapperTestUtils.newMapperService( createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()) .documentMapperParser(); String mapping = copyToStringFromClasspath( "/org/elasticsearch/index/mapper/attachment/test/unit/various-doc/test-mapping.json"); docMapper = mapperParser.parse("person", new CompressedXContent(mapping)); }
public void testNoDocValues() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field") .field("type", "ip") .field("doc_values", false) .endObject() .endObject() .endObject() .endObject() .string(); DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); assertEquals(mapping, mapper.mappingSource().toString()); ParsedDocument doc = mapper.parse( "test", "type", "1", XContentFactory.jsonBuilder().startObject().field("field", "::1").endObject().bytes()); IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(1, fields.length); IndexableField pointField = fields[0]; assertEquals(1, pointField.fieldType().pointDimensionCount()); assertEquals( new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))), pointField.binaryValue()); }
public void testNotIndexed() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field") .field("type", "ip") .field("index", false) .endObject() .endObject() .endObject() .endObject() .string(); DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); assertEquals(mapping, mapper.mappingSource().toString()); ParsedDocument doc = mapper.parse( "test", "type", "1", XContentFactory.jsonBuilder().startObject().field("field", "::1").endObject().bytes()); IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(1, fields.length); IndexableField dvField = fields[0]; assertEquals(DocValuesType.SORTED_SET, dvField.fieldType().docValuesType()); }
public void testSerializeDefaults() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field") .field("type", "ip") .endObject() .endObject() .endObject() .endObject() .string(); DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping)); IpFieldMapper mapper = (IpFieldMapper) docMapper.root().getMapper("field"); XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); mapper.doXContentBody(builder, true, ToXContent.EMPTY_PARAMS); String got = builder.endObject().string(); // it would be nice to check the entire serialized default mapper, but there are // a whole lot of bogus settings right now it picks up from calling super.doXContentBody... assertTrue(got, got.contains("\"null_value\":null")); assertTrue(got, got.contains("\"ignore_malformed\":false")); assertTrue(got, got.contains("\"include_in_all\":false")); }
@Test public void testMerge() throws IOException { String stage1Mapping = XContentFactory.jsonBuilder() .startObject() .startObject("person") .startObject("properties") .startObject("tc") .field("type", "token_count") .field("analyzer", "keyword") .endObject() .endObject() .endObject() .endObject() .string(); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); DocumentMapper stage1 = parser.parse(stage1Mapping); String stage2Mapping = XContentFactory.jsonBuilder() .startObject() .startObject("person") .startObject("properties") .startObject("tc") .field("type", "token_count") .field("analyzer", "standard") .endObject() .endObject() .endObject() .endObject() .string(); DocumentMapper stage2 = parser.parse(stage2Mapping); MergeResult mergeResult = stage1.merge(stage2.mapping(), true, false); assertThat(mergeResult.hasConflicts(), equalTo(false)); // Just simulated so merge hasn't happened yet assertThat( ((TokenCountFieldMapper) stage1.mappers().smartNameFieldMapper("tc")).analyzer(), equalTo("keyword")); mergeResult = stage1.merge(stage2.mapping(), false, false); assertThat(mergeResult.hasConflicts(), equalTo(false)); // Just simulated so merge hasn't happened yet assertThat( ((TokenCountFieldMapper) stage1.mappers().smartNameFieldMapper("tc")).analyzer(), equalTo("standard")); }
public void testTypeLevel() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("_all") .field("enabled", false) .endObject() .endObject() .endObject() .string(); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); assertThat(mapper.type(), equalTo("type")); assertThat(mapper.allFieldMapper().enabled(), equalTo(false)); }
public void testSimpleAllMappersWithReparse() throws Exception { DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json"); DocumentMapper docMapper = parser.parse(mapping); String builtMapping = docMapper.mappingSource().string(); // reparse it DocumentMapper builtDocMapper = parser.parse(builtMapping); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json"); Document doc = builtDocMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc(); AllField field = (AllField) doc.getField("_all"); AllEntries allEntries = field.getAllEntries(); assertThat(allEntries.fields().toString(), allEntries.fields().size(), equalTo(3)); assertThat(allEntries.fields().contains("address.last.location"), equalTo(true)); assertThat(allEntries.fields().contains("name.last"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true)); assertThat(field.fieldType().omitNorms(), equalTo(true)); }
public void testParseToJsonAndParse() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/simple/test-mapping.json"); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); DocumentMapper docMapper = parser.parse(mapping); String builtMapping = docMapper.mappingSource().string(); // reparse it DocumentMapper builtDocMapper = parser.parse(builtMapping); BytesReference json = new BytesArray( copyToBytesFromClasspath("/org/elasticsearch/index/mapper/simple/test1.json")); Document doc = builtDocMapper.parse("test", "person", "1", json).rootDoc(); assertThat( doc.get(docMapper.uidMapper().fieldType().names().indexName()), equalTo(Uid.createUid("person", "1"))); assertThat( doc.get(docMapper.mappers().getMapper("name.first").fieldType().names().indexName()), equalTo("shay")); }
// issue https://github.com/elasticsearch/elasticsearch/issues/5864 // test that RootObjectMapping still works public void testRootObjectMapperPropertiesDoNotCauseException() throws IOException { DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); String mapping = copyToStringFromClasspath( "/org/elasticsearch/index/mapper/all/type_dynamic_template_mapping.json"); parser.parse("test", mapping); mapping = copyToStringFromClasspath( "/org/elasticsearch/index/mapper/all/type_dynamic_date_formats_mapping.json"); parser.parse("test", mapping); mapping = copyToStringFromClasspath( "/org/elasticsearch/index/mapper/all/type_date_detection_mapping.json"); parser.parse("test", mapping); mapping = copyToStringFromClasspath( "/org/elasticsearch/index/mapper/all/type_numeric_detection_mapping.json"); parser.parse("test", mapping); }
@Test public void testSimpleMappings() throws Exception { String mapping = copyToStringFromClasspath( "/org/elasticsearch/index/mapper/attachment/test/unit/date/date-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); // Our mapping should be kept as a String assertThat( docMapper.mappers().fullName("file.date").mapper(), instanceOf(StringFieldMapper.class)); }
public void testHazardousFieldNames() throws Exception { IndexService indexService = createIndex("test"); DocumentMapperParser mapperParser = indexService.mapperService().documentMapperParser(); String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("foo.bar") .field("type", "string") .endObject() .endObject() .endObject() .string(); try { mapperParser.parse(mapping); fail("Mapping parse should have failed"); } catch (MapperParsingException e) { assertTrue(e.getMessage(), e.getMessage().contains("cannot contain '.'")); } }
public void testAcceptDocValuesFormat() throws IOException { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field") .field("type", "string") .field("doc_values_format", Codec.getDefault().docValuesFormat().getName()) .endObject() .endObject() .endObject() .endObject() .string(); int i = 0; for (Version v : VersionUtils.allVersions()) { if (v.onOrAfter(Version.V_2_0_0) == false) { // no need to test, we don't support upgrading from these versions continue; } IndexService indexService = createIndex( "test-" + i++, Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, v).build()); DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); try { parser.parse("type", new CompressedXContent(mapping)); if (v.onOrAfter(Version.V_2_0_0_beta1)) { fail("Elasticsearch 2.0 should not support custom postings formats"); } } catch (MapperParsingException e) { if (v.before(Version.V_2_0_0_beta1)) { // Elasticsearch 1.x should ignore custom postings formats throw e; } Assert.assertThat( e.getMessage(), containsString("unsupported parameters: [doc_values_format")); } } }
@Override public Mapper.Builder parse( String fieldName, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException { TextFieldMapper.Builder builder = new TextFieldMapper.Builder(fieldName); builder.fieldType().setIndexAnalyzer(parserContext.analysisService().defaultIndexAnalyzer()); builder .fieldType() .setSearchAnalyzer(parserContext.analysisService().defaultSearchAnalyzer()); builder .fieldType() .setSearchQuoteAnalyzer(parserContext.analysisService().defaultSearchQuoteAnalyzer()); parseTextField(builder, fieldName, node, parserContext); for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext(); ) { Map.Entry<String, Object> entry = iterator.next(); String propName = entry.getKey(); Object propNode = entry.getValue(); if (propName.equals("position_increment_gap")) { int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(propNode, -1); builder.positionIncrementGap(newPositionIncrementGap); iterator.remove(); } else if (propName.equals("fielddata")) { builder.fielddata(XContentMapValues.nodeBooleanValue(propNode)); iterator.remove(); } else if (propName.equals("eager_global_ordinals")) { builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode)); iterator.remove(); } else if (propName.equals("fielddata_frequency_filter")) { Map<?, ?> frequencyFilter = (Map<?, ?>) propNode; double minFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("min"), 0); double maxFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("max"), Integer.MAX_VALUE); int minSegmentSize = XContentMapValues.nodeIntegerValue(frequencyFilter.remove("min_segment_size"), 0); builder.fielddataFrequencyFilter(minFrequency, maxFrequency, minSegmentSize); DocumentMapperParser.checkNoRemainingFields( propName, frequencyFilter, parserContext.indexVersionCreated()); iterator.remove(); } } return builder; }
public void testStore() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field") .field("type", "ip") .field("store", true) .endObject() .endObject() .endObject() .endObject() .string(); DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); assertEquals(mapping, mapper.mappingSource().toString()); ParsedDocument doc = mapper.parse( "test", "type", "1", XContentFactory.jsonBuilder().startObject().field("field", "::1").endObject().bytes()); IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(3, fields.length); IndexableField pointField = fields[0]; assertEquals(1, pointField.fieldType().pointDimensionCount()); IndexableField dvField = fields[1]; assertEquals(DocValuesType.SORTED_SET, dvField.fieldType().docValuesType()); IndexableField storedField = fields[2]; assertTrue(storedField.fieldType().stored()); assertEquals( new BytesRef(InetAddressPoint.encode(InetAddress.getByName("::1"))), storedField.binaryValue()); }
public static void parseField( FieldMapper.Builder builder, String name, Map<String, Object> fieldNode, Mapper.TypeParser.ParserContext parserContext) { NamedAnalyzer indexAnalyzer = builder.fieldType().indexAnalyzer(); NamedAnalyzer searchAnalyzer = builder.fieldType().searchAnalyzer(); Version indexVersionCreated = parserContext.indexVersionCreated(); for (Iterator<Map.Entry<String, Object>> iterator = fieldNode.entrySet().iterator(); iterator.hasNext(); ) { Map.Entry<String, Object> entry = iterator.next(); final String propName = Strings.toUnderscoreCase(entry.getKey()); final Object propNode = entry.getValue(); if (propName.equals("index_name") && indexVersionCreated.before(Version.V_2_0_0_beta1)) { builder.indexName(propNode.toString()); iterator.remove(); } else if (propName.equals("store")) { builder.store(parseStore(name, propNode.toString())); iterator.remove(); } else if (propName.equals("index")) { parseIndex(name, propNode.toString(), builder); iterator.remove(); } else if (propName.equals("tokenized")) { builder.tokenized(nodeBooleanValue(propNode)); iterator.remove(); } else if (propName.equals(DOC_VALUES)) { builder.docValues(nodeBooleanValue(propNode)); iterator.remove(); } else if (propName.equals("term_vector")) { parseTermVector(name, propNode.toString(), builder); iterator.remove(); } else if (propName.equals("boost")) { builder.boost(nodeFloatValue(propNode)); iterator.remove(); } else if (propName.equals("store_term_vectors")) { builder.storeTermVectors(nodeBooleanValue(propNode)); iterator.remove(); } else if (propName.equals("store_term_vector_offsets")) { builder.storeTermVectorOffsets(nodeBooleanValue(propNode)); iterator.remove(); } else if (propName.equals("store_term_vector_positions")) { builder.storeTermVectorPositions(nodeBooleanValue(propNode)); iterator.remove(); } else if (propName.equals("store_term_vector_payloads")) { builder.storeTermVectorPayloads(nodeBooleanValue(propNode)); iterator.remove(); } else if (propName.equals(CQL_COLLECTION)) { switch (StringUtils.lowerCase(propNode.toString())) { case "list": builder.cqlCollection(CqlCollection.LIST); break; case "set": builder.cqlCollection(CqlCollection.SET); break; case "singleton": builder.cqlCollection(CqlCollection.SINGLETON); break; } iterator.remove(); } else if (propName.equals(CQL_STRUCT)) { switch (StringUtils.lowerCase(propNode.toString())) { case "map": builder.cqlStruct(CqlStruct.MAP); break; case "udt": builder.cqlStruct(CqlStruct.UDT); break; case "tuple": builder.cqlStruct(CqlStruct.TUPLE); break; } iterator.remove(); } else if (propName.equals(CQL_PARTIAL_UPDATE)) { builder.cqlPartialUpdate(nodeBooleanValue(propNode)); iterator.remove(); } else if (propName.equals("omit_norms")) { builder.omitNorms(nodeBooleanValue(propNode)); iterator.remove(); } else if (propName.equals("norms")) { final Map<String, Object> properties = nodeMapValue(propNode, "norms"); for (Iterator<Entry<String, Object>> propsIterator = properties.entrySet().iterator(); propsIterator.hasNext(); ) { Entry<String, Object> entry2 = propsIterator.next(); final String propName2 = Strings.toUnderscoreCase(entry2.getKey()); final Object propNode2 = entry2.getValue(); if (propName2.equals("enabled")) { builder.omitNorms(!nodeBooleanValue(propNode2)); propsIterator.remove(); } else if (propName2.equals(Loading.KEY)) { builder.normsLoading(Loading.parse(nodeStringValue(propNode2, null), null)); propsIterator.remove(); } } DocumentMapperParser.checkNoRemainingFields( propName, properties, parserContext.indexVersionCreated()); iterator.remove(); } else if (propName.equals("omit_term_freq_and_positions")) { final IndexOptions op = nodeBooleanValue(propNode) ? IndexOptions.DOCS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; if (indexVersionCreated.onOrAfter(Version.V_1_0_0_RC2)) { throw new ElasticsearchParseException( "'omit_term_freq_and_positions' is not supported anymore - use ['index_options' : 'docs'] instead"); } // deprecated option for BW compat builder.indexOptions(op); iterator.remove(); } else if (propName.equals("index_options")) { builder.indexOptions(nodeIndexOptionValue(propNode)); iterator.remove(); } else if (propName.equals("analyzer") || // for backcompat, reading old indexes, remove for v3.0 propName.equals("index_analyzer") && indexVersionCreated.before(Version.V_2_0_0_beta1)) { NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString()); if (analyzer == null) { throw new MapperParsingException( "analyzer [" + propNode.toString() + "] not found for field [" + name + "]"); } indexAnalyzer = analyzer; iterator.remove(); } else if (propName.equals("search_analyzer")) { NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString()); if (analyzer == null) { throw new MapperParsingException( "analyzer [" + propNode.toString() + "] not found for field [" + name + "]"); } searchAnalyzer = analyzer; iterator.remove(); } else if (propName.equals("include_in_all")) { builder.includeInAll(nodeBooleanValue(propNode)); iterator.remove(); } else if (propName.equals("postings_format") && indexVersionCreated.before(Version.V_2_0_0_beta1)) { // ignore for old indexes iterator.remove(); } else if (propName.equals("doc_values_format") && indexVersionCreated.before(Version.V_2_0_0_beta1)) { // ignore for old indexes iterator.remove(); } else if (propName.equals("similarity")) { builder.similarity(parserContext.similarityLookupService().similarity(propNode.toString())); iterator.remove(); } else if (propName.equals("fielddata")) { final Settings settings = Settings.builder() .put(SettingsLoader.Helper.loadNestedFromMap(nodeMapValue(propNode, "fielddata"))) .build(); builder.fieldDataSettings(settings); iterator.remove(); } else if (propName.equals("copy_to")) { if (parserContext.isWithinMultiField()) { if (indexVersionCreated.after(Version.V_2_1_0) || (indexVersionCreated.after(Version.V_2_0_1) && indexVersionCreated.before(Version.V_2_1_0))) { throw new MapperParsingException( "copy_to in multi fields is not allowed. Found the copy_to in field [" + name + "] which is within a multi field."); } else { ESLoggerFactory.getLogger("mapping [" + parserContext.type() + "]") .warn( "Found a copy_to in field [" + name + "] which is within a multi field. This feature has been removed and the copy_to will be ignored."); // we still parse this, otherwise the message will only appear once and the copy_to // removed. After that it will appear again. Better to have it always. } } parseCopyFields(propNode, builder); iterator.remove(); } } if (indexAnalyzer == null) { if (searchAnalyzer != null) { // If the index was created before 2.0 then we are trying to upgrade the mappings so use the // default indexAnalyzer // instead of throwing an exception so the user is able to upgrade if (parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { indexAnalyzer = parserContext.analysisService().defaultIndexAnalyzer(); } else { throw new MapperParsingException( "analyzer on field [" + name + "] must be set when search_analyzer is set"); } } } else if (searchAnalyzer == null) { searchAnalyzer = indexAnalyzer; } builder.indexAnalyzer(indexAnalyzer); builder.searchAnalyzer(searchAnalyzer); }
public void testRandom() throws Exception { boolean omitNorms = false; boolean stored = false; boolean enabled = true; boolean tv_stored = false; boolean tv_payloads = false; boolean tv_offsets = false; boolean tv_positions = false; String similarity = null; XContentBuilder mappingBuilder = jsonBuilder(); mappingBuilder.startObject().startObject("test"); List<Tuple<String, Boolean>> booleanOptionList = new ArrayList<>(); boolean allDefault = true; if (frequently()) { allDefault = false; mappingBuilder.startObject("_all"); if (randomBoolean()) { booleanOptionList.add(new Tuple<>("omit_norms", omitNorms = randomBoolean())); } if (randomBoolean()) { booleanOptionList.add(new Tuple<>("store", stored = randomBoolean())); } if (randomBoolean()) { booleanOptionList.add(new Tuple<>("store_term_vectors", tv_stored = randomBoolean())); } if (randomBoolean()) { booleanOptionList.add(new Tuple<>("enabled", enabled = randomBoolean())); } if (randomBoolean()) { booleanOptionList.add( new Tuple<>("store_term_vector_offsets", tv_offsets = randomBoolean())); } if (randomBoolean()) { booleanOptionList.add( new Tuple<>("store_term_vector_positions", tv_positions = randomBoolean())); } if (randomBoolean()) { booleanOptionList.add( new Tuple<>("store_term_vector_payloads", tv_payloads = randomBoolean())); } Collections.shuffle(booleanOptionList, getRandom()); for (Tuple<String, Boolean> option : booleanOptionList) { mappingBuilder.field(option.v1(), option.v2().booleanValue()); } tv_stored |= tv_positions || tv_payloads || tv_offsets; if (randomBoolean()) { mappingBuilder.field("similarity", similarity = randomBoolean() ? "BM25" : "TF/IDF"); } mappingBuilder.endObject(); } DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); String mapping = mappingBuilder.endObject().endObject().bytes().toUtf8(); logger.info(mapping); DocumentMapper docMapper = parser.parse(mapping); String builtMapping = docMapper.mappingSource().string(); // reparse it DocumentMapper builtDocMapper = parser.parse(builtMapping); byte[] json = jsonBuilder() .startObject() .field("foo", "bar") .field("foobar", "foobar") .endObject() .bytes() .toBytes(); Document doc = builtDocMapper.parse("test", "test", "1", new BytesArray(json)).rootDoc(); AllField field = (AllField) doc.getField("_all"); if (enabled) { assertThat(field.fieldType().omitNorms(), equalTo(omitNorms)); assertThat(field.fieldType().stored(), equalTo(stored)); assertThat(field.fieldType().storeTermVectorOffsets(), equalTo(tv_offsets)); assertThat(field.fieldType().storeTermVectorPayloads(), equalTo(tv_payloads)); assertThat(field.fieldType().storeTermVectorPositions(), equalTo(tv_positions)); assertThat(field.fieldType().storeTermVectors(), equalTo(tv_stored)); AllEntries allEntries = field.getAllEntries(); assertThat(allEntries.fields().size(), equalTo(2)); assertThat(allEntries.fields().contains("foobar"), equalTo(true)); assertThat(allEntries.fields().contains("foo"), equalTo(true)); if (!stored) { assertThat(field.stringValue(), nullValue()); } String text = stored ? field.stringValue() : "bar foobar"; assertThat(text.trim(), equalTo(allEntries.buildText().trim())); } else { assertThat(field, nullValue()); } if (similarity == null || similarity.equals("TF/IDF")) { assertThat(builtDocMapper.allFieldMapper().fieldType().similarity(), nullValue()); } else { assertThat( similarity, equalTo(builtDocMapper.allFieldMapper().fieldType().similarity().name())); } if (allDefault) { BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(0); XContentBuilder b = new XContentBuilder(XContentType.JSON.xContent(), bytesStreamOutput); XContentBuilder xContentBuilder = builtDocMapper.allFieldMapper().toXContent(b, ToXContent.EMPTY_PARAMS); xContentBuilder.flush(); assertThat(bytesStreamOutput.size(), equalTo(0)); } }
@Override public Mapper.Builder<?, ?> parse( String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException { ContentPath.Type pathType = null; FieldMapper.Builder mainFieldBuilder = null; List<FieldMapper.Builder> fields = null; String firstType = null; for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext(); ) { Map.Entry<String, Object> entry = iterator.next(); String fieldName = Strings.toUnderscoreCase(entry.getKey()); Object fieldNode = entry.getValue(); if (fieldName.equals("path") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { pathType = parsePathType(name, fieldNode.toString()); iterator.remove(); } else if (fieldName.equals("fields")) { Map<String, Object> fieldsNode = (Map<String, Object>) fieldNode; for (Iterator<Map.Entry<String, Object>> fieldsIterator = fieldsNode.entrySet().iterator(); fieldsIterator.hasNext(); ) { Map.Entry<String, Object> entry1 = fieldsIterator.next(); String propName = entry1.getKey(); Map<String, Object> propNode = (Map<String, Object>) entry1.getValue(); String type; Object typeNode = propNode.get("type"); if (typeNode != null) { type = typeNode.toString(); if (firstType == null) { firstType = type; } } else { throw new MapperParsingException( "no type specified for property [" + propName + "]"); } Mapper.TypeParser typeParser = parserContext.typeParser(type); if (typeParser == null) { throw new MapperParsingException( "no handler for type [" + type + "] declared on field [" + fieldName + "]"); } if (propName.equals(name)) { mainFieldBuilder = (FieldMapper.Builder) typeParser.parse(propName, propNode, parserContext); fieldsIterator.remove(); } else { if (fields == null) { fields = new ArrayList<>(2); } fields.add( (FieldMapper.Builder) typeParser.parse(propName, propNode, parserContext)); fieldsIterator.remove(); } } fieldsNode.remove("type"); DocumentMapperParser.checkNoRemainingFields( fieldName, fieldsNode, parserContext.indexVersionCreated()); iterator.remove(); } } if (mainFieldBuilder == null) { if (fields == null) { // No fields at all were specified in multi_field, so lets return a non indexed string // field. return new StringFieldMapper.Builder(name).index(false); } Mapper.TypeParser typeParser = parserContext.typeParser(firstType); if (typeParser == null) { // The first multi field's type is unknown mainFieldBuilder = new StringFieldMapper.Builder(name).index(false); } else { Mapper.Builder substitute = typeParser.parse(name, Collections.<String, Object>emptyMap(), parserContext); if (substitute instanceof FieldMapper.Builder) { mainFieldBuilder = ((FieldMapper.Builder) substitute).index(false); } else { // The first multi isn't a core field type mainFieldBuilder = new StringFieldMapper.Builder(name).index(false); } } } if (fields != null && pathType != null) { for (Mapper.Builder field : fields) { mainFieldBuilder.addMultiField(field); } mainFieldBuilder.multiFieldPathType(pathType); } else if (fields != null) { for (Mapper.Builder field : fields) { mainFieldBuilder.addMultiField(field); } } else if (pathType != null) { mainFieldBuilder.multiFieldPathType(pathType); } return mainFieldBuilder; }
@Test public void testMultipleDocsEncryptedLast() throws IOException { DocumentMapperParser mapperParser = new DocumentMapperParser( new Index("test"), ImmutableSettings.EMPTY, new AnalysisService(new Index("test")), null, null, null); mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/multipledocs/test-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); byte[] html = copyToBytesFromClasspath( "/org/elasticsearch/index/mapper/xcontent/htmlWithValidDateMeta.html"); byte[] pdf = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/encrypted.pdf"); BytesReference json = jsonBuilder() .startObject() .field("_id", 1) .field("file1", html) .field("file2", pdf) .endObject() .bytes(); ParseContext.Document doc = docMapper.parse(json).rootDoc(); assertThat( doc.get(docMapper.mappers().smartName("file1").mapper().names().indexName()), containsString("World")); assertThat( doc.get(docMapper.mappers().smartName("file1.title").mapper().names().indexName()), equalTo("Hello")); assertThat( doc.get(docMapper.mappers().smartName("file1.author").mapper().names().indexName()), equalTo("kimchy")); assertThat( doc.get(docMapper.mappers().smartName("file1.keywords").mapper().names().indexName()), equalTo("elasticsearch,cool,bonsai")); assertThat( doc.get(docMapper.mappers().smartName("file1.content_type").mapper().names().indexName()), equalTo("text/html; charset=ISO-8859-1")); assertThat( doc.getField( docMapper.mappers().smartName("file1.content_length").mapper().names().indexName()) .numericValue() .longValue(), is(344L)); assertThat( doc.get(docMapper.mappers().smartName("file2").mapper().names().indexName()), nullValue()); assertThat( doc.get(docMapper.mappers().smartName("file2.title").mapper().names().indexName()), nullValue()); assertThat( doc.get(docMapper.mappers().smartName("file2.author").mapper().names().indexName()), nullValue()); assertThat( doc.get(docMapper.mappers().smartName("file2.keywords").mapper().names().indexName()), nullValue()); assertThat( doc.get(docMapper.mappers().smartName("file2.content_type").mapper().names().indexName()), nullValue()); assertThat( doc.getField( docMapper.mappers().smartName("file2.content_length").mapper().names().indexName()), nullValue()); }
@Before public void setupMapperParser() { mapperParser = MapperTestUtils.newMapperParser(); mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); }
public void testExternalValues() throws Exception { Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); IndexService indexService = createIndex("test", settings); MapperRegistry mapperRegistry = new MapperRegistry( Collections.singletonMap( ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")), Collections.singletonMap( ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser())); DocumentMapperParser parser = new DocumentMapperParser( indexService.getIndexSettings(), indexService.mapperService(), indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext); DocumentMapper documentMapper = parser.parse( "type", new CompressedXContent( XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject(ExternalMetadataMapper.CONTENT_TYPE) .endObject() .startObject("properties") .startObject("field") .field("type", "external") .endObject() .endObject() .endObject() .endObject() .string())); ParsedDocument doc = documentMapper.parse( "test", "type", "1", XContentFactory.jsonBuilder().startObject().field("field", "1234").endObject().bytes()); assertThat(doc.rootDoc().getField("field.bool"), notNullValue()); assertThat(doc.rootDoc().getField("field.bool").stringValue(), is("T")); assertThat(doc.rootDoc().getField("field.point"), notNullValue()); if (version.before(Version.V_2_2_0)) { assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0")); } else { assertThat( Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoEncodingUtils.mortonHash(51.0, 42.0))); } assertThat(doc.rootDoc().getField("field.shape"), notNullValue()); assertThat(doc.rootDoc().getField("field.field"), notNullValue()); assertThat(doc.rootDoc().getField("field.field").stringValue(), is("foo")); assertThat( doc.rootDoc().getField(ExternalMetadataMapper.FIELD_NAME).stringValue(), is(ExternalMetadataMapper.FIELD_VALUE)); }
protected static void parseProperties( ObjectMapper.Builder objBuilder, Map<String, Object> propsNode, ParserContext parserContext) { Iterator<Map.Entry<String, Object>> iterator = propsNode.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry<String, Object> entry = iterator.next(); String propName = entry.getKey(); // Should accept empty arrays, as a work around for when the // user can't provide an empty Map. (PHP for example) boolean isEmptyList = entry.getValue() instanceof List && ((List<?>) entry.getValue()).isEmpty(); if (entry.getValue() instanceof Map) { @SuppressWarnings("unchecked") Map<String, Object> propNode = (Map<String, Object>) entry.getValue(); String type; Object typeNode = propNode.get("type"); if (typeNode != null) { type = typeNode.toString(); } else { // lets see if we can derive this... if (propNode.get("properties") != null) { type = ObjectMapper.CONTENT_TYPE; } else if (propNode.size() == 1 && propNode.get("enabled") != null) { // if there is a single property with the enabled // flag on it, make it an object // (usually, setting enabled to false to not index // any type, including core values, which type = ObjectMapper.CONTENT_TYPE; } else { throw new MapperParsingException("No type specified for property [" + propName + "]"); } } Mapper.TypeParser typeParser = parserContext.typeParser(type); if (typeParser == null) { throw new MapperParsingException( "No handler for type [" + type + "] declared on field [" + propName + "]"); } objBuilder.add(typeParser.parse(propName, propNode, parserContext)); propNode.remove("type"); DocumentMapperParser.checkNoRemainingFields( propName, propNode, parserContext.indexVersionCreated()); iterator.remove(); } else if (isEmptyList) { iterator.remove(); } else { throw new MapperParsingException( "Expected map for property [fields] on field [" + propName + "] but got a " + propName.getClass()); } } DocumentMapperParser.checkNoRemainingFields( propsNode, parserContext.indexVersionCreated(), "DocType mapping definition has unsupported parameters: "); }
public static boolean parseMultiField( FieldMapper.Builder builder, String name, Mapper.TypeParser.ParserContext parserContext, String propName, Object propNode) { parserContext = parserContext.createMultiFieldContext(parserContext); if (propName.equals("path") && parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { builder.multiFieldPathType(parsePathType(name, propNode.toString())); return true; } else if (propName.equals("fields")) { final Map<String, Object> multiFieldsPropNodes; if (propNode instanceof List && ((List<?>) propNode).isEmpty()) { multiFieldsPropNodes = Collections.emptyMap(); } else if (propNode instanceof Map) { multiFieldsPropNodes = (Map<String, Object>) propNode; } else { throw new MapperParsingException( "expected map for property [fields] on field [" + propNode + "] or " + "[" + propName + "] but got a " + propNode.getClass()); } for (Map.Entry<String, Object> multiFieldEntry : multiFieldsPropNodes.entrySet()) { String multiFieldName = multiFieldEntry.getKey(); if (multiFieldName.contains(".")) { throw new MapperParsingException( "Field name [" + multiFieldName + "] which is a multi field of [" + name + "] cannot contain '.'"); } if (!(multiFieldEntry.getValue() instanceof Map)) { throw new MapperParsingException( "illegal field [" + multiFieldName + "], only fields can be specified inside fields"); } @SuppressWarnings("unchecked") Map<String, Object> multiFieldNodes = (Map<String, Object>) multiFieldEntry.getValue(); String type; Object typeNode = multiFieldNodes.get("type"); if (typeNode != null) { type = typeNode.toString(); } else { throw new MapperParsingException( "no type specified for property [" + multiFieldName + "]"); } if (type.equals(ObjectMapper.CONTENT_TYPE) || type.equals(ObjectMapper.NESTED_CONTENT_TYPE)) { throw new MapperParsingException("Type [" + type + "] cannot be used in multi field"); } Mapper.TypeParser typeParser = parserContext.typeParser(type); if (typeParser == null) { throw new MapperParsingException( "no handler for type [" + type + "] declared on field [" + multiFieldName + "]"); } builder.addMultiField(typeParser.parse(multiFieldName, multiFieldNodes, parserContext)); multiFieldNodes.remove("type"); DocumentMapperParser.checkNoRemainingFields( propName, multiFieldNodes, parserContext.indexVersionCreated()); } return true; } return false; }
public void testNullValue() throws IOException { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field") .field("type", "ip") .endObject() .endObject() .endObject() .endObject() .string(); DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); assertEquals(mapping, mapper.mappingSource().toString()); ParsedDocument doc = mapper.parse( "test", "type", "1", XContentFactory.jsonBuilder().startObject().nullField("field").endObject().bytes()); assertArrayEquals(new IndexableField[0], doc.rootDoc().getFields("field")); mapping = XContentFactory.jsonBuilder() .startObject() .startObject("type") .startObject("properties") .startObject("field") .field("type", "ip") .field("null_value", "::1") .endObject() .endObject() .endObject() .endObject() .string(); mapper = parser.parse("type", new CompressedXContent(mapping)); assertEquals(mapping, mapper.mappingSource().toString()); doc = mapper.parse( "test", "type", "1", XContentFactory.jsonBuilder().startObject().nullField("field").endObject().bytes()); IndexableField[] fields = doc.rootDoc().getFields("field"); assertEquals(2, fields.length); IndexableField pointField = fields[0]; assertEquals(1, pointField.fieldType().pointDimensionCount()); assertEquals(16, pointField.fieldType().pointNumBytes()); assertFalse(pointField.fieldType().stored()); assertEquals( new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))), pointField.binaryValue()); IndexableField dvField = fields[1]; assertEquals(DocValuesType.SORTED_SET, dvField.fieldType().docValuesType()); assertEquals( new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))), dvField.binaryValue()); assertFalse(dvField.fieldType().stored()); }