// #6187: if no fields were boosted, we shouldn't use AllTokenStream public void testNoBoost() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/noboost-mapping.json"); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json"); Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc(); AllField field = (AllField) doc.getField("_all"); // no fields have boost, so we should not see AllTokenStream: assertThat( field.tokenStream(docMapper.mappers().indexAnalyzer(), null), Matchers.not(Matchers.instanceOf(AllTokenStream.class))); }
public void testAllMappersNoBoost() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/noboost-mapping.json"); IndexService index = createIndex("test"); DocumentMapper docMapper = index.mapperService().documentMapperParser().parse(mapping); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json"); Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc(); AllField field = (AllField) doc.getField("_all"); AllEntries allEntries = field.getAllEntries(); assertThat(allEntries.fields().size(), equalTo(3)); assertThat(allEntries.fields().contains("address.last.location"), equalTo(true)); assertThat(allEntries.fields().contains("name.last"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true)); assertThat(field.fieldType().omitNorms(), equalTo(false)); }
public void testSimple() throws Exception { String mapping = copyToStringFromClasspath( "/org/elasticsearch/index/mapper/dynamictemplate/genericstore/test-mapping.json"); IndexService index = createIndex("test"); client().admin().indices().preparePutMapping("test").setType("person").setSource(mapping).get(); DocumentMapper docMapper = index.mapperService().documentMapper("person"); byte[] json = copyToBytesFromClasspath( "/org/elasticsearch/index/mapper/dynamictemplate/genericstore/test-data.json"); ParsedDocument parsedDoc = docMapper.parse("test", "person", "1", new BytesArray(json)); client() .admin() .indices() .preparePutMapping("test") .setType("person") .setSource(parsedDoc.dynamicMappingsUpdate().toString()) .get(); docMapper = index.mapperService().documentMapper("person"); Document doc = parsedDoc.rootDoc(); IndexableField f = doc.getField("name"); assertThat(f.name(), equalTo("name")); assertThat(f.stringValue(), equalTo("some name")); assertThat(f.fieldType().stored(), equalTo(true)); FieldMapper fieldMapper = docMapper.mappers().getMapper("name"); assertThat(fieldMapper.fieldType().stored(), equalTo(true)); f = doc.getField("age"); assertThat(f.name(), equalTo("age")); assertThat(f.fieldType().stored(), equalTo(true)); fieldMapper = docMapper.mappers().getMapper("age"); assertThat(fieldMapper.fieldType().stored(), equalTo(true)); }
// #6187: if _all doesn't index positions then we never use AllTokenStream, even if some fields // have boost public void testBoostWithOmitPositions() throws Exception { String mapping = copyToStringFromClasspath( "/org/elasticsearch/index/mapper/all/mapping_boost_omit_positions_on_all.json"); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json"); Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc(); AllField field = (AllField) doc.getField("_all"); // _all field omits positions, so we should not get AllTokenStream even though fields are // boosted assertThat( field.tokenStream(docMapper.mappers().indexAnalyzer(), null), Matchers.not(Matchers.instanceOf(AllTokenStream.class))); }
public void testSimpleAllMappersWithStore() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/store-mapping.json"); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json"); Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc(); AllField field = (AllField) doc.getField("_all"); AllEntries allEntries = field.getAllEntries(); assertThat(allEntries.fields().size(), equalTo(2)); assertThat(allEntries.fields().contains("name.last"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true)); String text = field.stringValue(); assertThat(text, equalTo(allEntries.buildText())); assertThat(field.fieldType().omitNorms(), equalTo(false)); }
public void testSimpleAllMappersWithReparse() throws Exception { DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json"); DocumentMapper docMapper = parser.parse(mapping); String builtMapping = docMapper.mappingSource().string(); // reparse it DocumentMapper builtDocMapper = parser.parse(builtMapping); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json"); Document doc = builtDocMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc(); AllField field = (AllField) doc.getField("_all"); AllEntries allEntries = field.getAllEntries(); assertThat(allEntries.fields().toString(), allEntries.fields().size(), equalTo(3)); assertThat(allEntries.fields().contains("address.last.location"), equalTo(true)); assertThat(allEntries.fields().contains("name.last"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true)); assertThat(field.fieldType().omitNorms(), equalTo(true)); }
@Test public void testSimpleAllMappers() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json"); DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json"); Document doc = docMapper.parse(new BytesArray(json)).rootDoc(); AllField field = (AllField) doc.getField("_all"); AllEntries allEntries = ((AllTokenStream) field.tokenStream(docMapper.mappers().indexAnalyzer())).allEntries(); assertThat(allEntries.fields().size(), equalTo(3)); assertThat(allEntries.fields().contains("address.last.location"), equalTo(true)); assertThat(allEntries.fields().contains("name.last"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true)); FieldMapper mapper = docMapper.mappers().smartNameFieldMapper("_all"); assertThat(field.fieldType().omitNorms(), equalTo(true)); assertThat( mapper.queryStringTermQuery(new Term("_all", "foobar")), Matchers.instanceOf(AllTermQuery.class)); }
@Test public void testSimpleAllMappersWithReparse() throws Exception { String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json"); DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping); String builtMapping = docMapper.mappingSource().string(); // reparse it DocumentMapper builtDocMapper = MapperTestUtils.newParser().parse(builtMapping); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json"); Document doc = builtDocMapper.parse(new BytesArray(json)).rootDoc(); AllField field = (AllField) doc.getField("_all"); AllEntries allEntries = ((AllTokenStream) field.tokenStream(docMapper.mappers().indexAnalyzer())).allEntries(); assertThat(allEntries.fields().size(), equalTo(3)); assertThat(allEntries.fields().contains("address.last.location"), equalTo(true)); assertThat(allEntries.fields().contains("name.last"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true)); assertThat(field.fieldType().omitNorms(), equalTo(true)); }
public void testAllMappersTermQuery() throws Exception { String mapping = copyToStringFromClasspath( "/org/elasticsearch/index/mapper/all/mapping_omit_positions_on_all.json"); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json"); Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc(); AllField field = (AllField) doc.getField("_all"); AllEntries allEntries = field.getAllEntries(); assertThat(allEntries.fields().size(), equalTo(3)); assertThat(allEntries.fields().contains("address.last.location"), equalTo(true)); assertThat(allEntries.fields().contains("name.last"), equalTo(true)); assertThat(allEntries.fields().contains("simple1"), equalTo(true)); AllFieldMapper mapper = docMapper.allFieldMapper(); assertThat(field.fieldType().omitNorms(), equalTo(false)); assertThat( mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")), Matchers.instanceOf(AllTermQuery.class)); }
public void testMultiField_includeInAllSetToFalse() throws IOException { String mapping = copyToStringFromClasspath( "/org/elasticsearch/index/mapper/all/multifield-mapping_include_in_all_set_to_false.json"); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); XContentBuilder builder = XContentFactory.jsonBuilder(); builder .startObject() .field("foo") .startObject() .field("bar", "Elasticsearch rules!") .endObject() .endObject(); Document doc = docMapper.parse("test", "test", "1", builder.bytes()).rootDoc(); AllField field = (AllField) doc.getField("_all"); AllEntries allEntries = field.getAllEntries(); assertThat(allEntries.fields(), empty()); }
public void testRandom() throws Exception { boolean omitNorms = false; boolean stored = false; boolean enabled = true; boolean tv_stored = false; boolean tv_payloads = false; boolean tv_offsets = false; boolean tv_positions = false; String similarity = null; XContentBuilder mappingBuilder = jsonBuilder(); mappingBuilder.startObject().startObject("test"); List<Tuple<String, Boolean>> booleanOptionList = new ArrayList<>(); boolean allDefault = true; if (frequently()) { allDefault = false; mappingBuilder.startObject("_all"); if (randomBoolean()) { booleanOptionList.add(new Tuple<>("omit_norms", omitNorms = randomBoolean())); } if (randomBoolean()) { booleanOptionList.add(new Tuple<>("store", stored = randomBoolean())); } if (randomBoolean()) { booleanOptionList.add(new Tuple<>("store_term_vectors", tv_stored = randomBoolean())); } if (randomBoolean()) { booleanOptionList.add(new Tuple<>("enabled", enabled = randomBoolean())); } if (randomBoolean()) { booleanOptionList.add( new Tuple<>("store_term_vector_offsets", tv_offsets = randomBoolean())); } if (randomBoolean()) { booleanOptionList.add( new Tuple<>("store_term_vector_positions", tv_positions = randomBoolean())); } if (randomBoolean()) { booleanOptionList.add( new Tuple<>("store_term_vector_payloads", tv_payloads = randomBoolean())); } Collections.shuffle(booleanOptionList, getRandom()); for (Tuple<String, Boolean> option : booleanOptionList) { mappingBuilder.field(option.v1(), option.v2().booleanValue()); } tv_stored |= tv_positions || tv_payloads || tv_offsets; if (randomBoolean()) { mappingBuilder.field("similarity", similarity = randomBoolean() ? "BM25" : "TF/IDF"); } mappingBuilder.endObject(); } DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); String mapping = mappingBuilder.endObject().endObject().bytes().toUtf8(); logger.info(mapping); DocumentMapper docMapper = parser.parse(mapping); String builtMapping = docMapper.mappingSource().string(); // reparse it DocumentMapper builtDocMapper = parser.parse(builtMapping); byte[] json = jsonBuilder() .startObject() .field("foo", "bar") .field("foobar", "foobar") .endObject() .bytes() .toBytes(); Document doc = builtDocMapper.parse("test", "test", "1", new BytesArray(json)).rootDoc(); AllField field = (AllField) doc.getField("_all"); if (enabled) { assertThat(field.fieldType().omitNorms(), equalTo(omitNorms)); assertThat(field.fieldType().stored(), equalTo(stored)); assertThat(field.fieldType().storeTermVectorOffsets(), equalTo(tv_offsets)); assertThat(field.fieldType().storeTermVectorPayloads(), equalTo(tv_payloads)); assertThat(field.fieldType().storeTermVectorPositions(), equalTo(tv_positions)); assertThat(field.fieldType().storeTermVectors(), equalTo(tv_stored)); AllEntries allEntries = field.getAllEntries(); assertThat(allEntries.fields().size(), equalTo(2)); assertThat(allEntries.fields().contains("foobar"), equalTo(true)); assertThat(allEntries.fields().contains("foo"), equalTo(true)); if (!stored) { assertThat(field.stringValue(), nullValue()); } String text = stored ? field.stringValue() : "bar foobar"; assertThat(text.trim(), equalTo(allEntries.buildText().trim())); } else { assertThat(field, nullValue()); } if (similarity == null || similarity.equals("TF/IDF")) { assertThat(builtDocMapper.allFieldMapper().fieldType().similarity(), nullValue()); } else { assertThat( similarity, equalTo(builtDocMapper.allFieldMapper().fieldType().similarity().name())); } if (allDefault) { BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(0); XContentBuilder b = new XContentBuilder(XContentType.JSON.xContent(), bytesStreamOutput); XContentBuilder xContentBuilder = builtDocMapper.allFieldMapper().toXContent(b, ToXContent.EMPTY_PARAMS); xContentBuilder.flush(); assertThat(bytesStreamOutput.size(), equalTo(0)); } }
public void testFieldLevelBoost() throws Exception { String mapping = XContentFactory.jsonBuilder() .startObject() .startObject("person") .startObject("properties") .startObject("str_field") .field("type", "string") .endObject() .startObject("int_field") .field("type", "integer") .startObject("norms") .field("enabled", true) .endObject() .endObject() .startObject("byte_field") .field("type", "byte") .startObject("norms") .field("enabled", true) .endObject() .endObject() .startObject("date_field") .field("type", "date") .startObject("norms") .field("enabled", true) .endObject() .endObject() .startObject("double_field") .field("type", "double") .startObject("norms") .field("enabled", true) .endObject() .endObject() .startObject("float_field") .field("type", "float") .startObject("norms") .field("enabled", true) .endObject() .endObject() .startObject("long_field") .field("type", "long") .startObject("norms") .field("enabled", true) .endObject() .endObject() .startObject("short_field") .field("type", "short") .startObject("norms") .field("enabled", true) .endObject() .endObject() .string(); DocumentMapper docMapper = createIndex("test") .mapperService() .documentMapperParser() .parse("person", new CompressedXContent(mapping)); BytesReference json = XContentFactory.jsonBuilder() .startObject() .startObject("str_field") .field("boost", 2.0) .field("value", "some name") .endObject() .startObject("int_field") .field("boost", 3.0) .field("value", 10) .endObject() .startObject("byte_field") .field("boost", 4.0) .field("value", 20) .endObject() .startObject("date_field") .field("boost", 5.0) .field("value", "2012-01-10") .endObject() .startObject("double_field") .field("boost", 6.0) .field("value", 30.0) .endObject() .startObject("float_field") .field("boost", 7.0) .field("value", 40.0) .endObject() .startObject("long_field") .field("boost", 8.0) .field("value", 50) .endObject() .startObject("short_field") .field("boost", 9.0) .field("value", 60) .endObject() .bytes(); Document doc = docMapper.parse("test", "person", "1", json).rootDoc(); IndexableField f = doc.getField("str_field"); assertThat((double) f.boost(), closeTo(2.0, 0.001)); f = doc.getField("int_field"); assertThat((double) f.boost(), closeTo(3.0, 0.001)); f = doc.getField("byte_field"); assertThat((double) f.boost(), closeTo(4.0, 0.001)); f = doc.getField("date_field"); assertThat((double) f.boost(), closeTo(5.0, 0.001)); f = doc.getField("double_field"); assertThat((double) f.boost(), closeTo(6.0, 0.001)); f = doc.getField("float_field"); assertThat((double) f.boost(), closeTo(7.0, 0.001)); f = doc.getField("long_field"); assertThat((double) f.boost(), closeTo(8.0, 0.001)); f = doc.getField("short_field"); assertThat((double) f.boost(), closeTo(9.0, 0.001)); }
public void parse(ParseContext context) throws IOException { if (!enabled) { context.parser().skipChildren(); return; } XContentParser parser = context.parser(); String currentFieldName = parser.currentName(); XContentParser.Token token = parser.currentToken(); if (token == XContentParser.Token.VALUE_NULL) { // the object is null ("obj1" : null), simply bail return; } if (token.isValue() && !allowValue()) { // if we are parsing an object but it is just a value, its only allowed on root level parsers // with there // is a field name with the same name as the type throw new MapperParsingException( "object mapping for [" + name + "] tried to parse as object, but found a concrete value"); } if (nested.isNested()) { context = context.createNestedContext(fullPath); Document nestedDoc = context.doc(); Document parentDoc = nestedDoc.getParent(); // pre add the uid field if possible (id was already provided) IndexableField uidField = parentDoc.getField(UidFieldMapper.NAME); if (uidField != null) { // we don't need to add it as a full uid field in nested docs, since we don't need // versioning // we also rely on this for UidField#loadVersion // this is a deeply nested field nestedDoc.add( new Field( UidFieldMapper.NAME, uidField.stringValue(), UidFieldMapper.Defaults.NESTED_FIELD_TYPE)); } // the type of the nested doc starts with __, so we can identify that its a nested one in // filters // note, we don't prefix it with the type of the doc since it allows us to execute a nested // query // across types (for example, with similar nested objects) nestedDoc.add( new Field( TypeFieldMapper.NAME, nestedTypePathAsString, TypeFieldMapper.Defaults.FIELD_TYPE)); } ContentPath.Type origPathType = context.path().pathType(); context.path().pathType(pathType); // if we are at the end of the previous object, advance if (token == XContentParser.Token.END_OBJECT) { token = parser.nextToken(); } if (token == XContentParser.Token.START_OBJECT) { // if we are just starting an OBJECT, advance, this is the object we are parsing, we need the // name first token = parser.nextToken(); } while (token != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.START_OBJECT) { serializeObject(context, currentFieldName); } else if (token == XContentParser.Token.START_ARRAY) { serializeArray(context, currentFieldName); } else if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.VALUE_NULL) { serializeNullValue(context, currentFieldName); } else if (token == null) { throw new MapperParsingException( "object mapping for [" + name + "] tried to parse as object, but got EOF, has a concrete value been provided to it?"); } else if (token.isValue()) { serializeValue(context, currentFieldName, token); } token = parser.nextToken(); } // restore the enable path flag context.path().pathType(origPathType); if (nested.isNested()) { Document nestedDoc = context.doc(); Document parentDoc = nestedDoc.getParent(); if (nested.isIncludeInParent()) { for (IndexableField field : nestedDoc.getFields()) { if (field.name().equals(UidFieldMapper.NAME) || field.name().equals(TypeFieldMapper.NAME)) { continue; } else { parentDoc.add(field); } } } if (nested.isIncludeInRoot()) { Document rootDoc = context.rootDoc(); // don't add it twice, if its included in parent, and we are handling the master doc... if (!nested.isIncludeInParent() || parentDoc != rootDoc) { for (IndexableField field : nestedDoc.getFields()) { if (field.name().equals(UidFieldMapper.NAME) || field.name().equals(TypeFieldMapper.NAME)) { continue; } else { rootDoc.add(field); } } } } } }