// #6187: if no fields were boosted, we shouldn't use AllTokenStream
public void testNoBoost() throws Exception {
String mapping =
copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/noboost-mapping.json");
DocumentMapper docMapper =
createIndex("test").mapperService().documentMapperParser().parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
// no fields have boost, so we should not see AllTokenStream:
assertThat(
field.tokenStream(docMapper.mappers().indexAnalyzer(), null),
Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
}
public void testAllMappersNoBoost() throws Exception {
String mapping =
copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/noboost-mapping.json");
IndexService index = createIndex("test");
DocumentMapper docMapper = index.mapperService().documentMapperParser().parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries = field.getAllEntries();
assertThat(allEntries.fields().size(), equalTo(3));
assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
assertThat(field.fieldType().omitNorms(), equalTo(false));
}
// #6187: if _all doesn't index positions then we never use AllTokenStream, even if some fields
// have boost
public void testBoostWithOmitPositions() throws Exception {
String mapping =
copyToStringFromClasspath(
"/org/elasticsearch/index/mapper/all/mapping_boost_omit_positions_on_all.json");
DocumentMapper docMapper =
createIndex("test").mapperService().documentMapperParser().parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
// _all field omits positions, so we should not get AllTokenStream even though fields are
// boosted
assertThat(
field.tokenStream(docMapper.mappers().indexAnalyzer(), null),
Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
}
public void testSimpleAllMappersWithStore() throws Exception {
String mapping =
copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/store-mapping.json");
DocumentMapper docMapper =
createIndex("test").mapperService().documentMapperParser().parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries = field.getAllEntries();
assertThat(allEntries.fields().size(), equalTo(2));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
String text = field.stringValue();
assertThat(text, equalTo(allEntries.buildText()));
assertThat(field.fieldType().omitNorms(), equalTo(false));
}
public void testSimpleParserNoTypeNoId() throws Exception {
String mapping =
copyToStringFromClasspath("/org/elasticsearch/index/mapper/simple/test-mapping.json");
DocumentMapper docMapper =
createIndex("test").mapperService().documentMapperParser().parse(mapping);
BytesReference json =
new BytesArray(
copyToBytesFromClasspath(
"/org/elasticsearch/index/mapper/simple/test1-notype-noid.json"));
Document doc = docMapper.parse("test", "person", "1", json).rootDoc();
assertThat(
doc.get(docMapper.uidMapper().fieldType().names().indexName()),
equalTo(Uid.createUid("person", "1")));
assertThat(
doc.get(docMapper.mappers().getMapper("name.first").fieldType().names().indexName()),
equalTo("shay"));
}
public void testSimpleAllMappersWithReparse() throws Exception {
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
DocumentMapper docMapper = parser.parse(mapping);
String builtMapping = docMapper.mappingSource().string();
// reparse it
DocumentMapper builtDocMapper = parser.parse(builtMapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = builtDocMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries = field.getAllEntries();
assertThat(allEntries.fields().toString(), allEntries.fields().size(), equalTo(3));
assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
assertThat(field.fieldType().omitNorms(), equalTo(true));
}
@Test
public void testSimpleAllMappers() throws Exception {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = docMapper.parse(new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries =
((AllTokenStream) field.tokenStream(docMapper.mappers().indexAnalyzer())).allEntries();
assertThat(allEntries.fields().size(), equalTo(3));
assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
FieldMapper mapper = docMapper.mappers().smartNameFieldMapper("_all");
assertThat(field.fieldType().omitNorms(), equalTo(true));
assertThat(
mapper.queryStringTermQuery(new Term("_all", "foobar")),
Matchers.instanceOf(AllTermQuery.class));
}
@Test
public void testSimpleAllMappersWithReparse() throws Exception {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
String builtMapping = docMapper.mappingSource().string();
// reparse it
DocumentMapper builtDocMapper = MapperTestUtils.newParser().parse(builtMapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = builtDocMapper.parse(new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries =
((AllTokenStream) field.tokenStream(docMapper.mappers().indexAnalyzer())).allEntries();
assertThat(allEntries.fields().size(), equalTo(3));
assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
assertThat(field.fieldType().omitNorms(), equalTo(true));
}
public void testAllMappersTermQuery() throws Exception {
String mapping =
copyToStringFromClasspath(
"/org/elasticsearch/index/mapper/all/mapping_omit_positions_on_all.json");
DocumentMapper docMapper =
createIndex("test").mapperService().documentMapperParser().parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries = field.getAllEntries();
assertThat(allEntries.fields().size(), equalTo(3));
assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
AllFieldMapper mapper = docMapper.allFieldMapper();
assertThat(field.fieldType().omitNorms(), equalTo(false));
assertThat(
mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
Matchers.instanceOf(AllTermQuery.class));
}
public void testMultiField_includeInAllSetToFalse() throws IOException {
String mapping =
copyToStringFromClasspath(
"/org/elasticsearch/index/mapper/all/multifield-mapping_include_in_all_set_to_false.json");
DocumentMapper docMapper =
createIndex("test").mapperService().documentMapperParser().parse(mapping);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder
.startObject()
.field("foo")
.startObject()
.field("bar", "Elasticsearch rules!")
.endObject()
.endObject();
Document doc = docMapper.parse("test", "test", "1", builder.bytes()).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries = field.getAllEntries();
assertThat(allEntries.fields(), empty());
}
public void testSimpleMapper() throws Exception {
IndexService indexService = createIndex("test");
Settings settings = indexService.getIndexSettings().getSettings();
DocumentMapperParser mapperParser = indexService.mapperService().documentMapperParser();
DocumentMapper docMapper =
doc(
settings,
rootObject("person")
.add(object("name").add(stringField("first").store(true).index(false))),
indexService.mapperService())
.build(indexService.mapperService(), mapperParser);
BytesReference json =
new BytesArray(
copyToBytesFromClasspath("/org/elasticsearch/index/mapper/simple/test1.json"));
Document doc = docMapper.parse("test", "person", "1", json).rootDoc();
assertThat(
doc.get(docMapper.mappers().getMapper("name.first").fieldType().names().indexName()),
equalTo("shay"));
doc = docMapper.parse("test", "person", "1", json).rootDoc();
}
@Override
public void postParse(ParseContext context) throws IOException {
if (context.id() == null && !context.sourceToParse().flyweight()) {
throw new MapperParsingException("No id found while parsing the content source");
}
// if we did not have the id as part of the sourceToParse, then we need to parse it here
// it would have been filled in the _id parse phase
if (context.sourceToParse().id() == null) {
super.parse(context);
// since we did not have the uid in the pre phase, we did not add it automatically to the
// nested docs
// as they were created we need to make sure we add it to all the nested docs...
if (context.docs().size() > 1) {
final IndexableField uidField = context.rootDoc().getField(UidFieldMapper.NAME);
assert uidField != null;
// we need to go over the docs and add it...
for (int i = 1; i < context.docs().size(); i++) {
final Document doc = context.docs().get(i);
doc.add(
new Field(UidFieldMapper.NAME, uidField.stringValue(), Defaults.NESTED_FIELD_TYPE));
}
}
}
}
public void testSimple() throws Exception {
String mapping =
copyToStringFromClasspath(
"/org/elasticsearch/index/mapper/dynamictemplate/genericstore/test-mapping.json");
IndexService index = createIndex("test");
client().admin().indices().preparePutMapping("test").setType("person").setSource(mapping).get();
DocumentMapper docMapper = index.mapperService().documentMapper("person");
byte[] json =
copyToBytesFromClasspath(
"/org/elasticsearch/index/mapper/dynamictemplate/genericstore/test-data.json");
ParsedDocument parsedDoc = docMapper.parse("test", "person", "1", new BytesArray(json));
client()
.admin()
.indices()
.preparePutMapping("test")
.setType("person")
.setSource(parsedDoc.dynamicMappingsUpdate().toString())
.get();
docMapper = index.mapperService().documentMapper("person");
Document doc = parsedDoc.rootDoc();
IndexableField f = doc.getField("name");
assertThat(f.name(), equalTo("name"));
assertThat(f.stringValue(), equalTo("some name"));
assertThat(f.fieldType().stored(), equalTo(true));
FieldMapper fieldMapper = docMapper.mappers().getMapper("name");
assertThat(fieldMapper.fieldType().stored(), equalTo(true));
f = doc.getField("age");
assertThat(f.name(), equalTo("age"));
assertThat(f.fieldType().stored(), equalTo(true));
fieldMapper = docMapper.mappers().getMapper("age");
assertThat(fieldMapper.fieldType().stored(), equalTo(true));
}
public void testRandom() throws Exception {
boolean omitNorms = false;
boolean stored = false;
boolean enabled = true;
boolean tv_stored = false;
boolean tv_payloads = false;
boolean tv_offsets = false;
boolean tv_positions = false;
String similarity = null;
XContentBuilder mappingBuilder = jsonBuilder();
mappingBuilder.startObject().startObject("test");
List<Tuple<String, Boolean>> booleanOptionList = new ArrayList<>();
boolean allDefault = true;
if (frequently()) {
allDefault = false;
mappingBuilder.startObject("_all");
if (randomBoolean()) {
booleanOptionList.add(new Tuple<>("omit_norms", omitNorms = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(new Tuple<>("store", stored = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(new Tuple<>("store_term_vectors", tv_stored = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(new Tuple<>("enabled", enabled = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(
new Tuple<>("store_term_vector_offsets", tv_offsets = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(
new Tuple<>("store_term_vector_positions", tv_positions = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(
new Tuple<>("store_term_vector_payloads", tv_payloads = randomBoolean()));
}
Collections.shuffle(booleanOptionList, getRandom());
for (Tuple<String, Boolean> option : booleanOptionList) {
mappingBuilder.field(option.v1(), option.v2().booleanValue());
}
tv_stored |= tv_positions || tv_payloads || tv_offsets;
if (randomBoolean()) {
mappingBuilder.field("similarity", similarity = randomBoolean() ? "BM25" : "TF/IDF");
}
mappingBuilder.endObject();
}
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
String mapping = mappingBuilder.endObject().endObject().bytes().toUtf8();
logger.info(mapping);
DocumentMapper docMapper = parser.parse(mapping);
String builtMapping = docMapper.mappingSource().string();
// reparse it
DocumentMapper builtDocMapper = parser.parse(builtMapping);
byte[] json =
jsonBuilder()
.startObject()
.field("foo", "bar")
.field("foobar", "foobar")
.endObject()
.bytes()
.toBytes();
Document doc = builtDocMapper.parse("test", "test", "1", new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
if (enabled) {
assertThat(field.fieldType().omitNorms(), equalTo(omitNorms));
assertThat(field.fieldType().stored(), equalTo(stored));
assertThat(field.fieldType().storeTermVectorOffsets(), equalTo(tv_offsets));
assertThat(field.fieldType().storeTermVectorPayloads(), equalTo(tv_payloads));
assertThat(field.fieldType().storeTermVectorPositions(), equalTo(tv_positions));
assertThat(field.fieldType().storeTermVectors(), equalTo(tv_stored));
AllEntries allEntries = field.getAllEntries();
assertThat(allEntries.fields().size(), equalTo(2));
assertThat(allEntries.fields().contains("foobar"), equalTo(true));
assertThat(allEntries.fields().contains("foo"), equalTo(true));
if (!stored) {
assertThat(field.stringValue(), nullValue());
}
String text = stored ? field.stringValue() : "bar foobar";
assertThat(text.trim(), equalTo(allEntries.buildText().trim()));
} else {
assertThat(field, nullValue());
}
if (similarity == null || similarity.equals("TF/IDF")) {
assertThat(builtDocMapper.allFieldMapper().fieldType().similarity(), nullValue());
} else {
assertThat(
similarity, equalTo(builtDocMapper.allFieldMapper().fieldType().similarity().name()));
}
if (allDefault) {
BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(0);
XContentBuilder b = new XContentBuilder(XContentType.JSON.xContent(), bytesStreamOutput);
XContentBuilder xContentBuilder =
builtDocMapper.allFieldMapper().toXContent(b, ToXContent.EMPTY_PARAMS);
xContentBuilder.flush();
assertThat(bytesStreamOutput.size(), equalTo(0));
}
}
public void testFieldLevelBoost() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("person")
.startObject("properties")
.startObject("str_field")
.field("type", "string")
.endObject()
.startObject("int_field")
.field("type", "integer")
.startObject("norms")
.field("enabled", true)
.endObject()
.endObject()
.startObject("byte_field")
.field("type", "byte")
.startObject("norms")
.field("enabled", true)
.endObject()
.endObject()
.startObject("date_field")
.field("type", "date")
.startObject("norms")
.field("enabled", true)
.endObject()
.endObject()
.startObject("double_field")
.field("type", "double")
.startObject("norms")
.field("enabled", true)
.endObject()
.endObject()
.startObject("float_field")
.field("type", "float")
.startObject("norms")
.field("enabled", true)
.endObject()
.endObject()
.startObject("long_field")
.field("type", "long")
.startObject("norms")
.field("enabled", true)
.endObject()
.endObject()
.startObject("short_field")
.field("type", "short")
.startObject("norms")
.field("enabled", true)
.endObject()
.endObject()
.string();
DocumentMapper docMapper =
createIndex("test")
.mapperService()
.documentMapperParser()
.parse("person", new CompressedXContent(mapping));
BytesReference json =
XContentFactory.jsonBuilder()
.startObject()
.startObject("str_field")
.field("boost", 2.0)
.field("value", "some name")
.endObject()
.startObject("int_field")
.field("boost", 3.0)
.field("value", 10)
.endObject()
.startObject("byte_field")
.field("boost", 4.0)
.field("value", 20)
.endObject()
.startObject("date_field")
.field("boost", 5.0)
.field("value", "2012-01-10")
.endObject()
.startObject("double_field")
.field("boost", 6.0)
.field("value", 30.0)
.endObject()
.startObject("float_field")
.field("boost", 7.0)
.field("value", 40.0)
.endObject()
.startObject("long_field")
.field("boost", 8.0)
.field("value", 50)
.endObject()
.startObject("short_field")
.field("boost", 9.0)
.field("value", 60)
.endObject()
.bytes();
Document doc = docMapper.parse("test", "person", "1", json).rootDoc();
IndexableField f = doc.getField("str_field");
assertThat((double) f.boost(), closeTo(2.0, 0.001));
f = doc.getField("int_field");
assertThat((double) f.boost(), closeTo(3.0, 0.001));
f = doc.getField("byte_field");
assertThat((double) f.boost(), closeTo(4.0, 0.001));
f = doc.getField("date_field");
assertThat((double) f.boost(), closeTo(5.0, 0.001));
f = doc.getField("double_field");
assertThat((double) f.boost(), closeTo(6.0, 0.001));
f = doc.getField("float_field");
assertThat((double) f.boost(), closeTo(7.0, 0.001));
f = doc.getField("long_field");
assertThat((double) f.boost(), closeTo(8.0, 0.001));
f = doc.getField("short_field");
assertThat((double) f.boost(), closeTo(9.0, 0.001));
}
public void parse(ParseContext context) throws IOException {
if (!enabled) {
context.parser().skipChildren();
return;
}
XContentParser parser = context.parser();
String currentFieldName = parser.currentName();
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.VALUE_NULL) {
// the object is null ("obj1" : null), simply bail
return;
}
if (token.isValue() && !allowValue()) {
// if we are parsing an object but it is just a value, its only allowed on root level parsers
// with there
// is a field name with the same name as the type
throw new MapperParsingException(
"object mapping for [" + name + "] tried to parse as object, but found a concrete value");
}
if (nested.isNested()) {
context = context.createNestedContext(fullPath);
Document nestedDoc = context.doc();
Document parentDoc = nestedDoc.getParent();
// pre add the uid field if possible (id was already provided)
IndexableField uidField = parentDoc.getField(UidFieldMapper.NAME);
if (uidField != null) {
// we don't need to add it as a full uid field in nested docs, since we don't need
// versioning
// we also rely on this for UidField#loadVersion
// this is a deeply nested field
nestedDoc.add(
new Field(
UidFieldMapper.NAME,
uidField.stringValue(),
UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
}
// the type of the nested doc starts with __, so we can identify that its a nested one in
// filters
// note, we don't prefix it with the type of the doc since it allows us to execute a nested
// query
// across types (for example, with similar nested objects)
nestedDoc.add(
new Field(
TypeFieldMapper.NAME, nestedTypePathAsString, TypeFieldMapper.Defaults.FIELD_TYPE));
}
ContentPath.Type origPathType = context.path().pathType();
context.path().pathType(pathType);
// if we are at the end of the previous object, advance
if (token == XContentParser.Token.END_OBJECT) {
token = parser.nextToken();
}
if (token == XContentParser.Token.START_OBJECT) {
// if we are just starting an OBJECT, advance, this is the object we are parsing, we need the
// name first
token = parser.nextToken();
}
while (token != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.START_OBJECT) {
serializeObject(context, currentFieldName);
} else if (token == XContentParser.Token.START_ARRAY) {
serializeArray(context, currentFieldName);
} else if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.VALUE_NULL) {
serializeNullValue(context, currentFieldName);
} else if (token == null) {
throw new MapperParsingException(
"object mapping for ["
+ name
+ "] tried to parse as object, but got EOF, has a concrete value been provided to it?");
} else if (token.isValue()) {
serializeValue(context, currentFieldName, token);
}
token = parser.nextToken();
}
// restore the enable path flag
context.path().pathType(origPathType);
if (nested.isNested()) {
Document nestedDoc = context.doc();
Document parentDoc = nestedDoc.getParent();
if (nested.isIncludeInParent()) {
for (IndexableField field : nestedDoc.getFields()) {
if (field.name().equals(UidFieldMapper.NAME)
|| field.name().equals(TypeFieldMapper.NAME)) {
continue;
} else {
parentDoc.add(field);
}
}
}
if (nested.isIncludeInRoot()) {
Document rootDoc = context.rootDoc();
// don't add it twice, if its included in parent, and we are handling the master doc...
if (!nested.isIncludeInParent() || parentDoc != rootDoc) {
for (IndexableField field : nestedDoc.getFields()) {
if (field.name().equals(UidFieldMapper.NAME)
|| field.name().equals(TypeFieldMapper.NAME)) {
continue;
} else {
rootDoc.add(field);
}
}
}
}
}
}