// #6187: if no fields were boosted, we shouldn't use AllTokenStream
public void testNoBoost() throws Exception {
String mapping =
copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/noboost-mapping.json");
DocumentMapper docMapper =
createIndex("test").mapperService().documentMapperParser().parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
// no fields have boost, so we should not see AllTokenStream:
assertThat(
field.tokenStream(docMapper.mappers().indexAnalyzer(), null),
Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
}
public void testAllMappersNoBoost() throws Exception {
String mapping =
copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/noboost-mapping.json");
IndexService index = createIndex("test");
DocumentMapper docMapper = index.mapperService().documentMapperParser().parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries = field.getAllEntries();
assertThat(allEntries.fields().size(), equalTo(3));
assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
assertThat(field.fieldType().omitNorms(), equalTo(false));
}
// #6187: if _all doesn't index positions then we never use AllTokenStream, even if some fields
// have boost
public void testBoostWithOmitPositions() throws Exception {
String mapping =
copyToStringFromClasspath(
"/org/elasticsearch/index/mapper/all/mapping_boost_omit_positions_on_all.json");
DocumentMapper docMapper =
createIndex("test").mapperService().documentMapperParser().parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
// _all field omits positions, so we should not get AllTokenStream even though fields are
// boosted
assertThat(
field.tokenStream(docMapper.mappers().indexAnalyzer(), null),
Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
}
public void testSimpleAllMappersWithStore() throws Exception {
String mapping =
copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/store-mapping.json");
DocumentMapper docMapper =
createIndex("test").mapperService().documentMapperParser().parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries = field.getAllEntries();
assertThat(allEntries.fields().size(), equalTo(2));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
String text = field.stringValue();
assertThat(text, equalTo(allEntries.buildText()));
assertThat(field.fieldType().omitNorms(), equalTo(false));
}
public void testSimpleAllMappersWithReparse() throws Exception {
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
DocumentMapper docMapper = parser.parse(mapping);
String builtMapping = docMapper.mappingSource().string();
// reparse it
DocumentMapper builtDocMapper = parser.parse(builtMapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = builtDocMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries = field.getAllEntries();
assertThat(allEntries.fields().toString(), allEntries.fields().size(), equalTo(3));
assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
assertThat(field.fieldType().omitNorms(), equalTo(true));
}
public void testIncludeInObjectBackcompat() throws Exception {
String mapping =
jsonBuilder().startObject().startObject("type").endObject().endObject().string();
Settings settings =
Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
DocumentMapper docMapper =
createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);
ParsedDocument doc =
docMapper.parse(
"test",
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("_all", "foo").endObject().bytes());
assertNull(doc.rootDoc().get("_all"));
AllField field = (AllField) doc.rootDoc().getField("_all");
// the backcompat behavior is actually ignoring directly specifying _all
assertFalse(field.getAllEntries().fields().iterator().hasNext());
}
@Test
public void testSimpleAllMappers() throws Exception {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = docMapper.parse(new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries =
((AllTokenStream) field.tokenStream(docMapper.mappers().indexAnalyzer())).allEntries();
assertThat(allEntries.fields().size(), equalTo(3));
assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
FieldMapper mapper = docMapper.mappers().smartNameFieldMapper("_all");
assertThat(field.fieldType().omitNorms(), equalTo(true));
assertThat(
mapper.queryStringTermQuery(new Term("_all", "foobar")),
Matchers.instanceOf(AllTermQuery.class));
}
@Test
public void testSimpleAllMappersWithReparse() throws Exception {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
String builtMapping = docMapper.mappingSource().string();
// reparse it
DocumentMapper builtDocMapper = MapperTestUtils.newParser().parse(builtMapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = builtDocMapper.parse(new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries =
((AllTokenStream) field.tokenStream(docMapper.mappers().indexAnalyzer())).allEntries();
assertThat(allEntries.fields().size(), equalTo(3));
assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
assertThat(field.fieldType().omitNorms(), equalTo(true));
}
public void testAllMappersTermQuery() throws Exception {
String mapping =
copyToStringFromClasspath(
"/org/elasticsearch/index/mapper/all/mapping_omit_positions_on_all.json");
DocumentMapper docMapper =
createIndex("test").mapperService().documentMapperParser().parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries = field.getAllEntries();
assertThat(allEntries.fields().size(), equalTo(3));
assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
AllFieldMapper mapper = docMapper.allFieldMapper();
assertThat(field.fieldType().omitNorms(), equalTo(false));
assertThat(
mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
Matchers.instanceOf(AllTermQuery.class));
}
public void testMultiField_includeInAllSetToFalse() throws IOException {
String mapping =
copyToStringFromClasspath(
"/org/elasticsearch/index/mapper/all/multifield-mapping_include_in_all_set_to_false.json");
DocumentMapper docMapper =
createIndex("test").mapperService().documentMapperParser().parse(mapping);
XContentBuilder builder = XContentFactory.jsonBuilder();
builder
.startObject()
.field("foo")
.startObject()
.field("bar", "Elasticsearch rules!")
.endObject()
.endObject();
Document doc = docMapper.parse("test", "test", "1", builder.bytes()).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries = field.getAllEntries();
assertThat(allEntries.fields(), empty());
}
public void testRandom() throws Exception {
boolean omitNorms = false;
boolean stored = false;
boolean enabled = true;
boolean tv_stored = false;
boolean tv_payloads = false;
boolean tv_offsets = false;
boolean tv_positions = false;
String similarity = null;
XContentBuilder mappingBuilder = jsonBuilder();
mappingBuilder.startObject().startObject("test");
List<Tuple<String, Boolean>> booleanOptionList = new ArrayList<>();
boolean allDefault = true;
if (frequently()) {
allDefault = false;
mappingBuilder.startObject("_all");
if (randomBoolean()) {
booleanOptionList.add(new Tuple<>("omit_norms", omitNorms = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(new Tuple<>("store", stored = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(new Tuple<>("store_term_vectors", tv_stored = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(new Tuple<>("enabled", enabled = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(
new Tuple<>("store_term_vector_offsets", tv_offsets = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(
new Tuple<>("store_term_vector_positions", tv_positions = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(
new Tuple<>("store_term_vector_payloads", tv_payloads = randomBoolean()));
}
Collections.shuffle(booleanOptionList, getRandom());
for (Tuple<String, Boolean> option : booleanOptionList) {
mappingBuilder.field(option.v1(), option.v2().booleanValue());
}
tv_stored |= tv_positions || tv_payloads || tv_offsets;
if (randomBoolean()) {
mappingBuilder.field("similarity", similarity = randomBoolean() ? "BM25" : "TF/IDF");
}
mappingBuilder.endObject();
}
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
String mapping = mappingBuilder.endObject().endObject().bytes().toUtf8();
logger.info(mapping);
DocumentMapper docMapper = parser.parse(mapping);
String builtMapping = docMapper.mappingSource().string();
// reparse it
DocumentMapper builtDocMapper = parser.parse(builtMapping);
byte[] json =
jsonBuilder()
.startObject()
.field("foo", "bar")
.field("foobar", "foobar")
.endObject()
.bytes()
.toBytes();
Document doc = builtDocMapper.parse("test", "test", "1", new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
if (enabled) {
assertThat(field.fieldType().omitNorms(), equalTo(omitNorms));
assertThat(field.fieldType().stored(), equalTo(stored));
assertThat(field.fieldType().storeTermVectorOffsets(), equalTo(tv_offsets));
assertThat(field.fieldType().storeTermVectorPayloads(), equalTo(tv_payloads));
assertThat(field.fieldType().storeTermVectorPositions(), equalTo(tv_positions));
assertThat(field.fieldType().storeTermVectors(), equalTo(tv_stored));
AllEntries allEntries = field.getAllEntries();
assertThat(allEntries.fields().size(), equalTo(2));
assertThat(allEntries.fields().contains("foobar"), equalTo(true));
assertThat(allEntries.fields().contains("foo"), equalTo(true));
if (!stored) {
assertThat(field.stringValue(), nullValue());
}
String text = stored ? field.stringValue() : "bar foobar";
assertThat(text.trim(), equalTo(allEntries.buildText().trim()));
} else {
assertThat(field, nullValue());
}
if (similarity == null || similarity.equals("TF/IDF")) {
assertThat(builtDocMapper.allFieldMapper().fieldType().similarity(), nullValue());
} else {
assertThat(
similarity, equalTo(builtDocMapper.allFieldMapper().fieldType().similarity().name()));
}
if (allDefault) {
BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(0);
XContentBuilder b = new XContentBuilder(XContentType.JSON.xContent(), bytesStreamOutput);
XContentBuilder xContentBuilder =
builtDocMapper.allFieldMapper().toXContent(b, ToXContent.EMPTY_PARAMS);
xContentBuilder.flush();
assertThat(bytesStreamOutput.size(), equalTo(0));
}
}