@Test
public void testThatDisablingWorksWhenMerging() throws Exception {
String enabledMapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("_size")
.field("enabled", true)
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper enabledMapper = MapperTestUtils.newParser().parse(enabledMapping);
String disabledMapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("_size")
.field("enabled", false)
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper disabledMapper = MapperTestUtils.newParser().parse(disabledMapping);
enabledMapper.merge(disabledMapper, DocumentMapper.MergeFlags.mergeFlags().simulate(false));
assertThat(enabledMapper.SizeFieldMapper().enabled(), is(false));
}
@Test
public void testDynamicStrict() throws IOException {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.field("dynamic", "strict")
.startObject("properties")
.startObject("field1")
.field("type", "string")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(mapping);
try {
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder()
.startObject()
.field("field1", "value1")
.field("field2", "value2")
.bytes());
assert false;
} catch (StrictDynamicMappingException e) {
// all is well
}
}
@Test
public void testThatSerializationCombinesToOneAnalyzerFieldIfBothAreEqual() throws Exception {
String mapping =
jsonBuilder()
.startObject()
.startObject("type1")
.startObject("properties")
.startObject("completion")
.field("type", "completion")
.field("index_analyzer", "simple")
.field("search_analyzer", "simple")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(mapping);
FieldMapper fieldMapper = defaultMapper.mappers().name("completion").mapper();
assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class));
CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper;
XContentBuilder builder = jsonBuilder().startObject();
completionFieldMapper.toXContent(builder, null).endObject();
builder.close();
Map<String, Object> serializedMap =
JsonXContent.jsonXContent.createParser(builder.bytes()).mapAndClose();
Map<String, Object> configMap = (Map<String, Object>) serializedMap.get("completion");
assertThat(configMap.get("analyzer").toString(), is("simple"));
}
@Test
public void testDynamicFalse() throws IOException {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.field("dynamic", "false")
.startObject("properties")
.startObject("field1")
.field("type", "string")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(mapping);
ParsedDocument doc =
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder()
.startObject()
.field("field1", "value1")
.field("field2", "value2")
.bytes());
assertThat(doc.rootDoc().get("field1"), equalTo("value1"));
assertThat(doc.rootDoc().get("field2"), nullValue());
}
@Test
public void testPathMapping() throws IOException {
String mapping =
copyToStringFromClasspath("/org/elasticsearch/index/mapper/path/test-mapping.json");
DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
assertThat(docMapper.mappers().indexName("first1"), notNullValue());
assertThat(docMapper.mappers().indexName("name1.first1"), nullValue());
assertThat(docMapper.mappers().indexName("last1"), nullValue());
assertThat(docMapper.mappers().indexName("i_last_1"), notNullValue());
assertThat(docMapper.mappers().indexName("name1.last1"), nullValue());
assertThat(docMapper.mappers().indexName("name1.i_last_1"), nullValue());
assertThat(docMapper.mappers().indexName("first2"), nullValue());
assertThat(docMapper.mappers().indexName("name2.first2"), notNullValue());
assertThat(docMapper.mappers().indexName("last2"), nullValue());
assertThat(docMapper.mappers().indexName("i_last_2"), nullValue());
assertThat(docMapper.mappers().indexName("name2.i_last_2"), notNullValue());
assertThat(docMapper.mappers().indexName("name2.last2"), nullValue());
// test full name
assertThat(docMapper.mappers().fullName("first1"), nullValue());
assertThat(docMapper.mappers().fullName("name1.first1"), notNullValue());
assertThat(docMapper.mappers().fullName("last1"), nullValue());
assertThat(docMapper.mappers().fullName("i_last_1"), nullValue());
assertThat(docMapper.mappers().fullName("name1.last1"), notNullValue());
assertThat(docMapper.mappers().fullName("name1.i_last_1"), nullValue());
assertThat(docMapper.mappers().fullName("first2"), nullValue());
assertThat(docMapper.mappers().fullName("name2.first2"), notNullValue());
assertThat(docMapper.mappers().fullName("last2"), nullValue());
assertThat(docMapper.mappers().fullName("i_last_2"), nullValue());
assertThat(docMapper.mappers().fullName("name2.i_last_2"), nullValue());
assertThat(docMapper.mappers().fullName("name2.last2"), notNullValue());
}
@Test
public void testNumericDetectionDefault() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.endObject()
.endObject()
.string();
DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(mapping);
ParsedDocument doc =
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder()
.startObject()
.field("s_long", "100")
.field("s_double", "100.0")
.endObject()
.bytes());
FieldMapper mapper = defaultMapper.mappers().smartNameFieldMapper("s_long");
assertThat(mapper, instanceOf(StringFieldMapper.class));
mapper = defaultMapper.mappers().smartNameFieldMapper("s_double");
assertThat(mapper, instanceOf(StringFieldMapper.class));
}
@Test
public void testSimpleAllMappersWithReparse() throws Exception {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
String builtMapping = docMapper.mappingSource().string();
// reparse it
DocumentMapper builtDocMapper = MapperTestUtils.newParser().parse(builtMapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = builtDocMapper.parse(new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries =
((AllTokenStream) field.tokenStream(docMapper.mappers().indexAnalyzer())).allEntries();
assertThat(allEntries.fields().size(), equalTo(3));
assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
assertThat(field.fieldType().omitNorms(), equalTo(true));
}
@Test
public void testSizeNotSet() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.endObject()
.endObject()
.string();
DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
BytesReference source =
XContentFactory.jsonBuilder().startObject().field("field", "value").endObject().bytes();
ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1"));
assertThat(doc.rootDoc().getField("_size"), nullValue());
}
@Test
public void testSimpleAllMappersWithStore() throws Exception {
String mapping =
copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/store-mapping.json");
DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = docMapper.parse(new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries =
((AllTokenStream) field.tokenStream(docMapper.mappers().indexAnalyzer())).allEntries();
assertThat(allEntries.fields().size(), equalTo(2));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
String text = field.stringValue();
assertThat(text, equalTo(allEntries.buildText()));
assertThat(field.fieldType().omitNorms(), equalTo(false));
}
@Test
public void testSimpleAllMappers() throws Exception {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
Document doc = docMapper.parse(new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
AllEntries allEntries =
((AllTokenStream) field.tokenStream(docMapper.mappers().indexAnalyzer())).allEntries();
assertThat(allEntries.fields().size(), equalTo(3));
assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
assertThat(allEntries.fields().contains("name.last"), equalTo(true));
assertThat(allEntries.fields().contains("simple1"), equalTo(true));
FieldMapper mapper = docMapper.mappers().smartNameFieldMapper("_all");
assertThat(field.fieldType().omitNorms(), equalTo(true));
assertThat(
mapper.queryStringTermQuery(new Term("_all", "foobar")),
Matchers.instanceOf(AllTermQuery.class));
}
@Test
public void testThatSerializationIncludesAllElements() throws Exception {
String mapping =
jsonBuilder()
.startObject()
.startObject("type1")
.startObject("properties")
.startObject("completion")
.field("type", "completion")
.field("index_analyzer", "simple")
.field("search_analyzer", "standard")
.field("payloads", true)
.field("preserve_separators", false)
.field("preserve_position_increments", true)
.field("max_input_len", 14)
.endObject()
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(mapping);
FieldMapper fieldMapper = defaultMapper.mappers().name("completion").mapper();
assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class));
CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper;
XContentBuilder builder = jsonBuilder().startObject();
completionFieldMapper.toXContent(builder, null).endObject();
builder.close();
Map<String, Object> serializedMap =
JsonXContent.jsonXContent.createParser(builder.bytes()).mapAndClose();
Map<String, Object> configMap = (Map<String, Object>) serializedMap.get("completion");
assertThat(configMap.get("index_analyzer").toString(), is("simple"));
assertThat(configMap.get("search_analyzer").toString(), is("standard"));
assertThat(Boolean.valueOf(configMap.get("payloads").toString()), is(true));
assertThat(Boolean.valueOf(configMap.get("preserve_separators").toString()), is(false));
assertThat(Boolean.valueOf(configMap.get("preserve_position_increments").toString()), is(true));
assertThat(Integer.valueOf(configMap.get("max_input_len").toString()), is(14));
}
@Before
public void setup() throws Exception {
super.setUp();
// setup field mappers
strMapper =
new StringFieldMapper.Builder("str_value")
.build(new Mapper.BuilderContext(null, new ContentPath(1)));
lngMapper =
new LongFieldMapper.Builder("lng_value")
.build(new Mapper.BuilderContext(null, new ContentPath(1)));
dblMapper =
new DoubleFieldMapper.Builder("dbl_value")
.build(new Mapper.BuilderContext(null, new ContentPath(1)));
// create index and fielddata service
ifdService = new IndexFieldDataService(new Index("test"), new DummyCircuitBreakerService());
MapperService mapperService =
MapperTestUtils.newMapperService(
ifdService.index(), ImmutableSettings.Builder.EMPTY_SETTINGS);
ifdService.setIndexService(new StubIndexService(mapperService));
writer =
new IndexWriter(
new RAMDirectory(),
new IndexWriterConfig(Lucene.VERSION, new StandardAnalyzer(Lucene.VERSION)));
int numDocs = 10;
for (int i = 0; i < numDocs; i++) {
Document d = new Document();
d.add(new StringField(strMapper.names().indexName(), "str" + i, Field.Store.NO));
d.add(new LongField(lngMapper.names().indexName(), i, Field.Store.NO));
d.add(new DoubleField(dblMapper.names().indexName(), Double.valueOf(i), Field.Store.NO));
writer.addDocument(d);
}
reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(writer, true));
}
@Test
public void testDefaultConfiguration() throws IOException {
String mapping =
jsonBuilder()
.startObject()
.startObject("type1")
.startObject("properties")
.startObject("completion")
.field("type", "completion")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(mapping);
FieldMapper fieldMapper = defaultMapper.mappers().name("completion").mapper();
assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class));
CompletionFieldMapper completionFieldMapper = (CompletionFieldMapper) fieldMapper;
assertThat(completionFieldMapper.isStoringPayloads(), is(false));
}
@Test
public void testSizeEnabled() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("_size")
.field("enabled", true)
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
BytesReference source =
XContentFactory.jsonBuilder().startObject().field("field", "value").endObject().bytes();
ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1"));
assertThat(doc.rootDoc().getField("_size").fieldType().stored(), equalTo(false));
assertThat(
doc.rootDoc().getField("_size").tokenStream(docMapper.indexAnalyzer(), null),
notNullValue());
}
static SearchContext createSearchContext(String indexName, String parentType, String childType)
throws IOException {
final Index index = new Index(indexName);
final IdCache idCache = new SimpleIdCache(index, ImmutableSettings.EMPTY);
final CacheRecycler cacheRecycler = new CacheRecycler(ImmutableSettings.EMPTY);
Settings settings = ImmutableSettings.EMPTY;
MapperService mapperService = MapperTestUtils.newMapperService(index, settings);
final IndexService indexService = new SimpleIdCacheTests.StubIndexService(mapperService);
idCache.setIndexService(indexService);
// Id_cache is now registered as document type listener, so we can add mappings.
mapperService.merge(
childType,
new CompressedString(
PutMappingRequest.buildFromSimplifiedDef(childType, "_parent", "type=" + parentType)
.string()),
true);
ThreadPool threadPool = new ThreadPool();
NodeSettingsService nodeSettingsService = new NodeSettingsService(settings);
IndicesFilterCache indicesFilterCache =
new IndicesFilterCache(settings, threadPool, cacheRecycler, nodeSettingsService);
WeightedFilterCache filterCache = new WeightedFilterCache(index, settings, indicesFilterCache);
return new TestSearchContext(cacheRecycler, idCache, indexService, filterCache);
}
@Test
public void testIgnoreMalformedOption() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("field1")
.field("type", "integer")
.field("ignore_malformed", true)
.endObject()
.startObject("field2")
.field("type", "integer")
.field("ignore_malformed", false)
.endObject()
.startObject("field3")
.field("type", "integer")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(mapping);
ParsedDocument doc =
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder()
.startObject()
.field("field1", "a")
.field("field2", "1")
.endObject()
.bytes());
assertThat(doc.rootDoc().getField("field1"), nullValue());
assertThat(doc.rootDoc().getField("field2"), notNullValue());
try {
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("field2", "a").endObject().bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(NumberFormatException.class));
}
// Verify that the default is false
try {
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("field3", "a").endObject().bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(NumberFormatException.class));
}
// Unless the global ignore_malformed option is set to true
Settings indexSettings = settingsBuilder().put("index.mapping.ignore_malformed", true).build();
defaultMapper = MapperTestUtils.newParser(indexSettings).parse(mapping);
doc =
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("field3", "a").endObject().bytes());
assertThat(doc.rootDoc().getField("field3"), nullValue());
// This should still throw an exception, since field2 is specifically set to
// ignore_malformed=false
try {
defaultMapper.parse(
"type",
"1",
XContentFactory.jsonBuilder().startObject().field("field2", "a").endObject().bytes());
} catch (MapperParsingException e) {
assertThat(e.getCause(), instanceOf(NumberFormatException.class));
}
}
@Test
public void testRandom() throws Exception {
boolean omitNorms = false;
boolean stored = false;
boolean enabled = true;
boolean autoBoost = false;
boolean tv_stored = false;
boolean tv_payloads = false;
boolean tv_offsets = false;
boolean tv_positions = false;
String similarity = null;
boolean fieldData = false;
XContentBuilder mappingBuilder = jsonBuilder();
mappingBuilder.startObject().startObject("test");
List<Tuple<String, Boolean>> booleanOptionList = new ArrayList<Tuple<String, Boolean>>();
boolean allDefault = true;
if (frequently()) {
allDefault = false;
mappingBuilder.startObject("_all");
if (randomBoolean()) {
booleanOptionList.add(
new Tuple<String, Boolean>("omit_norms", omitNorms = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(new Tuple<String, Boolean>("store", stored = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(
new Tuple<String, Boolean>("store_term_vectors", tv_stored = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(new Tuple<String, Boolean>("enabled", enabled = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(
new Tuple<String, Boolean>("auto_boost", autoBoost = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(
new Tuple<String, Boolean>("store_term_vector_offsets", tv_offsets = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(
new Tuple<String, Boolean>(
"store_term_vector_positions", tv_positions = randomBoolean()));
}
if (randomBoolean()) {
booleanOptionList.add(
new Tuple<String, Boolean>(
"store_term_vector_payloads", tv_payloads = randomBoolean()));
}
Collections.shuffle(booleanOptionList, getRandom());
for (Tuple<String, Boolean> option : booleanOptionList) {
mappingBuilder.field(option.v1(), option.v2().booleanValue());
}
tv_stored |= tv_positions || tv_payloads || tv_offsets;
if (randomBoolean()) {
mappingBuilder.field("similarity", similarity = randomBoolean() ? "BM25" : "TF/IDF");
}
if (randomBoolean()) {
fieldData = true;
mappingBuilder.startObject("fielddata");
mappingBuilder.field("foo", "bar");
mappingBuilder.endObject();
}
mappingBuilder.endObject();
}
String mapping = mappingBuilder.endObject().endObject().bytes().toUtf8();
logger.info(mapping);
DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
String builtMapping = docMapper.mappingSource().string();
// reparse it
DocumentMapper builtDocMapper = MapperTestUtils.newParser().parse(builtMapping);
byte[] json =
jsonBuilder()
.startObject()
.startObject("test")
.field("foo", "bar")
.field("_id", 1)
.field("foobar", "foobar")
.endObject()
.endObject()
.bytes()
.array();
Document doc = builtDocMapper.parse(new BytesArray(json)).rootDoc();
AllField field = (AllField) doc.getField("_all");
if (enabled) {
assertThat(field.fieldType().omitNorms(), equalTo(omitNorms));
assertThat(field.fieldType().stored(), equalTo(stored));
assertThat(field.fieldType().storeTermVectorOffsets(), equalTo(tv_offsets));
assertThat(field.fieldType().storeTermVectorPayloads(), equalTo(tv_payloads));
assertThat(field.fieldType().storeTermVectorPositions(), equalTo(tv_positions));
assertThat(field.fieldType().storeTermVectors(), equalTo(tv_stored));
AllEntries allEntries =
((AllTokenStream) field.tokenStream(docMapper.mappers().indexAnalyzer())).allEntries();
assertThat(allEntries.fields().size(), equalTo(2));
assertThat(allEntries.fields().contains("foobar"), equalTo(true));
assertThat(allEntries.fields().contains("foo"), equalTo(true));
if (!stored) {
assertThat(field.stringValue(), nullValue());
}
String text = stored ? field.stringValue() : "bar foobar";
assertThat(text.trim(), equalTo(allEntries.buildText().trim()));
} else {
assertThat(field, nullValue());
}
Term term = new Term("foo", "bar");
Query query = builtDocMapper.allFieldMapper().queryStringTermQuery(term);
if (autoBoost) {
assertThat(query, equalTo((Query) new AllTermQuery(term)));
} else {
assertThat(query, equalTo((Query) new TermQuery(term)));
}
if (similarity == null || similarity.equals("TF/IDF")) {
assertThat(builtDocMapper.allFieldMapper().similarity(), nullValue());
} else {
assertThat(similarity, equalTo(builtDocMapper.allFieldMapper().similarity().name()));
}
assertThat(builtMapping.contains("fielddata"), is(fieldData));
if (allDefault) {
BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(0);
XContentBuilder b = new XContentBuilder(XContentType.JSON.xContent(), bytesStreamOutput);
XContentBuilder xContentBuilder =
builtDocMapper.allFieldMapper().toXContent(b, ToXContent.EMPTY_PARAMS);
xContentBuilder.flush();
assertThat(bytesStreamOutput.size(), equalTo(0));
}
}