@Test
public void testIpMultiField() throws Exception {
assertAcked(
client()
.admin()
.indices()
.prepareCreate("my-index")
.addMapping("my-type", createMappingSource("ip")));
GetMappingsResponse getMappingsResponse =
client().admin().indices().prepareGetMappings("my-index").get();
MappingMetaData mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type");
assertThat(mappingMetaData, not(nullValue()));
Map<String, Object> mappingSource = mappingMetaData.sourceAsMap();
Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource));
assertThat(aField.size(), equalTo(2));
assertThat(aField.get("type").toString(), equalTo("ip"));
assertThat(aField.get("fields"), notNullValue());
Map bField = ((Map) XContentMapValues.extractValue("properties.a.fields.b", mappingSource));
assertThat(bField.size(), equalTo(2));
assertThat(bField.get("type").toString(), equalTo("string"));
assertThat(bField.get("index").toString(), equalTo("not_analyzed"));
client()
.prepareIndex("my-index", "my-type", "1")
.setSource("a", "127.0.0.1")
.setRefresh(true)
.get();
CountResponse countResponse =
client().prepareCount("my-index").setQuery(matchQuery("a.b", "127.0.0.1")).get();
assertThat(countResponse.getCount(), equalTo(1l));
}
public void testTypeNotCreatedOnIndexFailure() throws IOException, InterruptedException {
XContentBuilder mapping =
jsonBuilder()
.startObject()
.startObject("_default_")
.field("dynamic", "strict")
.endObject()
.endObject();
IndexService indexService = createIndex("test", Settings.EMPTY, "_default_", mapping);
try {
client()
.prepareIndex()
.setIndex("test")
.setType("type")
.setSource(jsonBuilder().startObject().field("test", "test").endObject())
.get();
fail();
} catch (StrictDynamicMappingException e) {
}
GetMappingsResponse getMappingsResponse =
client().admin().indices().prepareGetMappings("test").get();
assertNull(getMappingsResponse.getMappings().get("test").get("type"));
}
@Test
public void testDefaultApplied() throws IOException {
createIndex("test1", ImmutableSettings.settingsBuilder().build());
createIndex("test2", ImmutableSettings.settingsBuilder().build());
XContentBuilder defaultMapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject(MapperService.DEFAULT_MAPPING)
.startObject("_size")
.field("enabled", true)
.endObject()
.endObject()
.endObject();
client()
.admin()
.indices()
.preparePutMapping()
.setType(MapperService.DEFAULT_MAPPING)
.setSource(defaultMapping)
.get();
XContentBuilder typeMapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("_all")
.field("enabled", false)
.endObject()
.endObject()
.endObject();
client()
.admin()
.indices()
.preparePutMapping("test1")
.setType("type")
.setSource(typeMapping)
.get();
client()
.admin()
.indices()
.preparePutMapping("test1", "test2")
.setType("type")
.setSource(typeMapping)
.get();
GetMappingsResponse response = client().admin().indices().prepareGetMappings("test2").get();
assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_all"));
assertFalse(
(Boolean)
((LinkedHashMap)
response.getMappings().get("test2").get("type").getSourceAsMap().get("_all"))
.get("enabled"));
assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_size"));
assertTrue(
(Boolean)
((LinkedHashMap)
response.getMappings().get("test2").get("type").getSourceAsMap().get("_size"))
.get("enabled"));
}
private void assertTTLMappingEnabled(String index, String type) throws IOException {
String errMsg =
String.format(
Locale.ROOT, "Expected ttl field mapping to be enabled for %s/%s", index, type);
GetMappingsResponse getMappingsResponse =
client().admin().indices().prepareGetMappings(index).addTypes(type).get();
Map<String, Object> mappingSource =
getMappingsResponse.getMappings().get(index).get(type).getSourceAsMap();
assertThat(errMsg, mappingSource, hasKey("_ttl"));
String ttlAsString = mappingSource.get("_ttl").toString();
assertThat(ttlAsString, is(notNullValue()));
assertThat(errMsg, ttlAsString, is("{enabled=true}"));
}
@Test
public void testMappingNodeClient() throws Exception {
final NodeClient es =
new NodeClient().flushInterval(TimeValue.timeValueSeconds(5)).newClient(client("1"));
es.addMapping("test", "{\"test\":{\"properties\":{\"location\":{\"type\":\"geo_point\"}}}}");
es.newIndex("test");
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test");
GetMappingsResponse getMappingsResponse =
es.client().admin().indices().getMappings(getMappingsRequest).actionGet();
logger.info("mappings={}", getMappingsResponse.getMappings());
es.shutdown();
if (es.hasThrowable()) {
logger.error("error", es.getThrowable());
}
assertFalse(es.hasThrowable());
}
@Test
public void updateMappingWithoutTypeMultiObjects() throws Exception {
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(
ImmutableSettings.settingsBuilder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0))
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
PutMappingResponse putMappingResponse =
client()
.admin()
.indices()
.preparePutMapping("test")
.setType("doc")
.setSource(
"{\"_source\":{\"enabled\":false},\"properties\":{\"date\":{\"type\":\"integer\"}}}")
.execute()
.actionGet();
assertThat(putMappingResponse.isAcknowledged(), equalTo(true));
GetMappingsResponse getMappingsResponse =
client().admin().indices().prepareGetMappings("test").execute().actionGet();
assertThat(
getMappingsResponse.mappings().get("test").get("doc").source().toString(),
equalTo(
"{\"doc\":{\"_source\":{\"enabled\":false},\"properties\":{\"date\":{\"type\":\"integer\"}}}}"));
}
public void testMappingsPropagatedToMasterNodeImmediately() throws IOException {
createIndex("index");
// works when the type has been dynamically created
client().prepareIndex("index", "type", "1").setSource("foo", 3).get();
GetMappingsResponse mappings =
client().admin().indices().prepareGetMappings("index").setTypes("type").get();
assertMappingsHaveField(mappings, "index", "type", "foo");
// works if the type already existed
client().prepareIndex("index", "type", "1").setSource("bar", "baz").get();
mappings = client().admin().indices().prepareGetMappings("index").setTypes("type").get();
assertMappingsHaveField(mappings, "index", "type", "bar");
// works if we indexed an empty document
client().prepareIndex("index", "type2", "1").setSource().get();
mappings = client().admin().indices().prepareGetMappings("index").setTypes("type2").get();
assertTrue(
mappings.getMappings().get("index").toString(),
mappings.getMappings().get("index").containsKey("type2"));
}
private static void assertMappingsHaveField(
GetMappingsResponse mappings, String index, String type, String field) throws IOException {
ImmutableOpenMap<String, MappingMetaData> indexMappings = mappings.getMappings().get("index");
assertNotNull(indexMappings);
MappingMetaData typeMappings = indexMappings.get(type);
assertNotNull(typeMappings);
Map<String, Object> typeMappingsMap = typeMappings.getSourceAsMap();
Map<String, Object> properties = (Map<String, Object>) typeMappingsMap.get("properties");
assertTrue(
"Could not find [" + field + "] in " + typeMappingsMap.toString(),
properties.containsKey(field));
}
/**
* Until we can verify that all instances have moved over to our new mapping, we need to handcheck
* all index mappings to make sure they have doc.associations.assoc_index as a doc value
*
* @param communityIdStrs
* @return
*/
@SuppressWarnings("unchecked")
private static boolean validateAssociationMapping(String[] communityIdStrs) {
// get all index mappings associated with these commids
String[] mappings = new String[communityIdStrs.length];
StringBuilder sb = new StringBuilder(", ");
for (int i = 0; i < communityIdStrs.length; i++) {
String s = communityIdStrs[i];
mappings[i] = "doc_" + s + "*";
sb.append("doc_").append(s).append("*, ");
}
ElasticSearchManager esm = ElasticSearchManager.getIndex(sb.substring(2, sb.length()));
GetMappingsResponse response =
esm.getRawClient().admin().indices().prepareGetMappings(mappings).get();
for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> mapping :
response.getMappings()) {
ImmutableOpenMap<String, MappingMetaData> mappingVal = mapping.value;
MappingMetaData mapping_meta = mappingVal.get("document_index");
try {
Map<String, Object> map = mapping_meta.getSourceAsMap();
Map<String, Object> props = (Map<String, Object>) map.get("properties");
Map<String, Object> assocs = (Map<String, Object>) props.get(DocumentPojo.associations_);
Map<String, Object> assocs_props = (Map<String, Object>) assocs.get("properties");
Map<String, Object> assoc_index =
(Map<String, Object>) assocs_props.get(AssociationPojo.assoc_index_);
if (!assoc_index.containsKey("doc_values") || !((Boolean) assoc_index.get("doc_values"))) {
// doc values doesn't exist in mapping or was false
return false;
}
} catch (Exception ex) {
// failed somehow
return false;
}
}
// if we fell through, all the checked indexes had the doc_value field set
return true;
}
@Test
public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception {
String masterNode = startMasterNode();
String redNodeDataPath = createTempDir().toString();
String redNode = startDataNode("red", redNodeDataPath);
// create red_index on red_node and same for red
client()
.admin()
.cluster()
.health(clusterHealthRequest().waitForYellowStatus().waitForNodes("2"))
.get();
assertAcked(
prepareCreate("red_index")
.setSettings(
Settings.builder()
.put("index.number_of_replicas", 0)
.put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red")));
index(
"red_index",
"doc",
"1",
jsonBuilder().startObject().field("text", "some text").endObject());
logger.info("--> wait for green red_index");
ensureGreen();
logger.info("--> wait for meta state written for red_index");
assertIndexInMetaState(redNode, "red_index");
assertIndexInMetaState(masterNode, "red_index");
waitForConcreteMappingsOnAll("red_index", "doc", "text");
logger.info("--> close red_index");
client().admin().indices().prepareClose("red_index").get();
// close the index
ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
assertThat(
clusterStateResponse.getState().getMetaData().index("red_index").getState().name(),
equalTo(IndexMetaData.State.CLOSE.name()));
logger.info("--> restart red node");
stopNode(redNode);
redNode = startDataNode("red", redNodeDataPath);
client()
.admin()
.indices()
.preparePutMapping("red_index")
.setType("doc")
.setSource(
jsonBuilder()
.startObject()
.startObject("properties")
.startObject("integer_field")
.field("type", "integer")
.endObject()
.endObject()
.endObject())
.get();
GetMappingsResponse getMappingsResponse =
client().admin().indices().prepareGetMappings("red_index").addTypes("doc").get();
assertNotNull(
((LinkedHashMap)
(getMappingsResponse
.getMappings()
.get("red_index")
.get("doc")
.getSourceAsMap()
.get("properties")))
.get("integer_field"));
// restart master with empty data folder and maybe red node
((InternalTestCluster) cluster()).stopCurrentMasterNode();
masterNode = startMasterNode();
ensureGreen("red_index");
assertIndexInMetaState(redNode, "red_index");
assertIndexInMetaState(masterNode, "red_index");
clusterStateResponse = client().admin().cluster().prepareState().get();
assertThat(
clusterStateResponse.getState().getMetaData().index("red_index").getState().name(),
equalTo(IndexMetaData.State.CLOSE.name()));
getMappingsResponse =
client().admin().indices().prepareGetMappings("red_index").addTypes("doc").get();
assertNotNull(
((LinkedHashMap)
(getMappingsResponse
.getMappings()
.get("red_index")
.get("doc")
.getSourceAsMap()
.get("properties")))
.get("integer_field"));
}
@SuppressWarnings("unchecked")
@Test
public void updateDefaultMappingSettings() throws Exception {
logger.info("Creating index with _default_ mappings");
client()
.admin()
.indices()
.prepareCreate("test")
.addMapping(
MapperService.DEFAULT_MAPPING,
JsonXContent.contentBuilder()
.startObject()
.startObject(MapperService.DEFAULT_MAPPING)
.field("date_detection", false)
.endObject()
.endObject())
.get();
GetMappingsResponse getResponse =
client()
.admin()
.indices()
.prepareGetMappings("test")
.addTypes(MapperService.DEFAULT_MAPPING)
.get();
Map<String, Object> defaultMapping =
getResponse.getMappings().get("test").get(MapperService.DEFAULT_MAPPING).sourceAsMap();
assertThat(defaultMapping, hasKey("date_detection"));
logger.info("Emptying _default_ mappings");
// now remove it
PutMappingResponse putResponse =
client()
.admin()
.indices()
.preparePutMapping("test")
.setType(MapperService.DEFAULT_MAPPING)
.setSource(
JsonXContent.contentBuilder()
.startObject()
.startObject(MapperService.DEFAULT_MAPPING)
.endObject()
.endObject())
.get();
assertThat(putResponse.isAcknowledged(), equalTo(true));
logger.info("Done Emptying _default_ mappings");
getResponse =
client()
.admin()
.indices()
.prepareGetMappings("test")
.addTypes(MapperService.DEFAULT_MAPPING)
.get();
defaultMapping =
getResponse.getMappings().get("test").get(MapperService.DEFAULT_MAPPING).sourceAsMap();
assertThat(defaultMapping, not(hasKey("date_detection")));
// now test you can change stuff that are normally unchangeable
logger.info("Creating _default_ mappings with an analyzed field");
putResponse =
client()
.admin()
.indices()
.preparePutMapping("test")
.setType(MapperService.DEFAULT_MAPPING)
.setSource(
JsonXContent.contentBuilder()
.startObject()
.startObject(MapperService.DEFAULT_MAPPING)
.startObject("properties")
.startObject("f")
.field("type", "string")
.field("index", "analyzed")
.endObject()
.endObject()
.endObject()
.endObject())
.get();
assertThat(putResponse.isAcknowledged(), equalTo(true));
logger.info("Changing _default_ mappings field from analyzed to non-analyzed");
putResponse =
client()
.admin()
.indices()
.preparePutMapping("test")
.setType(MapperService.DEFAULT_MAPPING)
.setSource(
JsonXContent.contentBuilder()
.startObject()
.startObject(MapperService.DEFAULT_MAPPING)
.startObject("properties")
.startObject("f")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
.endObject()
.endObject()
.endObject())
.get();
assertThat(putResponse.isAcknowledged(), equalTo(true));
logger.info("Done changing _default_ mappings field from analyzed to non-analyzed");
getResponse =
client()
.admin()
.indices()
.prepareGetMappings("test")
.addTypes(MapperService.DEFAULT_MAPPING)
.get();
defaultMapping =
getResponse.getMappings().get("test").get(MapperService.DEFAULT_MAPPING).sourceAsMap();
Map<String, Object> fieldSettings =
(Map<String, Object>) ((Map) defaultMapping.get("properties")).get("f");
assertThat(fieldSettings, hasEntry("index", (Object) "not_analyzed"));
// but we still validate the _default_ type
logger.info("Confirming _default_ mappings validation");
assertThrows(
client()
.admin()
.indices()
.preparePutMapping("test")
.setType(MapperService.DEFAULT_MAPPING)
.setSource(
JsonXContent.contentBuilder()
.startObject()
.startObject(MapperService.DEFAULT_MAPPING)
.startObject("properties")
.startObject("f")
.field("type", "DOESNT_EXIST")
.endObject()
.endObject()
.endObject()
.endObject()),
MapperParsingException.class);
}
@Test
public void test_ageOut() throws IOException, InterruptedException, ExecutionException {
// Call test_endToEnd_autoTime to create 5 time based indexes
// 2015-01-01 -> 2015-05-01
// How far is now from 2015-05-03
final Date d = TimeUtils.getDateFromSuffix("2015-03-02").success();
final long total_time_ms = new Date().getTime() - d.getTime();
final long total_days = total_time_ms / (1000L * 3600L * 24L);
final String age_out = ErrorUtils.get("{0} days", total_days);
final DataBucketBean bucket =
BeanTemplateUtils.build(DataBucketBean.class)
.with("full_name", "/test/end-end/auto-time")
.with(
DataBucketBean::data_schema,
BeanTemplateUtils.build(DataSchemaBean.class)
.with(
DataSchemaBean::temporal_schema,
BeanTemplateUtils.build(TemporalSchemaBean.class)
.with(TemporalSchemaBean::exist_age_max, age_out)
.done()
.get())
.done()
.get())
.done()
.get();
final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket);
test_endToEnd_autoTime(false);
_index_service
._crud_factory
.getClient()
.admin()
.indices()
.prepareCreate(template_name + "_2015-03-01_1")
.execute()
.actionGet();
final GetMappingsResponse gmr =
_index_service
._crud_factory
.getClient()
.admin()
.indices()
.prepareGetMappings(template_name + "*")
.execute()
.actionGet();
assertEquals(6, gmr.getMappings().keys().size());
CompletableFuture<BasicMessageBean> cf =
_index_service.getDataService().get().handleAgeOutRequest(bucket);
BasicMessageBean res = cf.get();
assertEquals(true, res.success());
assertTrue("sensible message: " + res.message(), res.message().contains(" 2 "));
assertTrue(
"Message marked as loggable: " + res.details(),
Optional.ofNullable(res.details()).filter(m -> m.containsKey("loggable")).isPresent());
System.out.println("Return from to delete: " + res.message());
Thread.sleep(5000L); // give the indexes time to delete
final GetMappingsResponse gmr2 =
_index_service
._crud_factory
.getClient()
.admin()
.indices()
.prepareGetMappings(template_name + "*")
.execute()
.actionGet();
assertEquals(3, gmr2.getMappings().keys().size());
// Check some edge cases:
// 1) Run it again, returns success but not loggable:
CompletableFuture<BasicMessageBean> cf2 =
_index_service.getDataService().get().handleAgeOutRequest(bucket);
BasicMessageBean res2 = cf2.get();
assertEquals(true, res2.success());
assertTrue("sensible message: " + res2.message(), res2.message().contains(" 0 "));
assertTrue(
"Message _not_ marked as loggable: " + res2.details(),
!Optional.ofNullable(res2.details()).map(m -> m.get("loggable")).isPresent());
// 2) No temporal settings
final DataBucketBean bucket3 =
BeanTemplateUtils.build(DataBucketBean.class)
.with("full_name", "/test/handle/age/out/delete/not/temporal")
.with(
DataBucketBean::data_schema,
BeanTemplateUtils.build(DataSchemaBean.class).done().get())
.done()
.get();
CompletableFuture<BasicMessageBean> cf3 =
_index_service.getDataService().get().handleAgeOutRequest(bucket3);
BasicMessageBean res3 = cf3.get();
// no temporal settings => returns success
assertEquals(true, res3.success());
// 3) Unparseable temporal settings (in theory won't validate but we can test here)
final DataBucketBean bucket4 =
BeanTemplateUtils.build(DataBucketBean.class)
.with("full_name", "/test/handle/age/out/delete/temporal/malformed")
.with(
DataBucketBean::data_schema,
BeanTemplateUtils.build(DataSchemaBean.class)
.with(
DataSchemaBean::temporal_schema,
BeanTemplateUtils.build(TemporalSchemaBean.class)
.with(TemporalSchemaBean::exist_age_max, "bananas")
.done()
.get())
.done()
.get())
.done()
.get();
CompletableFuture<BasicMessageBean> cf4 =
_index_service.getDataService().get().handleAgeOutRequest(bucket4);
BasicMessageBean res4 = cf4.get();
// no temporal settings => returns success
assertEquals(false, res4.success());
}
@Test
public void test_endToEnd_fixedFixed()
throws IOException, InterruptedException, ExecutionException {
final Calendar time_setter = GregorianCalendar.getInstance();
time_setter.set(2015, 1, 1, 13, 0, 0);
final String bucket_str =
Resources.toString(
Resources.getResource(
"com/ikanow/aleph2/search_service/elasticsearch/services/test_end_2_end_bucket2.json"),
Charsets.UTF_8);
final DataBucketBean bucket =
BeanTemplateUtils.build(bucket_str, DataBucketBean.class)
.with("_id", "2b_test_end_2_end")
.with("full_name", "/test/end-end/fixed/fixed")
.with("modified", time_setter.getTime())
.done()
.get();
final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket);
// Check starting from clean
{
try {
_crud_factory
.getClient()
.admin()
.indices()
.prepareDeleteTemplate(template_name)
.execute()
.actionGet();
} catch (Exception e) {
} // (This is fine, just means it doesn't exist)
try {
_crud_factory
.getClient()
.admin()
.indices()
.prepareDelete(template_name + "*")
.execute()
.actionGet();
} catch (Exception e) {
} // (This is fine, just means it doesn't exist)
final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
final GetIndexTemplatesResponse gtr =
_crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
assertTrue("No templates to start with", gtr.getIndexTemplates().isEmpty());
}
final ICrudService<JsonNode> index_service_crud =
_index_service
.getDataService()
.flatMap(
s ->
s.getWritableDataService(
JsonNode.class, bucket, Optional.empty(), Optional.empty()))
.flatMap(IDataWriteService::getCrudService)
.get();
// Check template added:
{
final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
final GetIndexTemplatesResponse gtr2 =
_crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
assertEquals(1, _index_service._bucket_template_cache.size());
assertEquals(1, gtr2.getIndexTemplates().size());
}
// Get batch sub-service
@SuppressWarnings("unchecked")
final Optional<ICrudService.IBatchSubservice<JsonNode>> batch_service =
index_service_crud
.getUnderlyingPlatformDriver(ICrudService.IBatchSubservice.class, Optional.empty())
.map(t -> (IBatchSubservice<JsonNode>) t);
{
assertTrue("Batch service must exist", batch_service.isPresent());
}
// Get information about the crud service
final ElasticsearchContext es_context =
(ElasticsearchContext)
index_service_crud
.getUnderlyingPlatformDriver(ElasticsearchContext.class, Optional.empty())
.get();
{
assertTrue("Read write index", es_context instanceof ElasticsearchContext.ReadWriteContext);
assertTrue(
"Temporal index",
es_context.indexContext()
instanceof
ElasticsearchContext.IndexContext.ReadWriteIndexContext.FixedRwIndexContext);
assertTrue(
"Auto type",
es_context.typeContext()
instanceof ElasticsearchContext.TypeContext.ReadWriteTypeContext.FixedRwTypeContext);
}
// Write some docs out
Arrays.asList(1, 2, 3, 4, 5)
.stream()
.map(
i -> {
time_setter.set(2015, i, 1, 13, 0, 0);
return time_setter.getTime();
})
.map(d -> (ObjectNode) _mapper.createObjectNode().put("@timestamp", d.getTime()))
.forEach(
o -> {
ObjectNode o1 = o.deepCopy();
o1.put("val1", 10);
ObjectNode o2 = o.deepCopy();
o2.put("val1", "test");
batch_service.get().storeObject(o1, false);
batch_service.get().storeObject(o2, false);
});
// (give it a chance to run)
Thread.sleep(5000L);
final GetMappingsResponse gmr =
es_context
.client()
.admin()
.indices()
.prepareGetMappings(template_name + "*")
.execute()
.actionGet();
// Should have 5 different indexes, each with 2 types + _default_
assertEquals(1, gmr.getMappings().keys().size());
final Set<String> expected_keys =
Arrays.asList("test_fixed_fixed__1cb6bdcdf44f").stream().collect(Collectors.toSet());
final Set<String> expected_types =
Arrays.asList("data_object").stream().collect(Collectors.toSet());
StreamSupport.stream(gmr.getMappings().spliterator(), false)
.forEach(
x -> {
assertTrue(
"Is one of the expected keys: "
+ x.key
+ " vs "
+ expected_keys.stream().collect(Collectors.joining(":")),
expected_keys.contains(x.key));
// Size 1: data_object
assertEquals(1, x.value.size());
// DEBUG
// System.out.println(" ? " + x.key);
StreamSupport.stream(x.value.spliterator(), false)
.forEach(
Lambdas.wrap_consumer_u(
y -> {
// DEBUG
// System.out.println("?? " + y.key + " --- " +
// y.value.sourceAsMap().toString());
assertTrue(
"Is expected type: " + y.key, expected_types.contains(y.key));
}));
});
// TEST DELETION:
test_handleDeleteOrPurge(bucket, false);
}
public void test_handleDeleteOrPurge(final DataBucketBean to_handle, boolean delete_not_purge)
throws InterruptedException, ExecutionException {
System.out.println("****** Checking delete/purge");
final String template_name = ElasticsearchIndexUtils.getBaseIndexName(to_handle);
final ICrudService<JsonNode> index_service_crud =
_index_service
.getDataService()
.flatMap(
s ->
s.getWritableDataService(
JsonNode.class, to_handle, Optional.empty(), Optional.empty()))
.flatMap(IDataWriteService::getCrudService)
.get();
final ElasticsearchContext es_context =
(ElasticsearchContext)
index_service_crud
.getUnderlyingPlatformDriver(ElasticsearchContext.class, Optional.empty())
.get();
// (Actually first off, check there's data and templates)
// Data:
{
final GetMappingsResponse gmr =
es_context
.client()
.admin()
.indices()
.prepareGetMappings(template_name + "*")
.execute()
.actionGet();
assertTrue("There are indexes", gmr.getMappings().keys().size() > 0);
}
// Templates:
{
final GetIndexTemplatesRequest gt_pre = new GetIndexTemplatesRequest().names(template_name);
final GetIndexTemplatesResponse gtr_pre =
_crud_factory.getClient().admin().indices().getTemplates(gt_pre).actionGet();
assertEquals(1, _index_service._bucket_template_cache.size());
assertEquals(1, gtr_pre.getIndexTemplates().size());
}
// Then, perform request
final BasicMessageBean result =
_index_service
.getDataService()
.get()
.handleBucketDeletionRequest(to_handle, Optional.empty(), delete_not_purge)
.get();
assertEquals("Deletion should succeed: " + result.message(), true, result.success());
// Check templates gone iff deleting not purging
if (delete_not_purge) {
final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
final GetIndexTemplatesResponse gtr =
_crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
assertTrue("No templates after deletion", gtr.getIndexTemplates().isEmpty());
} else {
final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
final GetIndexTemplatesResponse gtr2 =
_crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
assertEquals(1, _index_service._bucket_template_cache.size());
assertEquals(1, gtr2.getIndexTemplates().size());
}
// Check all files deleted
// Check via mappings
{
final GetMappingsResponse gmr =
es_context
.client()
.admin()
.indices()
.prepareGetMappings(template_name + "*")
.execute()
.actionGet();
assertEquals(0, gmr.getMappings().keys().size());
}
// Check via index size (recreates templates)
final ICrudService<JsonNode> index_service_crud_2 =
_index_service
.getDataService()
.flatMap(
s ->
s.getWritableDataService(
JsonNode.class, to_handle, Optional.empty(), Optional.empty()))
.flatMap(IDataWriteService::getCrudService)
.get();
assertEquals(0, index_service_crud_2.countObjects().get().intValue());
}
@SuppressWarnings("unchecked")
@Test
public void updateIncludeExclude() throws Exception {
assertAcked(
prepareCreate("test")
.addMapping(
"type",
jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("normal")
.field("type", "long")
.endObject()
.startObject("exclude")
.field("type", "long")
.endObject()
.startObject("include")
.field("type", "long")
.endObject()
.endObject()
.endObject()
.endObject()));
ensureGreen(); // make sure that replicas are initialized so the refresh command will work them
// too
logger.info("Index doc");
index(
"test",
"type",
"1",
JsonXContent.contentBuilder()
.startObject()
.field("normal", 1)
.field("exclude", 1)
.field("include", 1)
.endObject());
refresh(); // commit it for later testing.
logger.info("Adding exclude settings");
PutMappingResponse putResponse =
client()
.admin()
.indices()
.preparePutMapping("test")
.setType("type")
.setSource(
JsonXContent.contentBuilder()
.startObject()
.startObject("type")
.startObject("_source")
.startArray("excludes")
.value("exclude")
.endArray()
.endObject()
.endObject())
.get();
assertTrue(putResponse.isAcknowledged());
// changed mapping doesn't affect indexed documents (checking backward compatibility)
GetResponse getResponse = client().prepareGet("test", "type", "1").setRealtime(false).get();
assertThat(getResponse.getSource(), hasKey("normal"));
assertThat(getResponse.getSource(), hasKey("exclude"));
assertThat(getResponse.getSource(), hasKey("include"));
logger.info("Index doc again");
index(
"test",
"type",
"1",
JsonXContent.contentBuilder()
.startObject()
.field("normal", 2)
.field("exclude", 1)
.field("include", 2)
.endObject());
// but do affect newly indexed docs
getResponse = get("test", "type", "1");
assertThat(getResponse.getSource(), hasKey("normal"));
assertThat(getResponse.getSource(), not(hasKey("exclude")));
assertThat(getResponse.getSource(), hasKey("include"));
logger.info("Changing mapping to includes");
putResponse =
client()
.admin()
.indices()
.preparePutMapping("test")
.setType("type")
.setSource(
JsonXContent.contentBuilder()
.startObject()
.startObject("type")
.startObject("_source")
.startArray("excludes")
.endArray()
.startArray("includes")
.value("include")
.endArray()
.endObject()
.endObject())
.get();
assertTrue(putResponse.isAcknowledged());
GetMappingsResponse getMappingsResponse =
client().admin().indices().prepareGetMappings("test").get();
MappingMetaData typeMapping = getMappingsResponse.getMappings().get("test").get("type");
assertThat(
(Map<String, Object>) typeMapping.getSourceAsMap().get("_source"), hasKey("includes"));
ArrayList<String> includes =
(ArrayList<String>)
((Map<String, Object>) typeMapping.getSourceAsMap().get("_source")).get("includes");
assertThat(includes, contains("include"));
assertThat(
(Map<String, Object>) typeMapping.getSourceAsMap().get("_source"), hasKey("excludes"));
assertThat(
(ArrayList<String>)
((Map<String, Object>) typeMapping.getSourceAsMap().get("_source")).get("excludes"),
emptyIterable());
logger.info("Indexing doc yet again");
index(
"test",
"type",
"1",
JsonXContent.contentBuilder()
.startObject()
.field("normal", 3)
.field("exclude", 3)
.field("include", 3)
.endObject());
getResponse = get("test", "type", "1");
assertThat(getResponse.getSource(), not(hasKey("normal")));
assertThat(getResponse.getSource(), not(hasKey("exclude")));
assertThat(getResponse.getSource(), hasKey("include"));
logger.info("Adding excludes, but keep includes");
putResponse =
client()
.admin()
.indices()
.preparePutMapping("test")
.setType("type")
.setSource(
JsonXContent.contentBuilder()
.startObject()
.startObject("type")
.startObject("_source")
.startArray("excludes")
.value("*.excludes")
.endArray()
.endObject()
.endObject())
.get();
assertTrue(putResponse.isAcknowledged());
getMappingsResponse = client().admin().indices().prepareGetMappings("test").get();
typeMapping = getMappingsResponse.getMappings().get("test").get("type");
assertThat(
(Map<String, Object>) typeMapping.getSourceAsMap().get("_source"), hasKey("includes"));
includes =
(ArrayList<String>)
((Map<String, Object>) typeMapping.getSourceAsMap().get("_source")).get("includes");
assertThat(includes, contains("include"));
assertThat(
(Map<String, Object>) typeMapping.getSourceAsMap().get("_source"), hasKey("excludes"));
ArrayList<String> excludes =
(ArrayList<String>)
((Map<String, Object>) typeMapping.getSourceAsMap().get("_source")).get("excludes");
assertThat(excludes, contains("*.excludes"));
}
@Test
public void testMultiFields() throws Exception {
assertAcked(
client()
.admin()
.indices()
.prepareCreate("my-index")
.addMapping("my-type", createTypeSource()));
GetMappingsResponse getMappingsResponse =
client().admin().indices().prepareGetMappings("my-index").get();
MappingMetaData mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type");
assertThat(mappingMetaData, not(nullValue()));
Map<String, Object> mappingSource = mappingMetaData.sourceAsMap();
Map titleFields =
((Map) XContentMapValues.extractValue("properties.title.fields", mappingSource));
assertThat(titleFields.size(), equalTo(1));
assertThat(titleFields.get("not_analyzed"), notNullValue());
assertThat(
((Map) titleFields.get("not_analyzed")).get("index").toString(), equalTo("not_analyzed"));
client()
.prepareIndex("my-index", "my-type", "1")
.setSource("title", "Multi fields")
.setRefresh(true)
.get();
SearchResponse searchResponse =
client().prepareSearch("my-index").setQuery(matchQuery("title", "multi")).get();
assertThat(searchResponse.getHits().totalHits(), equalTo(1l));
searchResponse =
client()
.prepareSearch("my-index")
.setQuery(matchQuery("title.not_analyzed", "Multi fields"))
.get();
assertThat(searchResponse.getHits().totalHits(), equalTo(1l));
assertAcked(
client()
.admin()
.indices()
.preparePutMapping("my-index")
.setType("my-type")
.setSource(createPutMappingSource())
.setIgnoreConflicts(
true) // If updated with multi-field type, we need to ignore failures.
);
getMappingsResponse = client().admin().indices().prepareGetMappings("my-index").get();
mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type");
assertThat(mappingMetaData, not(nullValue()));
mappingSource = mappingMetaData.sourceAsMap();
assertThat(
((Map) XContentMapValues.extractValue("properties.title", mappingSource)).size(),
equalTo(2));
titleFields = ((Map) XContentMapValues.extractValue("properties.title.fields", mappingSource));
assertThat(titleFields.size(), equalTo(2));
assertThat(titleFields.get("not_analyzed"), notNullValue());
assertThat(
((Map) titleFields.get("not_analyzed")).get("index").toString(), equalTo("not_analyzed"));
assertThat(titleFields.get("uncased"), notNullValue());
assertThat(
((Map) titleFields.get("uncased")).get("analyzer").toString(), equalTo("whitespace"));
client()
.prepareIndex("my-index", "my-type", "1")
.setSource("title", "Multi fields")
.setRefresh(true)
.get();
searchResponse =
client().prepareSearch("my-index").setQuery(matchQuery("title.uncased", "Multi")).get();
assertThat(searchResponse.getHits().totalHits(), equalTo(1l));
}
