@Test
public void test_indexNotEnabled() {
final DataBucketBean db1 = BeanTemplateUtils.build(DataBucketBean.class).done().get();
assertEquals(
Optional.empty(),
_index_service
.getDataService()
.flatMap(
s ->
s.getWritableDataService(
JsonNode.class, db1, Optional.empty(), Optional.empty())));
final DataBucketBean db2 =
BeanTemplateUtils.build(DataBucketBean.class)
.with("data_schema", BeanTemplateUtils.build(DataSchemaBean.class).done().get())
.done()
.get();
assertEquals(
Optional.empty(),
_index_service
.getDataService()
.flatMap(
s ->
s.getWritableDataService(
JsonNode.class, db2, Optional.empty(), Optional.empty())));
final DataBucketBean db3 =
BeanTemplateUtils.build(DataBucketBean.class)
.with(
"data_schema",
BeanTemplateUtils.build(DataSchemaBean.class)
.with(
"search_index_schema",
BeanTemplateUtils.build(DataSchemaBean.SearchIndexSchemaBean.class)
.with("enabled", false)
.done()
.get())
.done()
.get())
.done()
.get();
assertEquals(
Optional.empty(),
_index_service
.getDataService()
.flatMap(
s ->
s.getWritableDataService(
JsonNode.class, db3, Optional.empty(), Optional.empty())));
}
@Test
public void test_deleteNonexistantBucket()
throws JsonParseException, JsonMappingException, IOException, InterruptedException,
ExecutionException {
final DataBucketBean bucket =
BeanTemplateUtils.build(DataBucketBean.class)
.with("_id", "2b_test_end_2_end_not_exist")
.with("full_name", "/test/end-end/fixed/fixed/not/exist")
.done()
.get();
final BasicMessageBean result =
_index_service
.getDataService()
.get()
.handleBucketDeletionRequest(bucket, Optional.empty(), true)
.get();
assertEquals("Deletion should succeed: " + result.message(), true, result.success());
}
@Test
public void test_handleMultiBucket() {
// Will currently fail because read-only indices not yet supported
final DataBucketBean multi_bucket =
BeanTemplateUtils.build(DataBucketBean.class)
.with("_id", "test_multi_bucket")
.with("multi_bucket_children", ImmutableSet.builder().add("test1").build())
.done()
.get();
try {
_index_service
.getDataService()
.flatMap(
s ->
s.getWritableDataService(
JsonNode.class, multi_bucket, Optional.empty(), Optional.empty()));
fail("Should have thrown an exception");
} catch (Exception e) {
// (don't care about anything else here, this is mostly just for coverage at this point)
}
}
@Test
public void test_ageOut() throws IOException, InterruptedException, ExecutionException {
// Call test_endToEnd_autoTime to create 5 time based indexes
// 2015-01-01 -> 2015-05-01
// How far is now from 2015-05-03
final Date d = TimeUtils.getDateFromSuffix("2015-03-02").success();
final long total_time_ms = new Date().getTime() - d.getTime();
final long total_days = total_time_ms / (1000L * 3600L * 24L);
final String age_out = ErrorUtils.get("{0} days", total_days);
final DataBucketBean bucket =
BeanTemplateUtils.build(DataBucketBean.class)
.with("full_name", "/test/end-end/auto-time")
.with(
DataBucketBean::data_schema,
BeanTemplateUtils.build(DataSchemaBean.class)
.with(
DataSchemaBean::temporal_schema,
BeanTemplateUtils.build(TemporalSchemaBean.class)
.with(TemporalSchemaBean::exist_age_max, age_out)
.done()
.get())
.done()
.get())
.done()
.get();
final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket);
test_endToEnd_autoTime(false);
_index_service
._crud_factory
.getClient()
.admin()
.indices()
.prepareCreate(template_name + "_2015-03-01_1")
.execute()
.actionGet();
final GetMappingsResponse gmr =
_index_service
._crud_factory
.getClient()
.admin()
.indices()
.prepareGetMappings(template_name + "*")
.execute()
.actionGet();
assertEquals(6, gmr.getMappings().keys().size());
CompletableFuture<BasicMessageBean> cf =
_index_service.getDataService().get().handleAgeOutRequest(bucket);
BasicMessageBean res = cf.get();
assertEquals(true, res.success());
assertTrue("sensible message: " + res.message(), res.message().contains(" 2 "));
assertTrue(
"Message marked as loggable: " + res.details(),
Optional.ofNullable(res.details()).filter(m -> m.containsKey("loggable")).isPresent());
System.out.println("Return from to delete: " + res.message());
Thread.sleep(5000L); // give the indexes time to delete
final GetMappingsResponse gmr2 =
_index_service
._crud_factory
.getClient()
.admin()
.indices()
.prepareGetMappings(template_name + "*")
.execute()
.actionGet();
assertEquals(3, gmr2.getMappings().keys().size());
// Check some edge cases:
// 1) Run it again, returns success but not loggable:
CompletableFuture<BasicMessageBean> cf2 =
_index_service.getDataService().get().handleAgeOutRequest(bucket);
BasicMessageBean res2 = cf2.get();
assertEquals(true, res2.success());
assertTrue("sensible message: " + res2.message(), res2.message().contains(" 0 "));
assertTrue(
"Message _not_ marked as loggable: " + res2.details(),
!Optional.ofNullable(res2.details()).map(m -> m.get("loggable")).isPresent());
// 2) No temporal settings
final DataBucketBean bucket3 =
BeanTemplateUtils.build(DataBucketBean.class)
.with("full_name", "/test/handle/age/out/delete/not/temporal")
.with(
DataBucketBean::data_schema,
BeanTemplateUtils.build(DataSchemaBean.class).done().get())
.done()
.get();
CompletableFuture<BasicMessageBean> cf3 =
_index_service.getDataService().get().handleAgeOutRequest(bucket3);
BasicMessageBean res3 = cf3.get();
// no temporal settings => returns success
assertEquals(true, res3.success());
// 3) Unparseable temporal settings (in theory won't validate but we can test here)
final DataBucketBean bucket4 =
BeanTemplateUtils.build(DataBucketBean.class)
.with("full_name", "/test/handle/age/out/delete/temporal/malformed")
.with(
DataBucketBean::data_schema,
BeanTemplateUtils.build(DataSchemaBean.class)
.with(
DataSchemaBean::temporal_schema,
BeanTemplateUtils.build(TemporalSchemaBean.class)
.with(TemporalSchemaBean::exist_age_max, "bananas")
.done()
.get())
.done()
.get())
.done()
.get();
CompletableFuture<BasicMessageBean> cf4 =
_index_service.getDataService().get().handleAgeOutRequest(bucket4);
BasicMessageBean res4 = cf4.get();
// no temporal settings => returns success
assertEquals(false, res4.success());
}
@Test
public void test_endToEnd_fixedFixed()
throws IOException, InterruptedException, ExecutionException {
final Calendar time_setter = GregorianCalendar.getInstance();
time_setter.set(2015, 1, 1, 13, 0, 0);
final String bucket_str =
Resources.toString(
Resources.getResource(
"com/ikanow/aleph2/search_service/elasticsearch/services/test_end_2_end_bucket2.json"),
Charsets.UTF_8);
final DataBucketBean bucket =
BeanTemplateUtils.build(bucket_str, DataBucketBean.class)
.with("_id", "2b_test_end_2_end")
.with("full_name", "/test/end-end/fixed/fixed")
.with("modified", time_setter.getTime())
.done()
.get();
final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket);
// Check starting from clean
{
try {
_crud_factory
.getClient()
.admin()
.indices()
.prepareDeleteTemplate(template_name)
.execute()
.actionGet();
} catch (Exception e) {
} // (This is fine, just means it doesn't exist)
try {
_crud_factory
.getClient()
.admin()
.indices()
.prepareDelete(template_name + "*")
.execute()
.actionGet();
} catch (Exception e) {
} // (This is fine, just means it doesn't exist)
final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
final GetIndexTemplatesResponse gtr =
_crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
assertTrue("No templates to start with", gtr.getIndexTemplates().isEmpty());
}
final ICrudService<JsonNode> index_service_crud =
_index_service
.getDataService()
.flatMap(
s ->
s.getWritableDataService(
JsonNode.class, bucket, Optional.empty(), Optional.empty()))
.flatMap(IDataWriteService::getCrudService)
.get();
// Check template added:
{
final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
final GetIndexTemplatesResponse gtr2 =
_crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
assertEquals(1, _index_service._bucket_template_cache.size());
assertEquals(1, gtr2.getIndexTemplates().size());
}
// Get batch sub-service
@SuppressWarnings("unchecked")
final Optional<ICrudService.IBatchSubservice<JsonNode>> batch_service =
index_service_crud
.getUnderlyingPlatformDriver(ICrudService.IBatchSubservice.class, Optional.empty())
.map(t -> (IBatchSubservice<JsonNode>) t);
{
assertTrue("Batch service must exist", batch_service.isPresent());
}
// Get information about the crud service
final ElasticsearchContext es_context =
(ElasticsearchContext)
index_service_crud
.getUnderlyingPlatformDriver(ElasticsearchContext.class, Optional.empty())
.get();
{
assertTrue("Read write index", es_context instanceof ElasticsearchContext.ReadWriteContext);
assertTrue(
"Temporal index",
es_context.indexContext()
instanceof
ElasticsearchContext.IndexContext.ReadWriteIndexContext.FixedRwIndexContext);
assertTrue(
"Auto type",
es_context.typeContext()
instanceof ElasticsearchContext.TypeContext.ReadWriteTypeContext.FixedRwTypeContext);
}
// Write some docs out
Arrays.asList(1, 2, 3, 4, 5)
.stream()
.map(
i -> {
time_setter.set(2015, i, 1, 13, 0, 0);
return time_setter.getTime();
})
.map(d -> (ObjectNode) _mapper.createObjectNode().put("@timestamp", d.getTime()))
.forEach(
o -> {
ObjectNode o1 = o.deepCopy();
o1.put("val1", 10);
ObjectNode o2 = o.deepCopy();
o2.put("val1", "test");
batch_service.get().storeObject(o1, false);
batch_service.get().storeObject(o2, false);
});
// (give it a chance to run)
Thread.sleep(5000L);
final GetMappingsResponse gmr =
es_context
.client()
.admin()
.indices()
.prepareGetMappings(template_name + "*")
.execute()
.actionGet();
// Should have 5 different indexes, each with 2 types + _default_
assertEquals(1, gmr.getMappings().keys().size());
final Set<String> expected_keys =
Arrays.asList("test_fixed_fixed__1cb6bdcdf44f").stream().collect(Collectors.toSet());
final Set<String> expected_types =
Arrays.asList("data_object").stream().collect(Collectors.toSet());
StreamSupport.stream(gmr.getMappings().spliterator(), false)
.forEach(
x -> {
assertTrue(
"Is one of the expected keys: "
+ x.key
+ " vs "
+ expected_keys.stream().collect(Collectors.joining(":")),
expected_keys.contains(x.key));
// Size 1: data_object
assertEquals(1, x.value.size());
// DEBUG
// System.out.println(" ? " + x.key);
StreamSupport.stream(x.value.spliterator(), false)
.forEach(
Lambdas.wrap_consumer_u(
y -> {
// DEBUG
// System.out.println("?? " + y.key + " --- " +
// y.value.sourceAsMap().toString());
assertTrue(
"Is expected type: " + y.key, expected_types.contains(y.key));
}));
});
// TEST DELETION:
test_handleDeleteOrPurge(bucket, false);
}
public void test_handleDeleteOrPurge(final DataBucketBean to_handle, boolean delete_not_purge)
throws InterruptedException, ExecutionException {
System.out.println("****** Checking delete/purge");
final String template_name = ElasticsearchIndexUtils.getBaseIndexName(to_handle);
final ICrudService<JsonNode> index_service_crud =
_index_service
.getDataService()
.flatMap(
s ->
s.getWritableDataService(
JsonNode.class, to_handle, Optional.empty(), Optional.empty()))
.flatMap(IDataWriteService::getCrudService)
.get();
final ElasticsearchContext es_context =
(ElasticsearchContext)
index_service_crud
.getUnderlyingPlatformDriver(ElasticsearchContext.class, Optional.empty())
.get();
// (Actually first off, check there's data and templates)
// Data:
{
final GetMappingsResponse gmr =
es_context
.client()
.admin()
.indices()
.prepareGetMappings(template_name + "*")
.execute()
.actionGet();
assertTrue("There are indexes", gmr.getMappings().keys().size() > 0);
}
// Templates:
{
final GetIndexTemplatesRequest gt_pre = new GetIndexTemplatesRequest().names(template_name);
final GetIndexTemplatesResponse gtr_pre =
_crud_factory.getClient().admin().indices().getTemplates(gt_pre).actionGet();
assertEquals(1, _index_service._bucket_template_cache.size());
assertEquals(1, gtr_pre.getIndexTemplates().size());
}
// Then, perform request
final BasicMessageBean result =
_index_service
.getDataService()
.get()
.handleBucketDeletionRequest(to_handle, Optional.empty(), delete_not_purge)
.get();
assertEquals("Deletion should succeed: " + result.message(), true, result.success());
// Check templates gone iff deleting not purging
if (delete_not_purge) {
final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
final GetIndexTemplatesResponse gtr =
_crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
assertTrue("No templates after deletion", gtr.getIndexTemplates().isEmpty());
} else {
final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
final GetIndexTemplatesResponse gtr2 =
_crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
assertEquals(1, _index_service._bucket_template_cache.size());
assertEquals(1, gtr2.getIndexTemplates().size());
}
// Check all files deleted
// Check via mappings
{
final GetMappingsResponse gmr =
es_context
.client()
.admin()
.indices()
.prepareGetMappings(template_name + "*")
.execute()
.actionGet();
assertEquals(0, gmr.getMappings().keys().size());
}
// Check via index size (recreates templates)
final ICrudService<JsonNode> index_service_crud_2 =
_index_service
.getDataService()
.flatMap(
s ->
s.getWritableDataService(
JsonNode.class, to_handle, Optional.empty(), Optional.empty()))
.flatMap(IDataWriteService::getCrudService)
.get();
assertEquals(0, index_service_crud_2.countObjects().get().intValue());
}
