@Test public void test_indexNotEnabled() { final DataBucketBean db1 = BeanTemplateUtils.build(DataBucketBean.class).done().get(); assertEquals( Optional.empty(), _index_service .getDataService() .flatMap( s -> s.getWritableDataService( JsonNode.class, db1, Optional.empty(), Optional.empty()))); final DataBucketBean db2 = BeanTemplateUtils.build(DataBucketBean.class) .with("data_schema", BeanTemplateUtils.build(DataSchemaBean.class).done().get()) .done() .get(); assertEquals( Optional.empty(), _index_service .getDataService() .flatMap( s -> s.getWritableDataService( JsonNode.class, db2, Optional.empty(), Optional.empty()))); final DataBucketBean db3 = BeanTemplateUtils.build(DataBucketBean.class) .with( "data_schema", BeanTemplateUtils.build(DataSchemaBean.class) .with( "search_index_schema", BeanTemplateUtils.build(DataSchemaBean.SearchIndexSchemaBean.class) .with("enabled", false) .done() .get()) .done() .get()) .done() .get(); assertEquals( Optional.empty(), _index_service .getDataService() .flatMap( s -> s.getWritableDataService( JsonNode.class, db3, Optional.empty(), Optional.empty()))); }
@Test public void test_deleteNonexistantBucket() throws JsonParseException, JsonMappingException, IOException, InterruptedException, ExecutionException { final DataBucketBean bucket = BeanTemplateUtils.build(DataBucketBean.class) .with("_id", "2b_test_end_2_end_not_exist") .with("full_name", "/test/end-end/fixed/fixed/not/exist") .done() .get(); final BasicMessageBean result = _index_service .getDataService() .get() .handleBucketDeletionRequest(bucket, Optional.empty(), true) .get(); assertEquals("Deletion should succeed: " + result.message(), true, result.success()); }
@Test public void test_handleMultiBucket() { // Will currently fail because read-only indices not yet supported final DataBucketBean multi_bucket = BeanTemplateUtils.build(DataBucketBean.class) .with("_id", "test_multi_bucket") .with("multi_bucket_children", ImmutableSet.builder().add("test1").build()) .done() .get(); try { _index_service .getDataService() .flatMap( s -> s.getWritableDataService( JsonNode.class, multi_bucket, Optional.empty(), Optional.empty())); fail("Should have thrown an exception"); } catch (Exception e) { // (don't care about anything else here, this is mostly just for coverage at this point) } }
@Test public void test_ageOut() throws IOException, InterruptedException, ExecutionException { // Call test_endToEnd_autoTime to create 5 time based indexes // 2015-01-01 -> 2015-05-01 // How far is now from 2015-05-03 final Date d = TimeUtils.getDateFromSuffix("2015-03-02").success(); final long total_time_ms = new Date().getTime() - d.getTime(); final long total_days = total_time_ms / (1000L * 3600L * 24L); final String age_out = ErrorUtils.get("{0} days", total_days); final DataBucketBean bucket = BeanTemplateUtils.build(DataBucketBean.class) .with("full_name", "/test/end-end/auto-time") .with( DataBucketBean::data_schema, BeanTemplateUtils.build(DataSchemaBean.class) .with( DataSchemaBean::temporal_schema, BeanTemplateUtils.build(TemporalSchemaBean.class) .with(TemporalSchemaBean::exist_age_max, age_out) .done() .get()) .done() .get()) .done() .get(); final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket); test_endToEnd_autoTime(false); _index_service ._crud_factory .getClient() .admin() .indices() .prepareCreate(template_name + "_2015-03-01_1") .execute() .actionGet(); final GetMappingsResponse gmr = _index_service ._crud_factory .getClient() .admin() .indices() .prepareGetMappings(template_name + "*") .execute() .actionGet(); assertEquals(6, gmr.getMappings().keys().size()); CompletableFuture<BasicMessageBean> cf = _index_service.getDataService().get().handleAgeOutRequest(bucket); BasicMessageBean res = cf.get(); assertEquals(true, res.success()); assertTrue("sensible message: " + res.message(), res.message().contains(" 2 ")); assertTrue( "Message marked as loggable: " + res.details(), Optional.ofNullable(res.details()).filter(m -> m.containsKey("loggable")).isPresent()); System.out.println("Return from to delete: " + res.message()); Thread.sleep(5000L); // give the indexes time to delete final GetMappingsResponse gmr2 = _index_service ._crud_factory .getClient() .admin() .indices() .prepareGetMappings(template_name + "*") .execute() .actionGet(); assertEquals(3, gmr2.getMappings().keys().size()); // Check some edge cases: // 1) Run it again, returns success but not loggable: CompletableFuture<BasicMessageBean> cf2 = _index_service.getDataService().get().handleAgeOutRequest(bucket); BasicMessageBean res2 = cf2.get(); assertEquals(true, res2.success()); assertTrue("sensible message: " + res2.message(), res2.message().contains(" 0 ")); assertTrue( "Message _not_ marked as loggable: " + res2.details(), !Optional.ofNullable(res2.details()).map(m -> m.get("loggable")).isPresent()); // 2) No temporal settings final DataBucketBean bucket3 = BeanTemplateUtils.build(DataBucketBean.class) .with("full_name", "/test/handle/age/out/delete/not/temporal") .with( DataBucketBean::data_schema, BeanTemplateUtils.build(DataSchemaBean.class).done().get()) .done() .get(); CompletableFuture<BasicMessageBean> cf3 = _index_service.getDataService().get().handleAgeOutRequest(bucket3); BasicMessageBean res3 = cf3.get(); // no temporal settings => returns success assertEquals(true, res3.success()); // 3) Unparseable temporal settings (in theory won't validate but we can test here) final DataBucketBean bucket4 = BeanTemplateUtils.build(DataBucketBean.class) .with("full_name", "/test/handle/age/out/delete/temporal/malformed") .with( DataBucketBean::data_schema, BeanTemplateUtils.build(DataSchemaBean.class) .with( DataSchemaBean::temporal_schema, BeanTemplateUtils.build(TemporalSchemaBean.class) .with(TemporalSchemaBean::exist_age_max, "bananas") .done() .get()) .done() .get()) .done() .get(); CompletableFuture<BasicMessageBean> cf4 = _index_service.getDataService().get().handleAgeOutRequest(bucket4); BasicMessageBean res4 = cf4.get(); // no temporal settings => returns success assertEquals(false, res4.success()); }
@Test public void test_endToEnd_fixedFixed() throws IOException, InterruptedException, ExecutionException { final Calendar time_setter = GregorianCalendar.getInstance(); time_setter.set(2015, 1, 1, 13, 0, 0); final String bucket_str = Resources.toString( Resources.getResource( "com/ikanow/aleph2/search_service/elasticsearch/services/test_end_2_end_bucket2.json"), Charsets.UTF_8); final DataBucketBean bucket = BeanTemplateUtils.build(bucket_str, DataBucketBean.class) .with("_id", "2b_test_end_2_end") .with("full_name", "/test/end-end/fixed/fixed") .with("modified", time_setter.getTime()) .done() .get(); final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket); // Check starting from clean { try { _crud_factory .getClient() .admin() .indices() .prepareDeleteTemplate(template_name) .execute() .actionGet(); } catch (Exception e) { } // (This is fine, just means it doesn't exist) try { _crud_factory .getClient() .admin() .indices() .prepareDelete(template_name + "*") .execute() .actionGet(); } catch (Exception e) { } // (This is fine, just means it doesn't exist) final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name); final GetIndexTemplatesResponse gtr = _crud_factory.getClient().admin().indices().getTemplates(gt).actionGet(); assertTrue("No templates to start with", gtr.getIndexTemplates().isEmpty()); } final ICrudService<JsonNode> index_service_crud = _index_service .getDataService() .flatMap( s -> s.getWritableDataService( JsonNode.class, bucket, Optional.empty(), Optional.empty())) .flatMap(IDataWriteService::getCrudService) .get(); // Check template added: { final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name); final GetIndexTemplatesResponse gtr2 = _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet(); assertEquals(1, _index_service._bucket_template_cache.size()); assertEquals(1, gtr2.getIndexTemplates().size()); } // Get batch sub-service @SuppressWarnings("unchecked") final Optional<ICrudService.IBatchSubservice<JsonNode>> batch_service = index_service_crud .getUnderlyingPlatformDriver(ICrudService.IBatchSubservice.class, Optional.empty()) .map(t -> (IBatchSubservice<JsonNode>) t); { assertTrue("Batch service must exist", batch_service.isPresent()); } // Get information about the crud service final ElasticsearchContext es_context = (ElasticsearchContext) index_service_crud .getUnderlyingPlatformDriver(ElasticsearchContext.class, Optional.empty()) .get(); { assertTrue("Read write index", es_context instanceof ElasticsearchContext.ReadWriteContext); assertTrue( "Temporal index", es_context.indexContext() instanceof ElasticsearchContext.IndexContext.ReadWriteIndexContext.FixedRwIndexContext); assertTrue( "Auto type", es_context.typeContext() instanceof ElasticsearchContext.TypeContext.ReadWriteTypeContext.FixedRwTypeContext); } // Write some docs out Arrays.asList(1, 2, 3, 4, 5) .stream() .map( i -> { time_setter.set(2015, i, 1, 13, 0, 0); return time_setter.getTime(); }) .map(d -> (ObjectNode) _mapper.createObjectNode().put("@timestamp", d.getTime())) .forEach( o -> { ObjectNode o1 = o.deepCopy(); o1.put("val1", 10); ObjectNode o2 = o.deepCopy(); o2.put("val1", "test"); batch_service.get().storeObject(o1, false); batch_service.get().storeObject(o2, false); }); // (give it a chance to run) Thread.sleep(5000L); final GetMappingsResponse gmr = es_context .client() .admin() .indices() .prepareGetMappings(template_name + "*") .execute() .actionGet(); // Should have 5 different indexes, each with 2 types + _default_ assertEquals(1, gmr.getMappings().keys().size()); final Set<String> expected_keys = Arrays.asList("test_fixed_fixed__1cb6bdcdf44f").stream().collect(Collectors.toSet()); final Set<String> expected_types = Arrays.asList("data_object").stream().collect(Collectors.toSet()); StreamSupport.stream(gmr.getMappings().spliterator(), false) .forEach( x -> { assertTrue( "Is one of the expected keys: " + x.key + " vs " + expected_keys.stream().collect(Collectors.joining(":")), expected_keys.contains(x.key)); // Size 1: data_object assertEquals(1, x.value.size()); // DEBUG // System.out.println(" ? " + x.key); StreamSupport.stream(x.value.spliterator(), false) .forEach( Lambdas.wrap_consumer_u( y -> { // DEBUG // System.out.println("?? " + y.key + " --- " + // y.value.sourceAsMap().toString()); assertTrue( "Is expected type: " + y.key, expected_types.contains(y.key)); })); }); // TEST DELETION: test_handleDeleteOrPurge(bucket, false); }
public void test_handleDeleteOrPurge(final DataBucketBean to_handle, boolean delete_not_purge) throws InterruptedException, ExecutionException { System.out.println("****** Checking delete/purge"); final String template_name = ElasticsearchIndexUtils.getBaseIndexName(to_handle); final ICrudService<JsonNode> index_service_crud = _index_service .getDataService() .flatMap( s -> s.getWritableDataService( JsonNode.class, to_handle, Optional.empty(), Optional.empty())) .flatMap(IDataWriteService::getCrudService) .get(); final ElasticsearchContext es_context = (ElasticsearchContext) index_service_crud .getUnderlyingPlatformDriver(ElasticsearchContext.class, Optional.empty()) .get(); // (Actually first off, check there's data and templates) // Data: { final GetMappingsResponse gmr = es_context .client() .admin() .indices() .prepareGetMappings(template_name + "*") .execute() .actionGet(); assertTrue("There are indexes", gmr.getMappings().keys().size() > 0); } // Templates: { final GetIndexTemplatesRequest gt_pre = new GetIndexTemplatesRequest().names(template_name); final GetIndexTemplatesResponse gtr_pre = _crud_factory.getClient().admin().indices().getTemplates(gt_pre).actionGet(); assertEquals(1, _index_service._bucket_template_cache.size()); assertEquals(1, gtr_pre.getIndexTemplates().size()); } // Then, perform request final BasicMessageBean result = _index_service .getDataService() .get() .handleBucketDeletionRequest(to_handle, Optional.empty(), delete_not_purge) .get(); assertEquals("Deletion should succeed: " + result.message(), true, result.success()); // Check templates gone iff deleting not purging if (delete_not_purge) { final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name); final GetIndexTemplatesResponse gtr = _crud_factory.getClient().admin().indices().getTemplates(gt).actionGet(); assertTrue("No templates after deletion", gtr.getIndexTemplates().isEmpty()); } else { final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name); final GetIndexTemplatesResponse gtr2 = _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet(); assertEquals(1, _index_service._bucket_template_cache.size()); assertEquals(1, gtr2.getIndexTemplates().size()); } // Check all files deleted // Check via mappings { final GetMappingsResponse gmr = es_context .client() .admin() .indices() .prepareGetMappings(template_name + "*") .execute() .actionGet(); assertEquals(0, gmr.getMappings().keys().size()); } // Check via index size (recreates templates) final ICrudService<JsonNode> index_service_crud_2 = _index_service .getDataService() .flatMap( s -> s.getWritableDataService( JsonNode.class, to_handle, Optional.empty(), Optional.empty())) .flatMap(IDataWriteService::getCrudService) .get(); assertEquals(0, index_service_crud_2.countObjects().get().intValue()); }