コード例 #1
0
  @Test
  public void test_endToEnd_fixedFixed()
      throws IOException, InterruptedException, ExecutionException {
    final Calendar time_setter = GregorianCalendar.getInstance();
    time_setter.set(2015, 1, 1, 13, 0, 0);
    final String bucket_str =
        Resources.toString(
            Resources.getResource(
                "com/ikanow/aleph2/search_service/elasticsearch/services/test_end_2_end_bucket2.json"),
            Charsets.UTF_8);
    final DataBucketBean bucket =
        BeanTemplateUtils.build(bucket_str, DataBucketBean.class)
            .with("_id", "2b_test_end_2_end")
            .with("full_name", "/test/end-end/fixed/fixed")
            .with("modified", time_setter.getTime())
            .done()
            .get();

    final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket);

    // Check starting from clean

    {
      try {
        _crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareDeleteTemplate(template_name)
            .execute()
            .actionGet();
      } catch (Exception e) {
      } // (This is fine, just means it doesn't exist)
      try {
        _crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareDelete(template_name + "*")
            .execute()
            .actionGet();
      } catch (Exception e) {
      } // (This is fine, just means it doesn't exist)

      final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr =
          _crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
      assertTrue("No templates to start with", gtr.getIndexTemplates().isEmpty());
    }

    final ICrudService<JsonNode> index_service_crud =
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, bucket, Optional.empty(), Optional.empty()))
            .flatMap(IDataWriteService::getCrudService)
            .get();

    // Check template added:

    {
      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(1, gtr2.getIndexTemplates().size());
    }

    // Get batch sub-service

    @SuppressWarnings("unchecked")
    final Optional<ICrudService.IBatchSubservice<JsonNode>> batch_service =
        index_service_crud
            .getUnderlyingPlatformDriver(ICrudService.IBatchSubservice.class, Optional.empty())
            .map(t -> (IBatchSubservice<JsonNode>) t);

    {
      assertTrue("Batch service must exist", batch_service.isPresent());
    }

    // Get information about the crud service

    final ElasticsearchContext es_context =
        (ElasticsearchContext)
            index_service_crud
                .getUnderlyingPlatformDriver(ElasticsearchContext.class, Optional.empty())
                .get();

    {
      assertTrue("Read write index", es_context instanceof ElasticsearchContext.ReadWriteContext);
      assertTrue(
          "Temporal index",
          es_context.indexContext()
              instanceof
              ElasticsearchContext.IndexContext.ReadWriteIndexContext.FixedRwIndexContext);
      assertTrue(
          "Auto type",
          es_context.typeContext()
              instanceof ElasticsearchContext.TypeContext.ReadWriteTypeContext.FixedRwTypeContext);
    }

    // Write some docs out

    Arrays.asList(1, 2, 3, 4, 5)
        .stream()
        .map(
            i -> {
              time_setter.set(2015, i, 1, 13, 0, 0);
              return time_setter.getTime();
            })
        .map(d -> (ObjectNode) _mapper.createObjectNode().put("@timestamp", d.getTime()))
        .forEach(
            o -> {
              ObjectNode o1 = o.deepCopy();
              o1.put("val1", 10);
              ObjectNode o2 = o.deepCopy();
              o2.put("val1", "test");
              batch_service.get().storeObject(o1, false);
              batch_service.get().storeObject(o2, false);
            });

    // (give it a chance to run)
    Thread.sleep(5000L);

    final GetMappingsResponse gmr =
        es_context
            .client()
            .admin()
            .indices()
            .prepareGetMappings(template_name + "*")
            .execute()
            .actionGet();

    // Should have 5 different indexes, each with 2 types + _default_

    assertEquals(1, gmr.getMappings().keys().size());
    final Set<String> expected_keys =
        Arrays.asList("test_fixed_fixed__1cb6bdcdf44f").stream().collect(Collectors.toSet());
    final Set<String> expected_types =
        Arrays.asList("data_object").stream().collect(Collectors.toSet());

    StreamSupport.stream(gmr.getMappings().spliterator(), false)
        .forEach(
            x -> {
              assertTrue(
                  "Is one of the expected keys: "
                      + x.key
                      + " vs  "
                      + expected_keys.stream().collect(Collectors.joining(":")),
                  expected_keys.contains(x.key));
              // Size 1: data_object
              assertEquals(1, x.value.size());
              // DEBUG
              // System.out.println(" ? " + x.key);
              StreamSupport.stream(x.value.spliterator(), false)
                  .forEach(
                      Lambdas.wrap_consumer_u(
                          y -> {
                            // DEBUG
                            // System.out.println("?? " + y.key + " --- " +
                            // y.value.sourceAsMap().toString());
                            assertTrue(
                                "Is expected type: " + y.key, expected_types.contains(y.key));
                          }));
            });

    // TEST DELETION:
    test_handleDeleteOrPurge(bucket, false);
  }
コード例 #2
0
  public void test_handleDeleteOrPurge(final DataBucketBean to_handle, boolean delete_not_purge)
      throws InterruptedException, ExecutionException {
    System.out.println("****** Checking delete/purge");

    final String template_name = ElasticsearchIndexUtils.getBaseIndexName(to_handle);
    final ICrudService<JsonNode> index_service_crud =
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, to_handle, Optional.empty(), Optional.empty()))
            .flatMap(IDataWriteService::getCrudService)
            .get();

    final ElasticsearchContext es_context =
        (ElasticsearchContext)
            index_service_crud
                .getUnderlyingPlatformDriver(ElasticsearchContext.class, Optional.empty())
                .get();

    // (Actually first off, check there's data and templates)
    // Data:
    {
      final GetMappingsResponse gmr =
          es_context
              .client()
              .admin()
              .indices()
              .prepareGetMappings(template_name + "*")
              .execute()
              .actionGet();
      assertTrue("There are indexes", gmr.getMappings().keys().size() > 0);
    }
    // Templates:
    {
      final GetIndexTemplatesRequest gt_pre = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr_pre =
          _crud_factory.getClient().admin().indices().getTemplates(gt_pre).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(1, gtr_pre.getIndexTemplates().size());
    }

    // Then, perform request
    final BasicMessageBean result =
        _index_service
            .getDataService()
            .get()
            .handleBucketDeletionRequest(to_handle, Optional.empty(), delete_not_purge)
            .get();
    assertEquals("Deletion should succeed: " + result.message(), true, result.success());

    // Check templates gone iff deleting not purging

    if (delete_not_purge) {
      final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr =
          _crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
      assertTrue("No templates after deletion", gtr.getIndexTemplates().isEmpty());
    } else {
      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(1, gtr2.getIndexTemplates().size());
    }

    // Check all files deleted

    // Check via mappings
    {
      final GetMappingsResponse gmr =
          es_context
              .client()
              .admin()
              .indices()
              .prepareGetMappings(template_name + "*")
              .execute()
              .actionGet();
      assertEquals(0, gmr.getMappings().keys().size());
    }
    // Check via index size (recreates templates)

    final ICrudService<JsonNode> index_service_crud_2 =
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, to_handle, Optional.empty(), Optional.empty()))
            .flatMap(IDataWriteService::getCrudService)
            .get();

    assertEquals(0, index_service_crud_2.countObjects().get().intValue());
  }