@Test
  public void test_verbosenessSettings() {
    final List<Object> l_true = Arrays.asList(1, "1", "true", true, "TRUE", "True");
    final List<Object> l_false = Arrays.asList(0, "0", "false", false, "FALSE", "False");

    for (Object o : l_true) {
      final DataSchemaBean.SearchIndexSchemaBean s =
          BeanTemplateUtils.build(DataSchemaBean.SearchIndexSchemaBean.class)
              .with(
                  DataSchemaBean.SearchIndexSchemaBean::technology_override_schema,
                  ImmutableMap.builder().put("verbose", o).build())
              .done()
              .get();
      assertEquals(true, ElasticsearchIndexService.is_verbose(s));
    }
    for (Object o : l_false) {
      final DataSchemaBean.SearchIndexSchemaBean s =
          BeanTemplateUtils.build(DataSchemaBean.SearchIndexSchemaBean.class)
              .with(
                  DataSchemaBean.SearchIndexSchemaBean::technology_override_schema,
                  ImmutableMap.builder().put("verbose", o).build())
              .done()
              .get();
      assertEquals(false, ElasticsearchIndexService.is_verbose(s));
    }

    // (not present)
    {
      final DataSchemaBean.SearchIndexSchemaBean s =
          BeanTemplateUtils.build(DataSchemaBean.SearchIndexSchemaBean.class)
              .with(
                  DataSchemaBean.SearchIndexSchemaBean::technology_override_schema,
                  ImmutableMap.builder().build())
              .done()
              .get();
      assertEquals(false, ElasticsearchIndexService.is_verbose(s));
    }
    {
      final DataSchemaBean.SearchIndexSchemaBean s =
          BeanTemplateUtils.build(DataSchemaBean.SearchIndexSchemaBean.class).done().get();
      assertEquals(false, ElasticsearchIndexService.is_verbose(s));
    }
  }
 protected DataBucketBean createBucket(final String harvest_tech_id) {
   // (Add streaming logic outside this via clone() - see cacheJars)
   return BeanTemplateUtils.build(DataBucketBean.class)
       .with(DataBucketBean::_id, "test1")
       .with(DataBucketBean::owner_id, "person_id")
       .with(DataBucketBean::full_name, "/test/path/")
       .with(DataBucketBean::harvest_technology_name_or_id, harvest_tech_id)
       .done()
       .get();
 }
  /**
   * Initializes the storm instance
   *
   * @return a real storm controller if possible, else a no controller
   */
  @SuppressWarnings("unchecked")
  public static IStormController getController() {
    final GlobalPropertiesBean globals =
        Lambdas.get(
            () -> {
              try {
                return BeanTemplateUtils.from(
                    PropertiesUtils.getSubConfig(
                            ModuleUtils.getStaticConfig(), GlobalPropertiesBean.PROPERTIES_ROOT)
                        .orElse(null),
                    GlobalPropertiesBean.class);
              } catch (IOException e) {
                _logger.error(
                    ErrorUtils.getLongForm(
                        "Couldn't set globals property bean in storm harvest tech onInit: {0}", e));
                return null;
              }
            });
    if (null == globals) {
      return new NoStormController();
    }
    _logger.info(
        "Loading storm config from: "
            + globals.local_yarn_config_dir()
            + File.separator
            + "storm.yaml");
    Yaml yaml = new Yaml();
    InputStream input;
    Map<String, Object> object;
    try {
      input =
          new FileInputStream(
              new File(globals.local_yarn_config_dir() + File.separator + "storm.yaml"));
      object = (Map<String, Object>) yaml.load(input);
    } catch (FileNotFoundException e) {
      _logger.error(
          ErrorUtils.getLongForm("Error reading storm.yaml in storm harvest tech onInit: {0}", e));
      object = new HashMap<String, Object>();
    }

    if (object.containsKey(backtype.storm.Config.NIMBUS_HOST)) {
      _logger.info("starting in remote mode v5");
      _logger.info(object.get(backtype.storm.Config.NIMBUS_HOST));
      // run in distributed mode
      IStormController storm_controller =
          StormControllerUtil.getRemoteStormController(
              (String) object.get(backtype.storm.Config.NIMBUS_HOST),
              (int) object.get(backtype.storm.Config.NIMBUS_THRIFT_PORT),
              (String) object.get(backtype.storm.Config.STORM_THRIFT_TRANSPORT_PLUGIN));

      return storm_controller;
    } else {
      return new NoStormController();
    }
  }
  protected List<SharedLibraryBean> createSharedLibraryBeans(Path path1, Path path2) {
    final SharedLibraryBean lib_element =
        BeanTemplateUtils.build(SharedLibraryBean.class)
            .with(SharedLibraryBean::_id, "test_tech_id_stream")
            .with(SharedLibraryBean::path_name, path1.toString())
            .with(
                SharedLibraryBean::misc_entry_point,
                "com.ikanow.aleph2.data_import.stream_enrichment.storm.PassthroughTopology")
            .done()
            .get();

    final SharedLibraryBean lib_element2 =
        BeanTemplateUtils.build(SharedLibraryBean.class)
            .with(SharedLibraryBean::_id, "test_module_id")
            .with(SharedLibraryBean::path_name, path2.toString())
            .done()
            .get();

    final SharedLibraryBean lib_element3 =
        BeanTemplateUtils.build(SharedLibraryBean.class)
            .with(SharedLibraryBean::_id, "failtest")
            .with(SharedLibraryBean::path_name, "/not_exist/here.fghgjhgjhg")
            .done()
            .get();

    // (result in classloader error)
    final SharedLibraryBean lib_element4 =
        BeanTemplateUtils.build(SharedLibraryBean.class)
            .with(SharedLibraryBean::_id, "test_tech_id_stream_fail")
            .with(SharedLibraryBean::path_name, path1.toString())
            .with(
                SharedLibraryBean::streaming_enrichment_entry_point,
                "com.ikanow.aleph2.test.example.ExampleStreamTopology")
            .done()
            .get();

    return Arrays.asList(lib_element, lib_element2, lib_element3, lib_element4);
  }
示例#5
0
 @Override
 public Tuple2<Long, IBatchRecord> getNextRecord(
     long currentFileIndex, String fileName, InputStream inStream) {
   ObjectMapper object_mapper = BeanTemplateUtils.configureMapper(Optional.empty());
   Tuple2<Long, IBatchRecord> t2 = null;
   try {
     JsonNode node = object_mapper.readTree(inStream);
     t2 =
         new Tuple2<Long, IBatchRecord>(
             currentFileIndex, new BeFileInputReader.BatchRecord(node, null));
   } catch (Exception e) {
     logger.error("JsonParser caught exception", e);
   }
   return t2;
 }
  @Test
  public void test_indexNotEnabled() {

    final DataBucketBean db1 = BeanTemplateUtils.build(DataBucketBean.class).done().get();
    assertEquals(
        Optional.empty(),
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, db1, Optional.empty(), Optional.empty())));

    final DataBucketBean db2 =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with("data_schema", BeanTemplateUtils.build(DataSchemaBean.class).done().get())
            .done()
            .get();
    assertEquals(
        Optional.empty(),
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, db2, Optional.empty(), Optional.empty())));

    final DataBucketBean db3 =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with(
                "data_schema",
                BeanTemplateUtils.build(DataSchemaBean.class)
                    .with(
                        "search_index_schema",
                        BeanTemplateUtils.build(DataSchemaBean.SearchIndexSchemaBean.class)
                            .with("enabled", false)
                            .done()
                            .get())
                    .done()
                    .get())
            .done()
            .get();
    assertEquals(
        Optional.empty(),
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, db3, Optional.empty(), Optional.empty())));
  }
  @Test
  public void test_deleteNonexistantBucket()
      throws JsonParseException, JsonMappingException, IOException, InterruptedException,
          ExecutionException {
    final DataBucketBean bucket =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with("_id", "2b_test_end_2_end_not_exist")
            .with("full_name", "/test/end-end/fixed/fixed/not/exist")
            .done()
            .get();

    final BasicMessageBean result =
        _index_service
            .getDataService()
            .get()
            .handleBucketDeletionRequest(bucket, Optional.empty(), true)
            .get();
    assertEquals("Deletion should succeed: " + result.message(), true, result.success());
  }
    @Override
    protected void configure() {
      final Config config = ModuleUtils.getStaticConfig();
      try {
        final DataImportConfigurationBean bean =
            BeanTemplateUtils.from(
                PropertiesUtils.getSubConfig(config, DataImportConfigurationBean.PROPERTIES_ROOT)
                    .orElse(null),
                DataImportConfigurationBean.class);
        this.bind(DataImportConfigurationBean.class).toInstance(bean);

        this.bind(AnalyticStateTriggerCheckFactory.class).in(Scopes.SINGLETON);
      } catch (Exception e) {
        throw new RuntimeException(
            ErrorUtils.get(
                ErrorUtils.INVALID_CONFIG_ERROR,
                DataImportConfigurationBean.class.toString(),
                config.getConfig(DataImportConfigurationBean.PROPERTIES_ROOT)),
            e);
      }
    }
  @Test
  public void test_handleMultiBucket() {
    // Will currently fail because read-only indices not yet supported

    final DataBucketBean multi_bucket =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with("_id", "test_multi_bucket")
            .with("multi_bucket_children", ImmutableSet.builder().add("test1").build())
            .done()
            .get();

    try {
      _index_service
          .getDataService()
          .flatMap(
              s ->
                  s.getWritableDataService(
                      JsonNode.class, multi_bucket, Optional.empty(), Optional.empty()));
      fail("Should have thrown an exception");
    } catch (Exception e) {
      // (don't care about anything else here, this is mostly just for coverage at this point)
    }
  }
示例#10
0
  /* (non-Javadoc)
   * @see akka.actor.AbstractActor#receive()
   */
  @Override
  public PartialFunction<Object, BoxedUnit> receive() {
    return ReceiveBuilder.match(
            BucketActionMessage.class,
            m ->
                !m.handling_clients().isEmpty()
                    && !m.handling_clients()
                        .contains(_context.getInformationService().getHostname()),
            __ -> {}) // (do nothing if it's not for me)
        .match(
            BucketActionOfferMessage.class,
            m -> {
              _logger.info(
                  ErrorUtils.get(
                      "Actor {0} received message {1} from {2} bucket {3}",
                      this.self(),
                      m.getClass().getSimpleName(),
                      this.sender(),
                      m.bucket().full_name()));

              final ActorRef closing_sender = this.sender();
              final ActorRef closing_self = this.self();

              final String hostname = _context.getInformationService().getHostname();

              // (this isn't async so doesn't require any futures)

              final boolean accept_or_ignore =
                  new File(_globals.local_yarn_config_dir() + File.separator + "storm.yaml")
                      .exists();

              final BucketActionReplyMessage reply =
                  accept_or_ignore
                      ? new BucketActionReplyMessage.BucketActionWillAcceptMessage(hostname)
                      : new BucketActionReplyMessage.BucketActionIgnoredMessage(hostname);

              closing_sender.tell(reply, closing_self);
            })
        .match(
            BucketActionMessage.class,
            m -> {
              _logger.info(
                  ErrorUtils.get(
                      "Actor {0} received message {1} from {2} bucket={3}",
                      this.self(),
                      m.getClass().getSimpleName(),
                      this.sender(),
                      m.bucket().full_name()));

              final ActorRef closing_sender = this.sender();
              final ActorRef closing_self = this.self();

              final String hostname = _context.getInformationService().getHostname();

              // (cacheJars can't throw checked or unchecked in this thread, only from within
              // exceptions)
              cacheJars(
                      m.bucket(),
                      _management_db,
                      _globals,
                      _fs,
                      _context.getServiceContext(),
                      hostname,
                      m)
                  .thenComposeAsync(
                      err_or_map -> {
                        final StreamingEnrichmentContext e_context =
                            _context.getNewStreamingEnrichmentContext();

                        final Validation<BasicMessageBean, IEnrichmentStreamingTopology>
                            err_or_tech_module =
                                getStreamingTopology(m.bucket(), m, hostname, err_or_map);

                        final CompletableFuture<BucketActionReplyMessage> ret =
                            talkToStream(
                                _storm_controller,
                                m.bucket(),
                                m,
                                err_or_tech_module,
                                err_or_map,
                                hostname,
                                e_context,
                                _globals.local_yarn_config_dir(),
                                _globals.local_cached_jar_dir());
                        return ret;
                      })
                  .thenAccept(
                      reply -> { // (reply can contain an error or successful reply, they're the
                        // same bean type)
                        // Some information logging:
                        Patterns.match(reply)
                            .andAct()
                            .when(
                                BucketActionHandlerMessage.class,
                                msg ->
                                    _logger.info(
                                        ErrorUtils.get(
                                            "Standard reply to message={0}, bucket={1}, success={2}",
                                            m.getClass().getSimpleName(),
                                            m.bucket().full_name(),
                                            msg.reply().success())))
                            .when(
                                BucketActionReplyMessage.BucketActionWillAcceptMessage.class,
                                msg ->
                                    _logger.info(
                                        ErrorUtils.get(
                                            "Standard reply to message={0}, bucket={1}",
                                            m.getClass().getSimpleName(), m.bucket().full_name())))
                            .otherwise(
                                msg ->
                                    _logger.info(
                                        ErrorUtils.get(
                                            "Unusual reply to message={0}, type={2}, bucket={1}",
                                            m.getClass().getSimpleName(),
                                            m.bucket().full_name(),
                                            msg.getClass().getSimpleName())));

                        closing_sender.tell(reply, closing_self);
                      })
                  .exceptionally(
                      e -> { // another bit of error handling that shouldn't ever be called but is a
                        // useful backstop
                        // Some information logging:
                        _logger.warn(
                            "Unexpected error replying to '{0}': error = {1}, bucket={2}",
                            BeanTemplateUtils.toJson(m).toString(),
                            ErrorUtils.getLongForm("{0}", e),
                            m.bucket().full_name());

                        final BasicMessageBean error_bean =
                            SharedErrorUtils.buildErrorMessage(
                                hostname,
                                m,
                                ErrorUtils.getLongForm(
                                    StreamErrorUtils.STREAM_UNKNOWN_ERROR,
                                    e,
                                    m.bucket().full_name()));
                        closing_sender.tell(
                            new BucketActionHandlerMessage(hostname, error_bean), closing_self);
                        return null;
                      });
            })
        .build();
  }
  @Test
  public void test_ageOut() throws IOException, InterruptedException, ExecutionException {

    // Call test_endToEnd_autoTime to create 5 time based indexes
    // 2015-01-01 -> 2015-05-01
    // How far is now from 2015-05-03
    final Date d = TimeUtils.getDateFromSuffix("2015-03-02").success();
    final long total_time_ms = new Date().getTime() - d.getTime();
    final long total_days = total_time_ms / (1000L * 3600L * 24L);
    final String age_out = ErrorUtils.get("{0} days", total_days);

    final DataBucketBean bucket =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with("full_name", "/test/end-end/auto-time")
            .with(
                DataBucketBean::data_schema,
                BeanTemplateUtils.build(DataSchemaBean.class)
                    .with(
                        DataSchemaBean::temporal_schema,
                        BeanTemplateUtils.build(TemporalSchemaBean.class)
                            .with(TemporalSchemaBean::exist_age_max, age_out)
                            .done()
                            .get())
                    .done()
                    .get())
            .done()
            .get();

    final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket);

    test_endToEnd_autoTime(false);

    _index_service
        ._crud_factory
        .getClient()
        .admin()
        .indices()
        .prepareCreate(template_name + "_2015-03-01_1")
        .execute()
        .actionGet();

    final GetMappingsResponse gmr =
        _index_service
            ._crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareGetMappings(template_name + "*")
            .execute()
            .actionGet();
    assertEquals(6, gmr.getMappings().keys().size());

    CompletableFuture<BasicMessageBean> cf =
        _index_service.getDataService().get().handleAgeOutRequest(bucket);

    BasicMessageBean res = cf.get();

    assertEquals(true, res.success());
    assertTrue("sensible message: " + res.message(), res.message().contains(" 2 "));

    assertTrue(
        "Message marked as loggable: " + res.details(),
        Optional.ofNullable(res.details()).filter(m -> m.containsKey("loggable")).isPresent());

    System.out.println("Return from to delete: " + res.message());

    Thread.sleep(5000L); // give the indexes time to delete

    final GetMappingsResponse gmr2 =
        _index_service
            ._crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareGetMappings(template_name + "*")
            .execute()
            .actionGet();
    assertEquals(3, gmr2.getMappings().keys().size());

    // Check some edge cases:

    // 1) Run it again, returns success but not loggable:

    CompletableFuture<BasicMessageBean> cf2 =
        _index_service.getDataService().get().handleAgeOutRequest(bucket);

    BasicMessageBean res2 = cf2.get();

    assertEquals(true, res2.success());
    assertTrue("sensible message: " + res2.message(), res2.message().contains(" 0 "));
    assertTrue(
        "Message _not_ marked as loggable: " + res2.details(),
        !Optional.ofNullable(res2.details()).map(m -> m.get("loggable")).isPresent());

    // 2) No temporal settings

    final DataBucketBean bucket3 =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with("full_name", "/test/handle/age/out/delete/not/temporal")
            .with(
                DataBucketBean::data_schema,
                BeanTemplateUtils.build(DataSchemaBean.class).done().get())
            .done()
            .get();

    CompletableFuture<BasicMessageBean> cf3 =
        _index_service.getDataService().get().handleAgeOutRequest(bucket3);
    BasicMessageBean res3 = cf3.get();
    // no temporal settings => returns success
    assertEquals(true, res3.success());

    // 3) Unparseable temporal settings (in theory won't validate but we can test here)

    final DataBucketBean bucket4 =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with("full_name", "/test/handle/age/out/delete/temporal/malformed")
            .with(
                DataBucketBean::data_schema,
                BeanTemplateUtils.build(DataSchemaBean.class)
                    .with(
                        DataSchemaBean::temporal_schema,
                        BeanTemplateUtils.build(TemporalSchemaBean.class)
                            .with(TemporalSchemaBean::exist_age_max, "bananas")
                            .done()
                            .get())
                    .done()
                    .get())
            .done()
            .get();

    CompletableFuture<BasicMessageBean> cf4 =
        _index_service.getDataService().get().handleAgeOutRequest(bucket4);
    BasicMessageBean res4 = cf4.get();
    // no temporal settings => returns success
    assertEquals(false, res4.success());
  }
  @Test
  public void test_endToEnd_fixedFixed()
      throws IOException, InterruptedException, ExecutionException {
    final Calendar time_setter = GregorianCalendar.getInstance();
    time_setter.set(2015, 1, 1, 13, 0, 0);
    final String bucket_str =
        Resources.toString(
            Resources.getResource(
                "com/ikanow/aleph2/search_service/elasticsearch/services/test_end_2_end_bucket2.json"),
            Charsets.UTF_8);
    final DataBucketBean bucket =
        BeanTemplateUtils.build(bucket_str, DataBucketBean.class)
            .with("_id", "2b_test_end_2_end")
            .with("full_name", "/test/end-end/fixed/fixed")
            .with("modified", time_setter.getTime())
            .done()
            .get();

    final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket);

    // Check starting from clean

    {
      try {
        _crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareDeleteTemplate(template_name)
            .execute()
            .actionGet();
      } catch (Exception e) {
      } // (This is fine, just means it doesn't exist)
      try {
        _crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareDelete(template_name + "*")
            .execute()
            .actionGet();
      } catch (Exception e) {
      } // (This is fine, just means it doesn't exist)

      final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr =
          _crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
      assertTrue("No templates to start with", gtr.getIndexTemplates().isEmpty());
    }

    final ICrudService<JsonNode> index_service_crud =
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, bucket, Optional.empty(), Optional.empty()))
            .flatMap(IDataWriteService::getCrudService)
            .get();

    // Check template added:

    {
      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(1, gtr2.getIndexTemplates().size());
    }

    // Get batch sub-service

    @SuppressWarnings("unchecked")
    final Optional<ICrudService.IBatchSubservice<JsonNode>> batch_service =
        index_service_crud
            .getUnderlyingPlatformDriver(ICrudService.IBatchSubservice.class, Optional.empty())
            .map(t -> (IBatchSubservice<JsonNode>) t);

    {
      assertTrue("Batch service must exist", batch_service.isPresent());
    }

    // Get information about the crud service

    final ElasticsearchContext es_context =
        (ElasticsearchContext)
            index_service_crud
                .getUnderlyingPlatformDriver(ElasticsearchContext.class, Optional.empty())
                .get();

    {
      assertTrue("Read write index", es_context instanceof ElasticsearchContext.ReadWriteContext);
      assertTrue(
          "Temporal index",
          es_context.indexContext()
              instanceof
              ElasticsearchContext.IndexContext.ReadWriteIndexContext.FixedRwIndexContext);
      assertTrue(
          "Auto type",
          es_context.typeContext()
              instanceof ElasticsearchContext.TypeContext.ReadWriteTypeContext.FixedRwTypeContext);
    }

    // Write some docs out

    Arrays.asList(1, 2, 3, 4, 5)
        .stream()
        .map(
            i -> {
              time_setter.set(2015, i, 1, 13, 0, 0);
              return time_setter.getTime();
            })
        .map(d -> (ObjectNode) _mapper.createObjectNode().put("@timestamp", d.getTime()))
        .forEach(
            o -> {
              ObjectNode o1 = o.deepCopy();
              o1.put("val1", 10);
              ObjectNode o2 = o.deepCopy();
              o2.put("val1", "test");
              batch_service.get().storeObject(o1, false);
              batch_service.get().storeObject(o2, false);
            });

    // (give it a chance to run)
    Thread.sleep(5000L);

    final GetMappingsResponse gmr =
        es_context
            .client()
            .admin()
            .indices()
            .prepareGetMappings(template_name + "*")
            .execute()
            .actionGet();

    // Should have 5 different indexes, each with 2 types + _default_

    assertEquals(1, gmr.getMappings().keys().size());
    final Set<String> expected_keys =
        Arrays.asList("test_fixed_fixed__1cb6bdcdf44f").stream().collect(Collectors.toSet());
    final Set<String> expected_types =
        Arrays.asList("data_object").stream().collect(Collectors.toSet());

    StreamSupport.stream(gmr.getMappings().spliterator(), false)
        .forEach(
            x -> {
              assertTrue(
                  "Is one of the expected keys: "
                      + x.key
                      + " vs  "
                      + expected_keys.stream().collect(Collectors.joining(":")),
                  expected_keys.contains(x.key));
              // Size 1: data_object
              assertEquals(1, x.value.size());
              // DEBUG
              // System.out.println(" ? " + x.key);
              StreamSupport.stream(x.value.spliterator(), false)
                  .forEach(
                      Lambdas.wrap_consumer_u(
                          y -> {
                            // DEBUG
                            // System.out.println("?? " + y.key + " --- " +
                            // y.value.sourceAsMap().toString());
                            assertTrue(
                                "Is expected type: " + y.key, expected_types.contains(y.key));
                          }));
            });

    // TEST DELETION:
    test_handleDeleteOrPurge(bucket, false);
  }
public class TestElasticsearchIndexService {

  public static ObjectMapper _mapper = BeanTemplateUtils.configureMapper(Optional.empty());

  protected MockElasticsearchIndexService _index_service;
  protected IElasticsearchCrudServiceFactory _crud_factory;
  protected ElasticsearchIndexServiceConfigBean _config_bean;

  // Set this string to connect vs a real DB
  private final String _connection_string = null;
  private final String _cluster_name = null;
  //	private final String _connection_string = "localhost:4093";
  //	private final String _cluster_name = "infinite-dev";

  @Before
  public void setupServices() {
    final Config full_config = ConfigFactory.empty();
    if (null == _connection_string) {
      _crud_factory = new MockElasticsearchCrudServiceFactory();
    } else {
      final ElasticsearchConfigurationBean config_bean =
          new ElasticsearchConfigurationBean(_connection_string, _cluster_name);
      _crud_factory = new ElasticsearchCrudServiceFactory(config_bean);
    }
    _config_bean = ElasticsearchIndexConfigUtils.buildConfigBean(full_config);

    _index_service = new MockElasticsearchIndexService(_crud_factory, _config_bean);
  }

  ////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  // VALIDATION

  @Test
  public void test_verbosenessSettings() {
    final List<Object> l_true = Arrays.asList(1, "1", "true", true, "TRUE", "True");
    final List<Object> l_false = Arrays.asList(0, "0", "false", false, "FALSE", "False");

    for (Object o : l_true) {
      final DataSchemaBean.SearchIndexSchemaBean s =
          BeanTemplateUtils.build(DataSchemaBean.SearchIndexSchemaBean.class)
              .with(
                  DataSchemaBean.SearchIndexSchemaBean::technology_override_schema,
                  ImmutableMap.builder().put("verbose", o).build())
              .done()
              .get();
      assertEquals(true, ElasticsearchIndexService.is_verbose(s));
    }
    for (Object o : l_false) {
      final DataSchemaBean.SearchIndexSchemaBean s =
          BeanTemplateUtils.build(DataSchemaBean.SearchIndexSchemaBean.class)
              .with(
                  DataSchemaBean.SearchIndexSchemaBean::technology_override_schema,
                  ImmutableMap.builder().put("verbose", o).build())
              .done()
              .get();
      assertEquals(false, ElasticsearchIndexService.is_verbose(s));
    }

    // (not present)
    {
      final DataSchemaBean.SearchIndexSchemaBean s =
          BeanTemplateUtils.build(DataSchemaBean.SearchIndexSchemaBean.class)
              .with(
                  DataSchemaBean.SearchIndexSchemaBean::technology_override_schema,
                  ImmutableMap.builder().build())
              .done()
              .get();
      assertEquals(false, ElasticsearchIndexService.is_verbose(s));
    }
    {
      final DataSchemaBean.SearchIndexSchemaBean s =
          BeanTemplateUtils.build(DataSchemaBean.SearchIndexSchemaBean.class).done().get();
      assertEquals(false, ElasticsearchIndexService.is_verbose(s));
    }
  }

  @Test
  public void test_validationSuccess() throws IOException {
    final String bucket_str =
        Resources.toString(
            Resources.getResource(
                "com/ikanow/aleph2/search_service/elasticsearch/services/test_bucket_validate_success.json"),
            Charsets.UTF_8);
    final DataBucketBean bucket =
        BeanTemplateUtils.build(bucket_str, DataBucketBean.class).done().get();

    // 1) Verbose mode off
    {
      final Collection<BasicMessageBean> res_col =
          _index_service.validateSchema(bucket.data_schema().columnar_schema(), bucket)._2();
      final Collection<BasicMessageBean> res_search =
          _index_service.validateSchema(bucket.data_schema().search_index_schema(), bucket)._2();
      final Collection<BasicMessageBean> res_time =
          _index_service.validateSchema(bucket.data_schema().temporal_schema(), bucket)._2();

      assertEquals(0, res_col.size());
      assertEquals(0, res_search.size());
      assertEquals(0, res_time.size());
    }

    // 2) Verbose mode on
    {
      final DataBucketBean bucket_verbose =
          BeanTemplateUtils.clone(bucket)
              .with(
                  DataBucketBean::data_schema,
                  BeanTemplateUtils.clone(bucket.data_schema())
                      .with(
                          DataSchemaBean::search_index_schema,
                          BeanTemplateUtils.clone(bucket.data_schema().search_index_schema())
                              .with(
                                  DataSchemaBean.SearchIndexSchemaBean::technology_override_schema,
                                  ImmutableMap.builder()
                                      .putAll(
                                          bucket
                                              .data_schema()
                                              .search_index_schema()
                                              .technology_override_schema())
                                      .put("verbose", true)
                                      .build())
                              .done())
                      .done())
              .done();

      final Collection<BasicMessageBean> res_col =
          _index_service
              .validateSchema(bucket_verbose.data_schema().columnar_schema(), bucket)
              ._2();
      final Collection<BasicMessageBean> res_search =
          _index_service
              .validateSchema(bucket_verbose.data_schema().search_index_schema(), bucket)
              ._2();
      final Collection<BasicMessageBean> res_time =
          _index_service
              .validateSchema(bucket_verbose.data_schema().temporal_schema(), bucket)
              ._2();

      assertEquals(0, res_col.size());
      assertEquals(0, res_time.size());
      assertEquals(2, res_search.size());
      assertEquals(true, res_search.stream().allMatch(BasicMessageBean::success));
      Iterator<BasicMessageBean> res_search_message = res_search.iterator();

      final String mapping_str =
          Resources.toString(
              Resources.getResource(
                  "com/ikanow/aleph2/search_service/elasticsearch/services/test_verbose_mapping_validate_results.json"),
              Charsets.UTF_8);
      final JsonNode mapping_json = _mapper.readTree(mapping_str.getBytes());
      assertEquals(
          mapping_json.toString(),
          _mapper.readTree(res_search_message.next().message()).toString());
      assertTrue(
          "Sets the max index override: "
              + res_search.stream().skip(1).map(m -> m.message()).collect(Collectors.joining()),
          res_search_message.next().message().contains("1,000 MB"));
    }

    // 3) Temporal

    {
      final DataBucketBean bucket_temporal_no_grouping =
          BeanTemplateUtils.clone(bucket)
              .with(
                  DataBucketBean::data_schema,
                  BeanTemplateUtils.clone(bucket.data_schema())
                      .with(
                          DataSchemaBean::temporal_schema,
                          BeanTemplateUtils.build(DataSchemaBean.TemporalSchemaBean.class)
                              .done()
                              .get())
                      .done())
              .done();

      assertEquals(
          "",
          _index_service
              .validateSchema(bucket_temporal_no_grouping.data_schema().temporal_schema(), bucket)
              ._1());

      final DataBucketBean bucket_temporal_grouping =
          BeanTemplateUtils.clone(bucket)
              .with(
                  DataBucketBean::data_schema,
                  BeanTemplateUtils.clone(bucket.data_schema())
                      .with(
                          DataSchemaBean::temporal_schema,
                          BeanTemplateUtils.build(DataSchemaBean.TemporalSchemaBean.class)
                              .with(DataSchemaBean.TemporalSchemaBean::grouping_time_period, "1d")
                              .done()
                              .get())
                      .done())
              .done();

      assertEquals(
          "_{yyyy-MM-dd}",
          _index_service
              .validateSchema(bucket_temporal_grouping.data_schema().temporal_schema(), bucket)
              ._1());
    }
  }

  @Test
  public void test_validationFail() throws IOException {

    final String bucket_str =
        Resources.toString(
            Resources.getResource(
                "com/ikanow/aleph2/search_service/elasticsearch/services/test_bucket_validate_fail.json"),
            Charsets.UTF_8);
    final DataBucketBean bucket =
        BeanTemplateUtils.build(bucket_str, DataBucketBean.class).done().get();

    // 1) Verbose mode off
    {
      final Collection<BasicMessageBean> res_col =
          _index_service.validateSchema(bucket.data_schema().columnar_schema(), bucket)._2();
      final Collection<BasicMessageBean> res_search =
          _index_service.validateSchema(bucket.data_schema().search_index_schema(), bucket)._2();
      final Collection<BasicMessageBean> res_time =
          _index_service.validateSchema(bucket.data_schema().temporal_schema(), bucket)._2();

      assertEquals(0, res_col.size());
      assertEquals(0, res_time.size());
      assertEquals(1, res_search.size());

      final BasicMessageBean res_search_message = res_search.iterator().next();
      assertEquals(false, res_search_message.success());
    }

    // 2) Check setting an invalid max index size
    {
      final String bucket_str_2 =
          Resources.toString(
              Resources.getResource(
                  "com/ikanow/aleph2/search_service/elasticsearch/services/test_bucket_validate_success.json"),
              Charsets.UTF_8);
      final DataBucketBean bucket2 =
          BeanTemplateUtils.build(bucket_str_2, DataBucketBean.class).done().get();
      final DataBucketBean bucket_too_small =
          BeanTemplateUtils.clone(bucket2)
              .with(
                  DataBucketBean::data_schema,
                  BeanTemplateUtils.clone(bucket2.data_schema())
                      .with(
                          DataSchemaBean::search_index_schema,
                          BeanTemplateUtils.clone(bucket2.data_schema().search_index_schema())
                              .with(DataSchemaBean.SearchIndexSchemaBean::target_index_size_mb, 10L)
                              .done())
                      .done())
              .done();
      final Collection<BasicMessageBean> res_search =
          _index_service
              .validateSchema(bucket.data_schema().search_index_schema(), bucket_too_small)
              ._2();
      assertEquals(1, res_search.size());
      assertEquals(false, res_search.stream().allMatch(BasicMessageBean::success));
      BasicMessageBean res_search_message = res_search.iterator().next();
      assertTrue(
          "Right message: " + res_search_message.message(),
          res_search_message.message().contains("10 MB"));
    }
  }

  ////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  // INDEX NOT ENABLED

  @Test
  public void test_indexNotEnabled() {

    final DataBucketBean db1 = BeanTemplateUtils.build(DataBucketBean.class).done().get();
    assertEquals(
        Optional.empty(),
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, db1, Optional.empty(), Optional.empty())));

    final DataBucketBean db2 =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with("data_schema", BeanTemplateUtils.build(DataSchemaBean.class).done().get())
            .done()
            .get();
    assertEquals(
        Optional.empty(),
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, db2, Optional.empty(), Optional.empty())));

    final DataBucketBean db3 =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with(
                "data_schema",
                BeanTemplateUtils.build(DataSchemaBean.class)
                    .with(
                        "search_index_schema",
                        BeanTemplateUtils.build(DataSchemaBean.SearchIndexSchemaBean.class)
                            .with("enabled", false)
                            .done()
                            .get())
                    .done()
                    .get())
            .done()
            .get();
    assertEquals(
        Optional.empty(),
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, db3, Optional.empty(), Optional.empty())));
  }

  ////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  // INDEX MANAGEMENT

  @Test
  public void test_indexCreation() throws IOException {

    final Calendar time_setter = GregorianCalendar.getInstance();
    time_setter.set(2015, 1, 1, 13, 0, 0);

    final String bucket_str =
        Resources.toString(
            Resources.getResource(
                "com/ikanow/aleph2/search_service/elasticsearch/services/test_bucket_validate_success.json"),
            Charsets.UTF_8);
    final DataBucketBean bucket =
        BeanTemplateUtils.build(bucket_str, DataBucketBean.class)
            .with("modified", time_setter.getTime())
            .done()
            .get();

    final String mapping_str =
        Resources.toString(
            Resources.getResource(
                "com/ikanow/aleph2/search_service/elasticsearch/services/test_verbose_mapping_validate_results.json"),
            Charsets.UTF_8);
    final JsonNode mapping_json = _mapper.readTree(mapping_str.getBytes());

    final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket);

    try {
      _crud_factory
          .getClient()
          .admin()
          .indices()
          .prepareDeleteTemplate(template_name)
          .execute()
          .actionGet();
    } catch (Exception e) {
    } // (This is fine, just means it doesn't exist)

    // Create index template from empty

    {
      final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr =
          _crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
      assertTrue("No templates to start with", gtr.getIndexTemplates().isEmpty());

      _index_service.handlePotentiallyNewIndex(
          bucket,
          Optional.empty(),
          ElasticsearchIndexConfigUtils.buildConfigBeanFromSchema(bucket, _config_bean, _mapper),
          "_default_");

      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(1, gtr2.getIndexTemplates().size());

      assertTrue(
          "Mappings should be equivalent",
          ElasticsearchIndexService.mappingsAreEquivalent(
              gtr2.getIndexTemplates().get(0), mapping_json, _mapper));
    }

    // Check is ignored subsequently (same date, same content; same date, different content)
    {
      _index_service.handlePotentiallyNewIndex(
          bucket,
          Optional.empty(),
          ElasticsearchIndexConfigUtils.buildConfigBeanFromSchema(bucket, _config_bean, _mapper),
          "_default_");

      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(1, gtr2.getIndexTemplates().size());
    }

    // Check is checked-but-left if time updated, content not
    {
      time_setter.set(2015, 1, 1, 14, 0, 0);
      final Date next_time = time_setter.getTime();
      final DataBucketBean bucket2 =
          BeanTemplateUtils.clone(bucket).with("modified", next_time).done();

      _index_service.handlePotentiallyNewIndex(
          bucket2,
          Optional.empty(),
          ElasticsearchIndexConfigUtils.buildConfigBeanFromSchema(bucket2, _config_bean, _mapper),
          "_default_");

      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(next_time, _index_service._bucket_template_cache.get(bucket._id()));
      assertEquals(1, gtr2.getIndexTemplates().size());
    }

    // Check is updated if time-and-content is different
    {
      time_setter.set(2015, 1, 1, 15, 0, 0);
      final String bucket_str2 =
          Resources.toString(
              Resources.getResource(
                  "com/ikanow/aleph2/search_service/elasticsearch/services/test_bucket2_validate_success.json"),
              Charsets.UTF_8);
      final DataBucketBean bucket2 =
          BeanTemplateUtils.build(bucket_str2, DataBucketBean.class)
              .with("modified", time_setter.getTime())
              .done()
              .get();

      _index_service.handlePotentiallyNewIndex(
          bucket2,
          Optional.empty(),
          ElasticsearchIndexConfigUtils.buildConfigBeanFromSchema(bucket2, _config_bean, _mapper),
          "_default_");

      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(time_setter.getTime(), _index_service._bucket_template_cache.get(bucket._id()));
      assertEquals(1, gtr2.getIndexTemplates().size());

      assertFalse(
          ElasticsearchIndexService.mappingsAreEquivalent(
              gtr2.getIndexTemplates().get(0), mapping_json, _mapper)); // has changed
    }

    // Check if mapping is deleted then next time bucket modified is updated then the mapping is
    // recreated

    {
      _crud_factory
          .getClient()
          .admin()
          .indices()
          .prepareDeleteTemplate(template_name)
          .execute()
          .actionGet();

      // (check with old date)

      final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr =
          _crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
      assertTrue("No templates to start with", gtr.getIndexTemplates().isEmpty());

      {
        _index_service.handlePotentiallyNewIndex(
            bucket,
            Optional.empty(),
            ElasticsearchIndexConfigUtils.buildConfigBeanFromSchema(bucket, _config_bean, _mapper),
            "_default_");

        final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
        final GetIndexTemplatesResponse gtr2 =
            _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
        assertTrue("Initially no change", gtr2.getIndexTemplates().isEmpty());
      }

      // Update date and retry

      {
        time_setter.set(2015, 1, 1, 16, 0, 0);
        final Date next_time = time_setter.getTime();
        final DataBucketBean bucket2 =
            BeanTemplateUtils.clone(bucket).with("modified", next_time).done();

        _index_service.handlePotentiallyNewIndex(
            bucket2,
            Optional.empty(),
            ElasticsearchIndexConfigUtils.buildConfigBeanFromSchema(bucket2, _config_bean, _mapper),
            "_default_");

        final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
        final GetIndexTemplatesResponse gtr2 =
            _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
        assertEquals(1, _index_service._bucket_template_cache.size());
        assertEquals(1, gtr2.getIndexTemplates().size());

        assertTrue(
            "Mappings should be equivalent",
            ElasticsearchIndexService.mappingsAreEquivalent(
                gtr2.getIndexTemplates().get(0), mapping_json, _mapper));
      }
    }
  }

  ////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  // END-TO-END

  @Test
  public void test_handleMultiBucket() {
    // Will currently fail because read-only indices not yet supported

    final DataBucketBean multi_bucket =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with("_id", "test_multi_bucket")
            .with("multi_bucket_children", ImmutableSet.builder().add("test1").build())
            .done()
            .get();

    try {
      _index_service
          .getDataService()
          .flatMap(
              s ->
                  s.getWritableDataService(
                      JsonNode.class, multi_bucket, Optional.empty(), Optional.empty()));
      fail("Should have thrown an exception");
    } catch (Exception e) {
      // (don't care about anything else here, this is mostly just for coverage at this point)
    }
  }

  // (including getCrudService)

  @Test
  public void test_endToEnd_autoTime()
      throws IOException, InterruptedException, ExecutionException {
    test_endToEnd_autoTime(true);
  }

  public void test_endToEnd_autoTime(boolean test_not_create_mode)
      throws IOException, InterruptedException, ExecutionException {
    final Calendar time_setter = GregorianCalendar.getInstance();
    time_setter.set(2015, 1, 1, 13, 0, 0);
    final String bucket_str =
        Resources.toString(
            Resources.getResource(
                "com/ikanow/aleph2/search_service/elasticsearch/services/test_end_2_end_bucket.json"),
            Charsets.UTF_8);
    final DataBucketBean bucket =
        BeanTemplateUtils.build(bucket_str, DataBucketBean.class)
            .with("_id", "test_end_2_end")
            .with("full_name", "/test/end-end/auto-time")
            .with("modified", time_setter.getTime())
            .done()
            .get();

    final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket);

    // Check starting from clean

    {
      try {
        _crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareDeleteTemplate(template_name)
            .execute()
            .actionGet();
      } catch (Exception e) {
      } // (This is fine, just means it doesn't exist)
      try {
        _crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareDelete(template_name + "*")
            .execute()
            .actionGet();
      } catch (Exception e) {
      } // (This is fine, just means it doesn't exist)

      final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr =
          _crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
      assertTrue("No templates to start with", gtr.getIndexTemplates().isEmpty());
    }

    final ICrudService<JsonNode> index_service_crud =
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, bucket, Optional.empty(), Optional.empty()))
            .flatMap(IDataWriteService::getCrudService)
            .get();

    // Check template added:

    {
      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(1, gtr2.getIndexTemplates().size());
    }

    // Get batch sub-service

    @SuppressWarnings("unchecked")
    final Optional<ICrudService.IBatchSubservice<JsonNode>> batch_service =
        index_service_crud
            .getUnderlyingPlatformDriver(ICrudService.IBatchSubservice.class, Optional.empty())
            .map(t -> (IBatchSubservice<JsonNode>) t);

    {
      assertTrue("Batch service must exist", batch_service.isPresent());
    }

    // Get information about the crud service

    final ElasticsearchContext es_context =
        (ElasticsearchContext)
            index_service_crud
                .getUnderlyingPlatformDriver(ElasticsearchContext.class, Optional.empty())
                .get();

    {
      assertTrue("Read write index", es_context instanceof ElasticsearchContext.ReadWriteContext);
      assertTrue(
          "Temporal index",
          es_context.indexContext()
              instanceof
              ElasticsearchContext.IndexContext.ReadWriteIndexContext.TimedRwIndexContext);
      assertTrue(
          "Auto type",
          es_context.typeContext()
              instanceof ElasticsearchContext.TypeContext.ReadWriteTypeContext.AutoRwTypeContext);

      // Check the the context contains the invalid

      final ElasticsearchContext.TypeContext.ReadWriteTypeContext.AutoRwTypeContext context =
          (ElasticsearchContext.TypeContext.ReadWriteTypeContext.AutoRwTypeContext)
              es_context.typeContext();

      assertEquals(
          Arrays.asList("@timestamp"),
          context.fixed_type_fields().stream().collect(Collectors.toList()));
    }

    // Write some docs out

    Arrays.asList(1, 2, 3, 4, 5)
        .stream()
        .map(
            i -> {
              time_setter.set(2015, i, 1, 13, 0, 0);
              return time_setter.getTime();
            })
        .map(d -> (ObjectNode) _mapper.createObjectNode().put("@timestamp", d.getTime()))
        .forEach(
            o -> {
              ObjectNode o1 = o.deepCopy();
              o1.set("val1", _mapper.createObjectNode().put("val2", "test"));
              ObjectNode o2 = o.deepCopy();
              o2.put("val1", "test");
              batch_service.get().storeObject(o1, false);
              batch_service.get().storeObject(o2, false);
            });

    for (int i = 0; i < 30; ++i) {
      Thread.sleep(1000L);
      if (index_service_crud.countObjects().get() >= 10) {
        System.out.println("Test end 2 end: (Got all the records)");
        break;
      }
    }

    final GetMappingsResponse gmr =
        es_context
            .client()
            .admin()
            .indices()
            .prepareGetMappings(template_name + "*")
            .execute()
            .actionGet();

    // Should have 5 different indexes, each with 2 types + _default_

    assertEquals(5, gmr.getMappings().keys().size());
    final Set<String> expected_keys =
        Arrays.asList(1, 2, 3, 4, 5)
            .stream()
            .map(i -> template_name + "_2015-0" + (i + 1) + "-01")
            .collect(Collectors.toSet());
    final Set<String> expected_types =
        Arrays.asList("_default_", "type_1", "type_2").stream().collect(Collectors.toSet());

    if (test_not_create_mode)
      StreamSupport.stream(gmr.getMappings().spliterator(), false)
          .forEach(
              x -> {
                assertTrue(
                    "Is one of the expected keys: "
                        + x.key
                        + " vs  "
                        + expected_keys.stream().collect(Collectors.joining(":")),
                    expected_keys.contains(x.key));
                // DEBUG
                // System.out.println(" ? " + x.key);
                StreamSupport.stream(x.value.spliterator(), false)
                    .forEach(
                        Lambdas.wrap_consumer_u(
                            y -> {
                              // DEBUG
                              // System.out.println("?? " + y.key + " --- " +
                              // y.value.sourceAsMap().toString());
                              // Size 3: _default_, type1 and type2
                              assertTrue(
                                  "Is expected type: " + y.key, expected_types.contains(y.key));
                            }));
                // Size 3: _default_, type_1, type_2
                assertEquals("Should have 3 indexes: " + x.value.toString(), 3, x.value.size());
              });

    // TEST DELETION:
    if (test_not_create_mode) test_handleDeleteOrPurge(bucket, true);
  }

  @Test
  public void test_endToEnd_fixedFixed()
      throws IOException, InterruptedException, ExecutionException {
    final Calendar time_setter = GregorianCalendar.getInstance();
    time_setter.set(2015, 1, 1, 13, 0, 0);
    final String bucket_str =
        Resources.toString(
            Resources.getResource(
                "com/ikanow/aleph2/search_service/elasticsearch/services/test_end_2_end_bucket2.json"),
            Charsets.UTF_8);
    final DataBucketBean bucket =
        BeanTemplateUtils.build(bucket_str, DataBucketBean.class)
            .with("_id", "2b_test_end_2_end")
            .with("full_name", "/test/end-end/fixed/fixed")
            .with("modified", time_setter.getTime())
            .done()
            .get();

    final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket);

    // Check starting from clean

    {
      try {
        _crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareDeleteTemplate(template_name)
            .execute()
            .actionGet();
      } catch (Exception e) {
      } // (This is fine, just means it doesn't exist)
      try {
        _crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareDelete(template_name + "*")
            .execute()
            .actionGet();
      } catch (Exception e) {
      } // (This is fine, just means it doesn't exist)

      final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr =
          _crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
      assertTrue("No templates to start with", gtr.getIndexTemplates().isEmpty());
    }

    final ICrudService<JsonNode> index_service_crud =
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, bucket, Optional.empty(), Optional.empty()))
            .flatMap(IDataWriteService::getCrudService)
            .get();

    // Check template added:

    {
      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(1, gtr2.getIndexTemplates().size());
    }

    // Get batch sub-service

    @SuppressWarnings("unchecked")
    final Optional<ICrudService.IBatchSubservice<JsonNode>> batch_service =
        index_service_crud
            .getUnderlyingPlatformDriver(ICrudService.IBatchSubservice.class, Optional.empty())
            .map(t -> (IBatchSubservice<JsonNode>) t);

    {
      assertTrue("Batch service must exist", batch_service.isPresent());
    }

    // Get information about the crud service

    final ElasticsearchContext es_context =
        (ElasticsearchContext)
            index_service_crud
                .getUnderlyingPlatformDriver(ElasticsearchContext.class, Optional.empty())
                .get();

    {
      assertTrue("Read write index", es_context instanceof ElasticsearchContext.ReadWriteContext);
      assertTrue(
          "Temporal index",
          es_context.indexContext()
              instanceof
              ElasticsearchContext.IndexContext.ReadWriteIndexContext.FixedRwIndexContext);
      assertTrue(
          "Auto type",
          es_context.typeContext()
              instanceof ElasticsearchContext.TypeContext.ReadWriteTypeContext.FixedRwTypeContext);
    }

    // Write some docs out

    Arrays.asList(1, 2, 3, 4, 5)
        .stream()
        .map(
            i -> {
              time_setter.set(2015, i, 1, 13, 0, 0);
              return time_setter.getTime();
            })
        .map(d -> (ObjectNode) _mapper.createObjectNode().put("@timestamp", d.getTime()))
        .forEach(
            o -> {
              ObjectNode o1 = o.deepCopy();
              o1.put("val1", 10);
              ObjectNode o2 = o.deepCopy();
              o2.put("val1", "test");
              batch_service.get().storeObject(o1, false);
              batch_service.get().storeObject(o2, false);
            });

    // (give it a chance to run)
    Thread.sleep(5000L);

    final GetMappingsResponse gmr =
        es_context
            .client()
            .admin()
            .indices()
            .prepareGetMappings(template_name + "*")
            .execute()
            .actionGet();

    // Should have 5 different indexes, each with 2 types + _default_

    assertEquals(1, gmr.getMappings().keys().size());
    final Set<String> expected_keys =
        Arrays.asList("test_fixed_fixed__1cb6bdcdf44f").stream().collect(Collectors.toSet());
    final Set<String> expected_types =
        Arrays.asList("data_object").stream().collect(Collectors.toSet());

    StreamSupport.stream(gmr.getMappings().spliterator(), false)
        .forEach(
            x -> {
              assertTrue(
                  "Is one of the expected keys: "
                      + x.key
                      + " vs  "
                      + expected_keys.stream().collect(Collectors.joining(":")),
                  expected_keys.contains(x.key));
              // Size 1: data_object
              assertEquals(1, x.value.size());
              // DEBUG
              // System.out.println(" ? " + x.key);
              StreamSupport.stream(x.value.spliterator(), false)
                  .forEach(
                      Lambdas.wrap_consumer_u(
                          y -> {
                            // DEBUG
                            // System.out.println("?? " + y.key + " --- " +
                            // y.value.sourceAsMap().toString());
                            assertTrue(
                                "Is expected type: " + y.key, expected_types.contains(y.key));
                          }));
            });

    // TEST DELETION:
    test_handleDeleteOrPurge(bucket, false);
  }

  ////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  // TEST AGE OUT

  @Test
  public void test_ageOut() throws IOException, InterruptedException, ExecutionException {

    // Call test_endToEnd_autoTime to create 5 time based indexes
    // 2015-01-01 -> 2015-05-01
    // How far is now from 2015-05-03
    final Date d = TimeUtils.getDateFromSuffix("2015-03-02").success();
    final long total_time_ms = new Date().getTime() - d.getTime();
    final long total_days = total_time_ms / (1000L * 3600L * 24L);
    final String age_out = ErrorUtils.get("{0} days", total_days);

    final DataBucketBean bucket =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with("full_name", "/test/end-end/auto-time")
            .with(
                DataBucketBean::data_schema,
                BeanTemplateUtils.build(DataSchemaBean.class)
                    .with(
                        DataSchemaBean::temporal_schema,
                        BeanTemplateUtils.build(TemporalSchemaBean.class)
                            .with(TemporalSchemaBean::exist_age_max, age_out)
                            .done()
                            .get())
                    .done()
                    .get())
            .done()
            .get();

    final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket);

    test_endToEnd_autoTime(false);

    _index_service
        ._crud_factory
        .getClient()
        .admin()
        .indices()
        .prepareCreate(template_name + "_2015-03-01_1")
        .execute()
        .actionGet();

    final GetMappingsResponse gmr =
        _index_service
            ._crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareGetMappings(template_name + "*")
            .execute()
            .actionGet();
    assertEquals(6, gmr.getMappings().keys().size());

    CompletableFuture<BasicMessageBean> cf =
        _index_service.getDataService().get().handleAgeOutRequest(bucket);

    BasicMessageBean res = cf.get();

    assertEquals(true, res.success());
    assertTrue("sensible message: " + res.message(), res.message().contains(" 2 "));

    assertTrue(
        "Message marked as loggable: " + res.details(),
        Optional.ofNullable(res.details()).filter(m -> m.containsKey("loggable")).isPresent());

    System.out.println("Return from to delete: " + res.message());

    Thread.sleep(5000L); // give the indexes time to delete

    final GetMappingsResponse gmr2 =
        _index_service
            ._crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareGetMappings(template_name + "*")
            .execute()
            .actionGet();
    assertEquals(3, gmr2.getMappings().keys().size());

    // Check some edge cases:

    // 1) Run it again, returns success but not loggable:

    CompletableFuture<BasicMessageBean> cf2 =
        _index_service.getDataService().get().handleAgeOutRequest(bucket);

    BasicMessageBean res2 = cf2.get();

    assertEquals(true, res2.success());
    assertTrue("sensible message: " + res2.message(), res2.message().contains(" 0 "));
    assertTrue(
        "Message _not_ marked as loggable: " + res2.details(),
        !Optional.ofNullable(res2.details()).map(m -> m.get("loggable")).isPresent());

    // 2) No temporal settings

    final DataBucketBean bucket3 =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with("full_name", "/test/handle/age/out/delete/not/temporal")
            .with(
                DataBucketBean::data_schema,
                BeanTemplateUtils.build(DataSchemaBean.class).done().get())
            .done()
            .get();

    CompletableFuture<BasicMessageBean> cf3 =
        _index_service.getDataService().get().handleAgeOutRequest(bucket3);
    BasicMessageBean res3 = cf3.get();
    // no temporal settings => returns success
    assertEquals(true, res3.success());

    // 3) Unparseable temporal settings (in theory won't validate but we can test here)

    final DataBucketBean bucket4 =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with("full_name", "/test/handle/age/out/delete/temporal/malformed")
            .with(
                DataBucketBean::data_schema,
                BeanTemplateUtils.build(DataSchemaBean.class)
                    .with(
                        DataSchemaBean::temporal_schema,
                        BeanTemplateUtils.build(TemporalSchemaBean.class)
                            .with(TemporalSchemaBean::exist_age_max, "bananas")
                            .done()
                            .get())
                    .done()
                    .get())
            .done()
            .get();

    CompletableFuture<BasicMessageBean> cf4 =
        _index_service.getDataService().get().handleAgeOutRequest(bucket4);
    BasicMessageBean res4 = cf4.get();
    // no temporal settings => returns success
    assertEquals(false, res4.success());
  }

  ////////////////////////////////////////////////////////////////////////////////////////////////////////////////

  // TEST DELETION

  // (these are called from the code above)

  public void test_handleDeleteOrPurge(final DataBucketBean to_handle, boolean delete_not_purge)
      throws InterruptedException, ExecutionException {
    System.out.println("****** Checking delete/purge");

    final String template_name = ElasticsearchIndexUtils.getBaseIndexName(to_handle);
    final ICrudService<JsonNode> index_service_crud =
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, to_handle, Optional.empty(), Optional.empty()))
            .flatMap(IDataWriteService::getCrudService)
            .get();

    final ElasticsearchContext es_context =
        (ElasticsearchContext)
            index_service_crud
                .getUnderlyingPlatformDriver(ElasticsearchContext.class, Optional.empty())
                .get();

    // (Actually first off, check there's data and templates)
    // Data:
    {
      final GetMappingsResponse gmr =
          es_context
              .client()
              .admin()
              .indices()
              .prepareGetMappings(template_name + "*")
              .execute()
              .actionGet();
      assertTrue("There are indexes", gmr.getMappings().keys().size() > 0);
    }
    // Templates:
    {
      final GetIndexTemplatesRequest gt_pre = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr_pre =
          _crud_factory.getClient().admin().indices().getTemplates(gt_pre).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(1, gtr_pre.getIndexTemplates().size());
    }

    // Then, perform request
    final BasicMessageBean result =
        _index_service
            .getDataService()
            .get()
            .handleBucketDeletionRequest(to_handle, Optional.empty(), delete_not_purge)
            .get();
    assertEquals("Deletion should succeed: " + result.message(), true, result.success());

    // Check templates gone iff deleting not purging

    if (delete_not_purge) {
      final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr =
          _crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
      assertTrue("No templates after deletion", gtr.getIndexTemplates().isEmpty());
    } else {
      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(1, gtr2.getIndexTemplates().size());
    }

    // Check all files deleted

    // Check via mappings
    {
      final GetMappingsResponse gmr =
          es_context
              .client()
              .admin()
              .indices()
              .prepareGetMappings(template_name + "*")
              .execute()
              .actionGet();
      assertEquals(0, gmr.getMappings().keys().size());
    }
    // Check via index size (recreates templates)

    final ICrudService<JsonNode> index_service_crud_2 =
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, to_handle, Optional.empty(), Optional.empty()))
            .flatMap(IDataWriteService::getCrudService)
            .get();

    assertEquals(0, index_service_crud_2.countObjects().get().intValue());
  }

  @Test
  public void test_deleteNonexistantBucket()
      throws JsonParseException, JsonMappingException, IOException, InterruptedException,
          ExecutionException {
    final DataBucketBean bucket =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with("_id", "2b_test_end_2_end_not_exist")
            .with("full_name", "/test/end-end/fixed/fixed/not/exist")
            .done()
            .get();

    final BasicMessageBean result =
        _index_service
            .getDataService()
            .get()
            .handleBucketDeletionRequest(bucket, Optional.empty(), true)
            .get();
    assertEquals("Deletion should succeed: " + result.message(), true, result.success());
  }
}
  @Test
  public void test_indexCreation() throws IOException {

    final Calendar time_setter = GregorianCalendar.getInstance();
    time_setter.set(2015, 1, 1, 13, 0, 0);

    final String bucket_str =
        Resources.toString(
            Resources.getResource(
                "com/ikanow/aleph2/search_service/elasticsearch/services/test_bucket_validate_success.json"),
            Charsets.UTF_8);
    final DataBucketBean bucket =
        BeanTemplateUtils.build(bucket_str, DataBucketBean.class)
            .with("modified", time_setter.getTime())
            .done()
            .get();

    final String mapping_str =
        Resources.toString(
            Resources.getResource(
                "com/ikanow/aleph2/search_service/elasticsearch/services/test_verbose_mapping_validate_results.json"),
            Charsets.UTF_8);
    final JsonNode mapping_json = _mapper.readTree(mapping_str.getBytes());

    final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket);

    try {
      _crud_factory
          .getClient()
          .admin()
          .indices()
          .prepareDeleteTemplate(template_name)
          .execute()
          .actionGet();
    } catch (Exception e) {
    } // (This is fine, just means it doesn't exist)

    // Create index template from empty

    {
      final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr =
          _crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
      assertTrue("No templates to start with", gtr.getIndexTemplates().isEmpty());

      _index_service.handlePotentiallyNewIndex(
          bucket,
          Optional.empty(),
          ElasticsearchIndexConfigUtils.buildConfigBeanFromSchema(bucket, _config_bean, _mapper),
          "_default_");

      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(1, gtr2.getIndexTemplates().size());

      assertTrue(
          "Mappings should be equivalent",
          ElasticsearchIndexService.mappingsAreEquivalent(
              gtr2.getIndexTemplates().get(0), mapping_json, _mapper));
    }

    // Check is ignored subsequently (same date, same content; same date, different content)
    {
      _index_service.handlePotentiallyNewIndex(
          bucket,
          Optional.empty(),
          ElasticsearchIndexConfigUtils.buildConfigBeanFromSchema(bucket, _config_bean, _mapper),
          "_default_");

      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(1, gtr2.getIndexTemplates().size());
    }

    // Check is checked-but-left if time updated, content not
    {
      time_setter.set(2015, 1, 1, 14, 0, 0);
      final Date next_time = time_setter.getTime();
      final DataBucketBean bucket2 =
          BeanTemplateUtils.clone(bucket).with("modified", next_time).done();

      _index_service.handlePotentiallyNewIndex(
          bucket2,
          Optional.empty(),
          ElasticsearchIndexConfigUtils.buildConfigBeanFromSchema(bucket2, _config_bean, _mapper),
          "_default_");

      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(next_time, _index_service._bucket_template_cache.get(bucket._id()));
      assertEquals(1, gtr2.getIndexTemplates().size());
    }

    // Check is updated if time-and-content is different
    {
      time_setter.set(2015, 1, 1, 15, 0, 0);
      final String bucket_str2 =
          Resources.toString(
              Resources.getResource(
                  "com/ikanow/aleph2/search_service/elasticsearch/services/test_bucket2_validate_success.json"),
              Charsets.UTF_8);
      final DataBucketBean bucket2 =
          BeanTemplateUtils.build(bucket_str2, DataBucketBean.class)
              .with("modified", time_setter.getTime())
              .done()
              .get();

      _index_service.handlePotentiallyNewIndex(
          bucket2,
          Optional.empty(),
          ElasticsearchIndexConfigUtils.buildConfigBeanFromSchema(bucket2, _config_bean, _mapper),
          "_default_");

      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(time_setter.getTime(), _index_service._bucket_template_cache.get(bucket._id()));
      assertEquals(1, gtr2.getIndexTemplates().size());

      assertFalse(
          ElasticsearchIndexService.mappingsAreEquivalent(
              gtr2.getIndexTemplates().get(0), mapping_json, _mapper)); // has changed
    }

    // Check if mapping is deleted then next time bucket modified is updated then the mapping is
    // recreated

    {
      _crud_factory
          .getClient()
          .admin()
          .indices()
          .prepareDeleteTemplate(template_name)
          .execute()
          .actionGet();

      // (check with old date)

      final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr =
          _crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
      assertTrue("No templates to start with", gtr.getIndexTemplates().isEmpty());

      {
        _index_service.handlePotentiallyNewIndex(
            bucket,
            Optional.empty(),
            ElasticsearchIndexConfigUtils.buildConfigBeanFromSchema(bucket, _config_bean, _mapper),
            "_default_");

        final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
        final GetIndexTemplatesResponse gtr2 =
            _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
        assertTrue("Initially no change", gtr2.getIndexTemplates().isEmpty());
      }

      // Update date and retry

      {
        time_setter.set(2015, 1, 1, 16, 0, 0);
        final Date next_time = time_setter.getTime();
        final DataBucketBean bucket2 =
            BeanTemplateUtils.clone(bucket).with("modified", next_time).done();

        _index_service.handlePotentiallyNewIndex(
            bucket2,
            Optional.empty(),
            ElasticsearchIndexConfigUtils.buildConfigBeanFromSchema(bucket2, _config_bean, _mapper),
            "_default_");

        final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
        final GetIndexTemplatesResponse gtr2 =
            _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
        assertEquals(1, _index_service._bucket_template_cache.size());
        assertEquals(1, gtr2.getIndexTemplates().size());

        assertTrue(
            "Mappings should be equivalent",
            ElasticsearchIndexService.mappingsAreEquivalent(
                gtr2.getIndexTemplates().get(0), mapping_json, _mapper));
      }
    }
  }
示例#15
0
  /**
   * Given a bucket ...returns either - a future containing the first error encountered, _or_ a map
   * (both name and id as keys) of path names (and guarantee that the file has been cached when the
   * future completes)
   *
   * @param bucket
   * @param management_db
   * @param globals
   * @param fs
   * @param handler_for_errors
   * @param msg_for_errors
   * @return a future containing the first error encountered, _or_ a map (both name and id as keys)
   *     of path names
   */
  @SuppressWarnings("unchecked")
  protected static <M>
      CompletableFuture<
              Validation<BasicMessageBean, Map<String, Tuple2<SharedLibraryBean, String>>>>
          cacheJars(
              final DataBucketBean bucket,
              final IManagementDbService management_db,
              final GlobalPropertiesBean globals,
              final IStorageService fs,
              final IServiceContext context,
              final String handler_for_errors,
              final M msg_for_errors) {
    try {
      MethodNamingHelper<SharedLibraryBean> helper =
          BeanTemplateUtils.from(SharedLibraryBean.class);
      final Optional<QueryComponent<SharedLibraryBean>> spec = getQuery(bucket);
      if (!spec.isPresent()) {
        return CompletableFuture.completedFuture(
            Validation.<BasicMessageBean, Map<String, Tuple2<SharedLibraryBean, String>>>success(
                Collections.emptyMap()));
      }

      return management_db
          .getSharedLibraryStore()
          .secured(context, new AuthorizationBean(bucket.owner_id()))
          .getObjectsBySpec(spec.get())
          .thenComposeAsync(
              cursor -> {
                // This is a map of futures from the cache call - either an error or the path name
                // note we use a tuple of (id, name) as the key and then flatten out later
                final Map<
                        Tuple2<String, String>,
                        Tuple2<
                            SharedLibraryBean,
                            CompletableFuture<Validation<BasicMessageBean, String>>>>
                    map_of_futures =
                        StreamSupport.stream(cursor.spliterator(), true)
                            .filter(
                                lib -> {
                                  return true;
                                })
                            .collect(
                                Collectors
                                    .<SharedLibraryBean, Tuple2<String, String>,
                                        Tuple2<
                                            SharedLibraryBean,
                                            CompletableFuture<
                                                Validation<BasicMessageBean, String>>>>
                                        toMap(
                                            // want to keep both the name and id versions - will
                                            // flatten out below
                                            lib -> Tuples._2T(lib.path_name(), lib._id()), // (key)
                                            // spin off a future in which the file is being copied -
                                            // save the shared library bean also
                                            lib ->
                                                Tuples._2T(
                                                    lib, // (value)
                                                    JarCacheUtils.getCachedJar(
                                                        globals.local_cached_jar_dir(),
                                                        lib,
                                                        fs,
                                                        handler_for_errors,
                                                        msg_for_errors))));

                // denest from map of futures to future of maps, also handle any errors here:
                // (some sort of "lift" function would be useful here - this are a somewhat
                // inelegant few steps)

                final CompletableFuture<Validation<BasicMessageBean, String>>[] futures =
                    (CompletableFuture<Validation<BasicMessageBean, String>>[])
                        map_of_futures
                            .values()
                            .stream()
                            .map(t2 -> t2._2())
                            .collect(Collectors.toList())
                            .toArray(new CompletableFuture[0]);

                // (have to embed this thenApply instead of bringing it outside as part of the
                // toCompose chain, because otherwise we'd lose map_of_futures scope)
                return CompletableFuture.allOf(futures)
                    .<Validation<BasicMessageBean, Map<String, Tuple2<SharedLibraryBean, String>>>>
                        thenApply(
                            f -> {
                              try {
                                final Map<String, Tuple2<SharedLibraryBean, String>> almost_there =
                                    map_of_futures
                                        .entrySet()
                                        .stream()
                                        .flatMap(
                                            kv -> {
                                              final Validation<BasicMessageBean, String> ret =
                                                  kv.getValue()
                                                      ._2()
                                                      .join(); // (must have already returned if
                                              // here
                                              return ret
                                                  .<Stream<
                                                          Tuple2<
                                                              String,
                                                              Tuple2<SharedLibraryBean, String>>>>
                                                      validation(
                                                          // Error:
                                                          err -> {
                                                            throw new RuntimeException(
                                                                err.message());
                                                          } // (not ideal, but will do)
                                                          ,
                                                          // Normal:
                                                          s -> {
                                                            return Arrays.asList(
                                                                    Tuples._2T(
                                                                        kv.getKey()._1(),
                                                                        Tuples._2T(
                                                                            kv.getValue()._1(),
                                                                            s)), // result object
                                                                    // with path_name
                                                                    Tuples._2T(
                                                                        kv.getKey()._2(),
                                                                        Tuples._2T(
                                                                            kv.getValue()._1(),
                                                                            s))) // result object
                                                                // with id
                                                                .stream();
                                                          });
                                            })
                                        .collect(
                                            Collectors
                                                .<Tuple2<String, Tuple2<SharedLibraryBean, String>>,
                                                    String, Tuple2<SharedLibraryBean, String>>
                                                    toMap(
                                                        idname_path -> idname_path._1(), // (key)
                                                        idname_path -> idname_path._2() // (value)
                                                        ));
                                return Validation
                                    .<BasicMessageBean,
                                        Map<String, Tuple2<SharedLibraryBean, String>>>
                                        success(almost_there);
                              } catch (
                                  Exception e) { // handle the exception thrown above containing the
                                // message bean from whatever the original error was!
                                return Validation
                                    .<BasicMessageBean,
                                        Map<String, Tuple2<SharedLibraryBean, String>>>
                                        fail(
                                            SharedErrorUtils.buildErrorMessage(
                                                handler_for_errors.toString(),
                                                msg_for_errors,
                                                e.getMessage()));
                              }
                            });
              });
    } catch (Throwable e) { // (can only occur if the DB call errors)
      return CompletableFuture.completedFuture(
          Validation.fail(
              SharedErrorUtils.buildErrorMessage(
                  handler_for_errors.toString(),
                  msg_for_errors,
                  ErrorUtils.getLongForm(
                      SharedErrorUtils.ERROR_CACHING_SHARED_LIBS, e, bucket.full_name()))));
    }
  }
  @Test
  public void test_validationFail() throws IOException {

    final String bucket_str =
        Resources.toString(
            Resources.getResource(
                "com/ikanow/aleph2/search_service/elasticsearch/services/test_bucket_validate_fail.json"),
            Charsets.UTF_8);
    final DataBucketBean bucket =
        BeanTemplateUtils.build(bucket_str, DataBucketBean.class).done().get();

    // 1) Verbose mode off
    {
      final Collection<BasicMessageBean> res_col =
          _index_service.validateSchema(bucket.data_schema().columnar_schema(), bucket)._2();
      final Collection<BasicMessageBean> res_search =
          _index_service.validateSchema(bucket.data_schema().search_index_schema(), bucket)._2();
      final Collection<BasicMessageBean> res_time =
          _index_service.validateSchema(bucket.data_schema().temporal_schema(), bucket)._2();

      assertEquals(0, res_col.size());
      assertEquals(0, res_time.size());
      assertEquals(1, res_search.size());

      final BasicMessageBean res_search_message = res_search.iterator().next();
      assertEquals(false, res_search_message.success());
    }

    // 2) Check setting an invalid max index size
    {
      final String bucket_str_2 =
          Resources.toString(
              Resources.getResource(
                  "com/ikanow/aleph2/search_service/elasticsearch/services/test_bucket_validate_success.json"),
              Charsets.UTF_8);
      final DataBucketBean bucket2 =
          BeanTemplateUtils.build(bucket_str_2, DataBucketBean.class).done().get();
      final DataBucketBean bucket_too_small =
          BeanTemplateUtils.clone(bucket2)
              .with(
                  DataBucketBean::data_schema,
                  BeanTemplateUtils.clone(bucket2.data_schema())
                      .with(
                          DataSchemaBean::search_index_schema,
                          BeanTemplateUtils.clone(bucket2.data_schema().search_index_schema())
                              .with(DataSchemaBean.SearchIndexSchemaBean::target_index_size_mb, 10L)
                              .done())
                      .done())
              .done();
      final Collection<BasicMessageBean> res_search =
          _index_service
              .validateSchema(bucket.data_schema().search_index_schema(), bucket_too_small)
              ._2();
      assertEquals(1, res_search.size());
      assertEquals(false, res_search.stream().allMatch(BasicMessageBean::success));
      BasicMessageBean res_search_message = res_search.iterator().next();
      assertTrue(
          "Right message: " + res_search_message.message(),
          res_search_message.message().contains("10 MB"));
    }
  }
  @Test
  public void test_validationSuccess() throws IOException {
    final String bucket_str =
        Resources.toString(
            Resources.getResource(
                "com/ikanow/aleph2/search_service/elasticsearch/services/test_bucket_validate_success.json"),
            Charsets.UTF_8);
    final DataBucketBean bucket =
        BeanTemplateUtils.build(bucket_str, DataBucketBean.class).done().get();

    // 1) Verbose mode off
    {
      final Collection<BasicMessageBean> res_col =
          _index_service.validateSchema(bucket.data_schema().columnar_schema(), bucket)._2();
      final Collection<BasicMessageBean> res_search =
          _index_service.validateSchema(bucket.data_schema().search_index_schema(), bucket)._2();
      final Collection<BasicMessageBean> res_time =
          _index_service.validateSchema(bucket.data_schema().temporal_schema(), bucket)._2();

      assertEquals(0, res_col.size());
      assertEquals(0, res_search.size());
      assertEquals(0, res_time.size());
    }

    // 2) Verbose mode on
    {
      final DataBucketBean bucket_verbose =
          BeanTemplateUtils.clone(bucket)
              .with(
                  DataBucketBean::data_schema,
                  BeanTemplateUtils.clone(bucket.data_schema())
                      .with(
                          DataSchemaBean::search_index_schema,
                          BeanTemplateUtils.clone(bucket.data_schema().search_index_schema())
                              .with(
                                  DataSchemaBean.SearchIndexSchemaBean::technology_override_schema,
                                  ImmutableMap.builder()
                                      .putAll(
                                          bucket
                                              .data_schema()
                                              .search_index_schema()
                                              .technology_override_schema())
                                      .put("verbose", true)
                                      .build())
                              .done())
                      .done())
              .done();

      final Collection<BasicMessageBean> res_col =
          _index_service
              .validateSchema(bucket_verbose.data_schema().columnar_schema(), bucket)
              ._2();
      final Collection<BasicMessageBean> res_search =
          _index_service
              .validateSchema(bucket_verbose.data_schema().search_index_schema(), bucket)
              ._2();
      final Collection<BasicMessageBean> res_time =
          _index_service
              .validateSchema(bucket_verbose.data_schema().temporal_schema(), bucket)
              ._2();

      assertEquals(0, res_col.size());
      assertEquals(0, res_time.size());
      assertEquals(2, res_search.size());
      assertEquals(true, res_search.stream().allMatch(BasicMessageBean::success));
      Iterator<BasicMessageBean> res_search_message = res_search.iterator();

      final String mapping_str =
          Resources.toString(
              Resources.getResource(
                  "com/ikanow/aleph2/search_service/elasticsearch/services/test_verbose_mapping_validate_results.json"),
              Charsets.UTF_8);
      final JsonNode mapping_json = _mapper.readTree(mapping_str.getBytes());
      assertEquals(
          mapping_json.toString(),
          _mapper.readTree(res_search_message.next().message()).toString());
      assertTrue(
          "Sets the max index override: "
              + res_search.stream().skip(1).map(m -> m.message()).collect(Collectors.joining()),
          res_search_message.next().message().contains("1,000 MB"));
    }

    // 3) Temporal

    {
      final DataBucketBean bucket_temporal_no_grouping =
          BeanTemplateUtils.clone(bucket)
              .with(
                  DataBucketBean::data_schema,
                  BeanTemplateUtils.clone(bucket.data_schema())
                      .with(
                          DataSchemaBean::temporal_schema,
                          BeanTemplateUtils.build(DataSchemaBean.TemporalSchemaBean.class)
                              .done()
                              .get())
                      .done())
              .done();

      assertEquals(
          "",
          _index_service
              .validateSchema(bucket_temporal_no_grouping.data_schema().temporal_schema(), bucket)
              ._1());

      final DataBucketBean bucket_temporal_grouping =
          BeanTemplateUtils.clone(bucket)
              .with(
                  DataBucketBean::data_schema,
                  BeanTemplateUtils.clone(bucket.data_schema())
                      .with(
                          DataSchemaBean::temporal_schema,
                          BeanTemplateUtils.build(DataSchemaBean.TemporalSchemaBean.class)
                              .with(DataSchemaBean.TemporalSchemaBean::grouping_time_period, "1d")
                              .done()
                              .get())
                      .done())
              .done();

      assertEquals(
          "_{yyyy-MM-dd}",
          _index_service
              .validateSchema(bucket_temporal_grouping.data_schema().temporal_schema(), bucket)
              ._1());
    }
  }
示例#18
0
  protected static CompletableFuture<BucketActionReplyMessage> talkToStream(
      final IStormController storm_controller,
      final DataBucketBean bucket,
      final BucketActionMessage m,
      final Validation<BasicMessageBean, IEnrichmentStreamingTopology> err_or_user_topology,
      final Validation<BasicMessageBean, Map<String, Tuple2<SharedLibraryBean, String>>> err_or_map,
      final String source,
      final StreamingEnrichmentContext context,
      final String yarn_config_dir,
      final String cached_jars_dir) {
    try {
      // handle getting the user libs
      final List<String> user_lib_paths =
          err_or_map.<List<String>>validation(
              fail -> Collections.emptyList() // (going to die soon anyway)
              ,
              success ->
                  success
                      .values()
                      .stream()
                      .map(tuple -> tuple._2.replaceFirst("file:", ""))
                      .collect(Collectors.toList()));

      return err_or_user_topology.<CompletableFuture<BucketActionReplyMessage>>validation(
          // ERROR getting enrichment topology
          error -> {
            return CompletableFuture.completedFuture(new BucketActionHandlerMessage(source, error));
          },
          // NORMAL grab enrichment topology
          enrichment_topology -> {
            final String entry_point = enrichment_topology.getClass().getName();
            context.setBucket(bucket);
            context.setUserTopologyEntryPoint(entry_point);
            // also set the library bean - note if here then must have been set, else
            // IHarvestTechnologyModule wouldn't exist
            err_or_map.forEach(
                map -> {
                  context.setLibraryConfig(
                      map.values()
                          .stream()
                          .map(t2 -> t2._1())
                          .filter(
                              lib ->
                                  entry_point.equals(lib.misc_entry_point())
                                      || entry_point.equals(lib.streaming_enrichment_entry_point()))
                          .findFirst()
                          .orElse(BeanTemplateUtils.build(SharedLibraryBean.class).done().get()));
                  // (else this is a passthrough topology, so just use a dummy library bean)
                });

            _logger.info(
                "Set active class="
                    + enrichment_topology.getClass()
                    + " message="
                    + m.getClass().getSimpleName()
                    + " bucket="
                    + bucket.full_name());

            return Patterns.match(m)
                .<CompletableFuture<BucketActionReplyMessage>>andReturn()
                .when(
                    BucketActionMessage.DeleteBucketActionMessage.class,
                    msg -> {
                      return StormControllerUtil.stopJob(storm_controller, bucket);
                    })
                .when(
                    BucketActionMessage.NewBucketActionMessage.class,
                    msg -> {
                      if (!msg.is_suspended())
                        return StormControllerUtil.startJob(
                            storm_controller,
                            bucket,
                            context,
                            user_lib_paths,
                            enrichment_topology,
                            cached_jars_dir);
                      else
                        return StormControllerUtil.stopJob(
                            storm_controller,
                            bucket); // (nothing to do but just do this to return something
                      // sensible)
                    })
                .when(
                    BucketActionMessage.UpdateBucketActionMessage.class,
                    msg -> {
                      if (msg.is_enabled())
                        return StormControllerUtil.restartJob(
                            storm_controller,
                            bucket,
                            context,
                            user_lib_paths,
                            enrichment_topology,
                            cached_jars_dir);
                      else return StormControllerUtil.stopJob(storm_controller, bucket);
                    })
                .when(
                    BucketActionMessage.TestBucketActionMessage.class,
                    msg -> {
                      // TODO (ALEPH-25): in the future run this test with local storm rather than
                      // remote storm_controller
                      return StormControllerUtil.restartJob(
                          storm_controller,
                          bucket,
                          context,
                          user_lib_paths,
                          enrichment_topology,
                          cached_jars_dir);
                    })
                .otherwise(
                    msg -> {
                      return CompletableFuture.completedFuture(
                          new BucketActionHandlerMessage(
                              source,
                              new BasicMessageBean(
                                  new Date(),
                                  false,
                                  null,
                                  "Unknown message",
                                  0,
                                  "Unknown message",
                                  null)));
                    });
          });
    } catch (Throwable e) { // (trying to use Validation to avoid this, but just in case...)
      return CompletableFuture.completedFuture(
          new BucketActionHandlerMessage(
              source,
              new BasicMessageBean(
                  new Date(),
                  false,
                  null,
                  ErrorUtils.getLongForm("Error loading streaming class: {0}", e),
                  0,
                  ErrorUtils.getLongForm("Error loading streaming class: {0}", e),
                  null)));
    }
  }
  @Test
  public void test_cacheJars()
      throws UnsupportedFileSystemException, InterruptedException, ExecutionException {
    try {
      // Preamble:
      // 0) Insert 2 library beans into the management db

      final DataBucketBean bucket = createBucket("test_tech_id_stream");

      final String pathname1 =
          System.getProperty("user.dir") + "/misc_test_assets/simple-harvest-example.jar";
      final Path path1 = FileContext.getLocalFSFileContext().makeQualified(new Path(pathname1));
      final String pathname2 =
          System.getProperty("user.dir") + "/misc_test_assets/simple-harvest-example2.jar";
      final Path path2 = FileContext.getLocalFSFileContext().makeQualified(new Path(pathname2));

      List<SharedLibraryBean> lib_elements = createSharedLibraryBeans(path1, path2);

      final IManagementDbService underlying_db =
          _service_context.getService(IManagementDbService.class, Optional.empty()).get();
      final IManagementCrudService<SharedLibraryBean> library_crud =
          underlying_db.getSharedLibraryStore();
      library_crud.deleteDatastore();
      assertEquals("Cleansed library store", 0L, (long) library_crud.countObjects().get());
      library_crud.storeObjects(lib_elements).get();

      assertEquals("Should have 4 library beans", 4L, (long) library_crud.countObjects().get());

      // 0a) Check with no streaming, gets nothing
      {
        CompletableFuture<
                Validation<BasicMessageBean, Map<String, Tuple2<SharedLibraryBean, String>>>>
            reply_structure =
                DataBucketChangeActor.cacheJars(
                    bucket,
                    _service_context.getCoreManagementDbService(),
                    _service_context.getGlobalProperties(),
                    _service_context.getStorageService(),
                    _service_context,
                    "test1_source",
                    "test1_command");

        if (reply_structure.get().isFail()) {
          fail("About to crash with: " + reply_structure.get().fail().message());
        }
        assertTrue("cacheJars should return valid reply", reply_structure.get().isSuccess());

        final Map<String, Tuple2<SharedLibraryBean, String>> reply_map =
            reply_structure.get().success();

        assertEquals(0L, reply_map.size()); // (both modules, 1x for _id and 1x for name)
      }

      // 0b) Create the more complex bucket

      final EnrichmentControlMetadataBean enrichment_module =
          new EnrichmentControlMetadataBean(
              "test_name",
              Collections.emptyList(),
              true,
              null,
              Arrays.asList("test_tech_id_stream", "test_module_id"),
              null,
              new LinkedHashMap<>());

      final DataBucketBean bucket2 =
          BeanTemplateUtils.clone(bucket)
              .with(DataBucketBean::streaming_enrichment_topology, enrichment_module)
              .done();

      // 1) Normal operation

      CompletableFuture<
              Validation<BasicMessageBean, Map<String, Tuple2<SharedLibraryBean, String>>>>
          reply_structure =
              DataBucketChangeActor.cacheJars(
                  bucket2,
                  _service_context.getCoreManagementDbService(),
                  _service_context.getGlobalProperties(),
                  _service_context.getStorageService(),
                  _service_context,
                  "test1_source",
                  "test1_command");

      if (reply_structure.get().isFail()) {
        fail("About to crash with: " + reply_structure.get().fail().message());
      }
      assertTrue("cacheJars should return valid reply", reply_structure.get().isSuccess());

      final Map<String, Tuple2<SharedLibraryBean, String>> reply_map =
          reply_structure.get().success();

      assertEquals(
          "Should have 4 beans: " + reply_map.toString(),
          4L,
          reply_map.size()); // (both modules, 1x for _id and 1x for name)

      // 3) Couple of error cases:

      final EnrichmentControlMetadataBean enrichment_module2 =
          new EnrichmentControlMetadataBean(
              "test_name",
              Collections.emptyList(),
              true,
              null,
              Arrays.asList("test_tech_id_stream", "test_module_id", "failtest"),
              null,
              new LinkedHashMap<>());

      final DataBucketBean bucket3 =
          BeanTemplateUtils.clone(bucket)
              .with(DataBucketBean::streaming_enrichment_topology, enrichment_module2)
              .done();

      CompletableFuture<
              Validation<BasicMessageBean, Map<String, Tuple2<SharedLibraryBean, String>>>>
          reply_structure3 =
              DataBucketChangeActor.cacheJars(
                  bucket3,
                  _service_context.getCoreManagementDbService(),
                  _service_context.getGlobalProperties(),
                  _service_context.getStorageService(),
                  _service_context,
                  "test2_source",
                  "test2_command");

      assertTrue("cacheJars should return error", reply_structure3.get().isFail());
    } catch (Exception e) {
      System.out.println(ErrorUtils.getLongForm("guice? {0}", e));
      throw e;
    }
  }
  @Test
  public void test_getStreamingTopology()
      throws UnsupportedFileSystemException, InterruptedException, ExecutionException {
    final DataBucketBean bucket = createBucket("test_tech_id_stream");

    final String pathname1 =
        System.getProperty("user.dir") + "/misc_test_assets/simple-topology-example.jar";
    final Path path1 = FileContext.getLocalFSFileContext().makeQualified(new Path(pathname1));
    final String pathname2 =
        System.getProperty("user.dir") + "/misc_test_assets/simple-harvest-example2.jar";
    final Path path2 = FileContext.getLocalFSFileContext().makeQualified(new Path(pathname2));

    List<SharedLibraryBean> lib_elements = createSharedLibraryBeans(path1, path2);

    //////////////////////////////////////////////////////

    // 1) Check - if called with an error, then just passes that error along

    final BasicMessageBean error =
        SharedErrorUtils.buildErrorMessage("test_source", "test_message", "test_error");

    final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test1 =
        DataBucketChangeActor.getStreamingTopology(
            bucket,
            new BucketActionMessage.BucketActionOfferMessage(bucket),
            "test_source2",
            Validation.fail(error));

    assertTrue("Got error back", test1.isFail());
    assertEquals("test_source", test1.fail().source());
    assertEquals("test_message", test1.fail().command());
    assertEquals("test_error", test1.fail().message());

    //////////////////////////////////////////////////////

    // 2) Check the error handling inside getStreamingTopology

    final ImmutableMap<String, Tuple2<SharedLibraryBean, String>> test2_input =
        ImmutableMap.<String, Tuple2<SharedLibraryBean, String>>builder()
            .put("test_tech_id_stream_2b", Tuples._2T(null, null))
            .build();

    final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test2a =
        DataBucketChangeActor.getStreamingTopology(
            BeanTemplateUtils.clone(bucket)
                .with(
                    DataBucketBean::streaming_enrichment_topology,
                    BeanTemplateUtils.build(EnrichmentControlMetadataBean.class)
                        .with(
                            EnrichmentControlMetadataBean::library_names_or_ids,
                            Arrays.asList("test_tech_id_stream_2a"))
                        .done()
                        .get())
                .done(),
            new BucketActionMessage.BucketActionOfferMessage(bucket),
            "test_source2a",
            Validation.success(test2_input));

    assertTrue("Got error back", test2a.isFail());
    assertEquals("test_source2a", test2a.fail().source());
    assertEquals("BucketActionOfferMessage", test2a.fail().command());
    assertEquals(
        ErrorUtils.get(
            SharedErrorUtils.SHARED_LIBRARY_NAME_NOT_FOUND,
            bucket.full_name(),
            "(unknown)"), // (cloned bucket above)
        test2a.fail().message());

    final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test2b =
        DataBucketChangeActor.getStreamingTopology(
            BeanTemplateUtils.clone(bucket)
                .with(
                    DataBucketBean::streaming_enrichment_topology,
                    BeanTemplateUtils.build(EnrichmentControlMetadataBean.class)
                        .with(
                            EnrichmentControlMetadataBean::library_names_or_ids,
                            Arrays.asList("test_tech_id_stream_2b"))
                        .done()
                        .get())
                .done(),
            new BucketActionMessage.BucketActionOfferMessage(bucket),
            "test_source2b",
            Validation.success(test2_input));

    assertTrue("Got error back", test2b.isFail());
    assertEquals("test_source2b", test2b.fail().source());
    assertEquals("BucketActionOfferMessage", test2b.fail().command());
    assertEquals(
        ErrorUtils.get(
            SharedErrorUtils.SHARED_LIBRARY_NAME_NOT_FOUND,
            bucket.full_name(),
            "(unknown)"), // (cloned bucket above)
        test2a.fail().message());

    //////////////////////////////////////////////////////

    // 3) OK now it will actually do something

    final String java_name =
        _service_context.getGlobalProperties().local_cached_jar_dir()
            + File.separator
            + "test_tech_id_stream.cache.jar";

    _logger.info(
        "Needed to delete locally cached file? " + java_name + ": " + new File(java_name).delete());

    // Requires that the file has already been cached:
    final Validation<BasicMessageBean, String> cached_file =
        JarCacheUtils.getCachedJar(
                _service_context.getGlobalProperties().local_cached_jar_dir(),
                lib_elements.get(0),
                _service_context.getStorageService(),
                "test3",
                "test3")
            .get();

    if (cached_file.isFail()) {
      fail("About to crash with: " + cached_file.fail().message());
    }

    assertTrue("The cached file exists: " + java_name, new File(java_name).exists());

    // OK the setup is done and validated now actually test the underlying call:

    final ImmutableMap<String, Tuple2<SharedLibraryBean, String>> test3_input =
        ImmutableMap.<String, Tuple2<SharedLibraryBean, String>>builder()
            .put("test_tech_id_stream", Tuples._2T(lib_elements.get(0), cached_file.success()))
            .build();

    final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test3 =
        DataBucketChangeActor.getStreamingTopology(
            BeanTemplateUtils.clone(bucket)
                .with(
                    DataBucketBean::streaming_enrichment_topology,
                    BeanTemplateUtils.build(EnrichmentControlMetadataBean.class)
                        .with(
                            EnrichmentControlMetadataBean::library_names_or_ids,
                            Arrays.asList("test_tech_id_stream"))
                        .done()
                        .get())
                .done(),
            new BucketActionMessage.BucketActionOfferMessage(bucket),
            "test_source3",
            Validation.success(test3_input));

    if (test3.isFail()) {
      fail("About to crash with: " + test3.fail().message());
    }
    assertTrue("getStreamingTopology call succeeded", test3.isSuccess());
    assertTrue("topology created: ", test3.success() != null);
    assertEquals(lib_elements.get(0).misc_entry_point(), test3.success().getClass().getName());

    // (Try again but with failing version, due to class not found)

    final ImmutableMap<String, Tuple2<SharedLibraryBean, String>> test3a_input =
        ImmutableMap.<String, Tuple2<SharedLibraryBean, String>>builder()
            .put("test_tech_id_stream_fail", Tuples._2T(lib_elements.get(3), cached_file.success()))
            .build();

    final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test3a =
        DataBucketChangeActor.getStreamingTopology(
            BeanTemplateUtils.clone(bucket)
                .with(
                    DataBucketBean::streaming_enrichment_topology,
                    BeanTemplateUtils.build(EnrichmentControlMetadataBean.class)
                        .with(
                            EnrichmentControlMetadataBean::library_names_or_ids,
                            Arrays.asList("test_tech_id_stream_fail"))
                        .done()
                        .get())
                .done(),
            new BucketActionMessage.BucketActionOfferMessage(bucket),
            "test_source3",
            Validation.success(test3a_input));

    assertTrue("Got error back", test3a.isFail());
    assertTrue(
        "Right error: " + test3a.fail().message(),
        test3a.fail().message().contains("com.ikanow.aleph2.test.example.ExampleStreamTopology"));

    // Now check with the "not just the harvest tech" flag set

    final String java_name2 =
        _service_context.getGlobalProperties().local_cached_jar_dir()
            + File.separator
            + "test_module_id.cache.jar";

    _logger.info(
        "Needed to delete locally cached file? "
            + java_name2
            + ": "
            + new File(java_name2).delete());

    // Requires that the file has already been cached:
    final Validation<BasicMessageBean, String> cached_file2 =
        JarCacheUtils.getCachedJar(
                _service_context.getGlobalProperties().local_cached_jar_dir(),
                lib_elements.get(1),
                _service_context.getStorageService(),
                "test3b",
                "test3b")
            .get();

    if (cached_file2.isFail()) {
      fail("About to crash with: " + cached_file2.fail().message());
    }

    assertTrue("The cached file exists: " + java_name, new File(java_name2).exists());

    final ImmutableMap<String, Tuple2<SharedLibraryBean, String>> test3b_input =
        ImmutableMap.<String, Tuple2<SharedLibraryBean, String>>builder()
            .put("test_tech_id_stream", Tuples._2T(lib_elements.get(0), cached_file.success()))
            .put("test_module_id", Tuples._2T(lib_elements.get(1), cached_file.success()))
            .build();

    final EnrichmentControlMetadataBean enrichment_module =
        new EnrichmentControlMetadataBean(
            "test_tech_name",
            Collections.emptyList(),
            true,
            null,
            Arrays.asList("test_tech_id_stream", "test_module_id"),
            null,
            null);

    final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test3b =
        DataBucketChangeActor.getStreamingTopology(
            BeanTemplateUtils.clone(bucket)
                .with(DataBucketBean::streaming_enrichment_topology, enrichment_module)
                .done(),
            new BucketActionMessage.BucketActionOfferMessage(bucket),
            "test_source3b",
            Validation.success(test3b_input));

    if (test3b.isFail()) {
      fail("About to crash with: " + test3b.fail().message());
    }
    assertTrue("getStreamingTopology call succeeded", test3b.isSuccess());
    assertTrue("topology created: ", test3b.success() != null);
    assertEquals(lib_elements.get(0).misc_entry_point(), test3b.success().getClass().getName());

    // TODO add a test for disabled streaming but config given (should default to passthrough top
    // and
    // ignore given topology
  }