Ejemplo n.º 1
0
  @Test
  public void test_getStreamingTopology()
      throws UnsupportedFileSystemException, InterruptedException, ExecutionException {
    final DataBucketBean bucket = createBucket("test_tech_id_stream");

    final String pathname1 =
        System.getProperty("user.dir") + "/misc_test_assets/simple-topology-example.jar";
    final Path path1 = FileContext.getLocalFSFileContext().makeQualified(new Path(pathname1));
    final String pathname2 =
        System.getProperty("user.dir") + "/misc_test_assets/simple-harvest-example2.jar";
    final Path path2 = FileContext.getLocalFSFileContext().makeQualified(new Path(pathname2));

    List<SharedLibraryBean> lib_elements = createSharedLibraryBeans(path1, path2);

    //////////////////////////////////////////////////////

    // 1) Check - if called with an error, then just passes that error along

    final BasicMessageBean error =
        SharedErrorUtils.buildErrorMessage("test_source", "test_message", "test_error");

    final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test1 =
        DataBucketChangeActor.getStreamingTopology(
            bucket,
            new BucketActionMessage.BucketActionOfferMessage(bucket),
            "test_source2",
            Validation.fail(error));

    assertTrue("Got error back", test1.isFail());
    assertEquals("test_source", test1.fail().source());
    assertEquals("test_message", test1.fail().command());
    assertEquals("test_error", test1.fail().message());

    //////////////////////////////////////////////////////

    // 2) Check the error handling inside getStreamingTopology

    final ImmutableMap<String, Tuple2<SharedLibraryBean, String>> test2_input =
        ImmutableMap.<String, Tuple2<SharedLibraryBean, String>>builder()
            .put("test_tech_id_stream_2b", Tuples._2T(null, null))
            .build();

    final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test2a =
        DataBucketChangeActor.getStreamingTopology(
            BeanTemplateUtils.clone(bucket)
                .with(
                    DataBucketBean::streaming_enrichment_topology,
                    BeanTemplateUtils.build(EnrichmentControlMetadataBean.class)
                        .with(
                            EnrichmentControlMetadataBean::library_names_or_ids,
                            Arrays.asList("test_tech_id_stream_2a"))
                        .done()
                        .get())
                .done(),
            new BucketActionMessage.BucketActionOfferMessage(bucket),
            "test_source2a",
            Validation.success(test2_input));

    assertTrue("Got error back", test2a.isFail());
    assertEquals("test_source2a", test2a.fail().source());
    assertEquals("BucketActionOfferMessage", test2a.fail().command());
    assertEquals(
        ErrorUtils.get(
            SharedErrorUtils.SHARED_LIBRARY_NAME_NOT_FOUND,
            bucket.full_name(),
            "(unknown)"), // (cloned bucket above)
        test2a.fail().message());

    final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test2b =
        DataBucketChangeActor.getStreamingTopology(
            BeanTemplateUtils.clone(bucket)
                .with(
                    DataBucketBean::streaming_enrichment_topology,
                    BeanTemplateUtils.build(EnrichmentControlMetadataBean.class)
                        .with(
                            EnrichmentControlMetadataBean::library_names_or_ids,
                            Arrays.asList("test_tech_id_stream_2b"))
                        .done()
                        .get())
                .done(),
            new BucketActionMessage.BucketActionOfferMessage(bucket),
            "test_source2b",
            Validation.success(test2_input));

    assertTrue("Got error back", test2b.isFail());
    assertEquals("test_source2b", test2b.fail().source());
    assertEquals("BucketActionOfferMessage", test2b.fail().command());
    assertEquals(
        ErrorUtils.get(
            SharedErrorUtils.SHARED_LIBRARY_NAME_NOT_FOUND,
            bucket.full_name(),
            "(unknown)"), // (cloned bucket above)
        test2a.fail().message());

    //////////////////////////////////////////////////////

    // 3) OK now it will actually do something

    final String java_name =
        _service_context.getGlobalProperties().local_cached_jar_dir()
            + File.separator
            + "test_tech_id_stream.cache.jar";

    _logger.info(
        "Needed to delete locally cached file? " + java_name + ": " + new File(java_name).delete());

    // Requires that the file has already been cached:
    final Validation<BasicMessageBean, String> cached_file =
        JarCacheUtils.getCachedJar(
                _service_context.getGlobalProperties().local_cached_jar_dir(),
                lib_elements.get(0),
                _service_context.getStorageService(),
                "test3",
                "test3")
            .get();

    if (cached_file.isFail()) {
      fail("About to crash with: " + cached_file.fail().message());
    }

    assertTrue("The cached file exists: " + java_name, new File(java_name).exists());

    // OK the setup is done and validated now actually test the underlying call:

    final ImmutableMap<String, Tuple2<SharedLibraryBean, String>> test3_input =
        ImmutableMap.<String, Tuple2<SharedLibraryBean, String>>builder()
            .put("test_tech_id_stream", Tuples._2T(lib_elements.get(0), cached_file.success()))
            .build();

    final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test3 =
        DataBucketChangeActor.getStreamingTopology(
            BeanTemplateUtils.clone(bucket)
                .with(
                    DataBucketBean::streaming_enrichment_topology,
                    BeanTemplateUtils.build(EnrichmentControlMetadataBean.class)
                        .with(
                            EnrichmentControlMetadataBean::library_names_or_ids,
                            Arrays.asList("test_tech_id_stream"))
                        .done()
                        .get())
                .done(),
            new BucketActionMessage.BucketActionOfferMessage(bucket),
            "test_source3",
            Validation.success(test3_input));

    if (test3.isFail()) {
      fail("About to crash with: " + test3.fail().message());
    }
    assertTrue("getStreamingTopology call succeeded", test3.isSuccess());
    assertTrue("topology created: ", test3.success() != null);
    assertEquals(lib_elements.get(0).misc_entry_point(), test3.success().getClass().getName());

    // (Try again but with failing version, due to class not found)

    final ImmutableMap<String, Tuple2<SharedLibraryBean, String>> test3a_input =
        ImmutableMap.<String, Tuple2<SharedLibraryBean, String>>builder()
            .put("test_tech_id_stream_fail", Tuples._2T(lib_elements.get(3), cached_file.success()))
            .build();

    final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test3a =
        DataBucketChangeActor.getStreamingTopology(
            BeanTemplateUtils.clone(bucket)
                .with(
                    DataBucketBean::streaming_enrichment_topology,
                    BeanTemplateUtils.build(EnrichmentControlMetadataBean.class)
                        .with(
                            EnrichmentControlMetadataBean::library_names_or_ids,
                            Arrays.asList("test_tech_id_stream_fail"))
                        .done()
                        .get())
                .done(),
            new BucketActionMessage.BucketActionOfferMessage(bucket),
            "test_source3",
            Validation.success(test3a_input));

    assertTrue("Got error back", test3a.isFail());
    assertTrue(
        "Right error: " + test3a.fail().message(),
        test3a.fail().message().contains("com.ikanow.aleph2.test.example.ExampleStreamTopology"));

    // Now check with the "not just the harvest tech" flag set

    final String java_name2 =
        _service_context.getGlobalProperties().local_cached_jar_dir()
            + File.separator
            + "test_module_id.cache.jar";

    _logger.info(
        "Needed to delete locally cached file? "
            + java_name2
            + ": "
            + new File(java_name2).delete());

    // Requires that the file has already been cached:
    final Validation<BasicMessageBean, String> cached_file2 =
        JarCacheUtils.getCachedJar(
                _service_context.getGlobalProperties().local_cached_jar_dir(),
                lib_elements.get(1),
                _service_context.getStorageService(),
                "test3b",
                "test3b")
            .get();

    if (cached_file2.isFail()) {
      fail("About to crash with: " + cached_file2.fail().message());
    }

    assertTrue("The cached file exists: " + java_name, new File(java_name2).exists());

    final ImmutableMap<String, Tuple2<SharedLibraryBean, String>> test3b_input =
        ImmutableMap.<String, Tuple2<SharedLibraryBean, String>>builder()
            .put("test_tech_id_stream", Tuples._2T(lib_elements.get(0), cached_file.success()))
            .put("test_module_id", Tuples._2T(lib_elements.get(1), cached_file.success()))
            .build();

    final EnrichmentControlMetadataBean enrichment_module =
        new EnrichmentControlMetadataBean(
            "test_tech_name",
            Collections.emptyList(),
            true,
            null,
            Arrays.asList("test_tech_id_stream", "test_module_id"),
            null,
            null);

    final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test3b =
        DataBucketChangeActor.getStreamingTopology(
            BeanTemplateUtils.clone(bucket)
                .with(DataBucketBean::streaming_enrichment_topology, enrichment_module)
                .done(),
            new BucketActionMessage.BucketActionOfferMessage(bucket),
            "test_source3b",
            Validation.success(test3b_input));

    if (test3b.isFail()) {
      fail("About to crash with: " + test3b.fail().message());
    }
    assertTrue("getStreamingTopology call succeeded", test3b.isSuccess());
    assertTrue("topology created: ", test3b.success() != null);
    assertEquals(lib_elements.get(0).misc_entry_point(), test3b.success().getClass().getName());

    // TODO add a test for disabled streaming but config given (should default to passthrough top
    // and
    // ignore given topology
  }
Ejemplo n.º 2
0
  @Test
  public void test_cacheJars()
      throws UnsupportedFileSystemException, InterruptedException, ExecutionException {
    try {
      // Preamble:
      // 0) Insert 2 library beans into the management db

      final DataBucketBean bucket = createBucket("test_tech_id_stream");

      final String pathname1 =
          System.getProperty("user.dir") + "/misc_test_assets/simple-harvest-example.jar";
      final Path path1 = FileContext.getLocalFSFileContext().makeQualified(new Path(pathname1));
      final String pathname2 =
          System.getProperty("user.dir") + "/misc_test_assets/simple-harvest-example2.jar";
      final Path path2 = FileContext.getLocalFSFileContext().makeQualified(new Path(pathname2));

      List<SharedLibraryBean> lib_elements = createSharedLibraryBeans(path1, path2);

      final IManagementDbService underlying_db =
          _service_context.getService(IManagementDbService.class, Optional.empty()).get();
      final IManagementCrudService<SharedLibraryBean> library_crud =
          underlying_db.getSharedLibraryStore();
      library_crud.deleteDatastore();
      assertEquals("Cleansed library store", 0L, (long) library_crud.countObjects().get());
      library_crud.storeObjects(lib_elements).get();

      assertEquals("Should have 4 library beans", 4L, (long) library_crud.countObjects().get());

      // 0a) Check with no streaming, gets nothing
      {
        CompletableFuture<
                Validation<BasicMessageBean, Map<String, Tuple2<SharedLibraryBean, String>>>>
            reply_structure =
                DataBucketChangeActor.cacheJars(
                    bucket,
                    _service_context.getCoreManagementDbService(),
                    _service_context.getGlobalProperties(),
                    _service_context.getStorageService(),
                    _service_context,
                    "test1_source",
                    "test1_command");

        if (reply_structure.get().isFail()) {
          fail("About to crash with: " + reply_structure.get().fail().message());
        }
        assertTrue("cacheJars should return valid reply", reply_structure.get().isSuccess());

        final Map<String, Tuple2<SharedLibraryBean, String>> reply_map =
            reply_structure.get().success();

        assertEquals(0L, reply_map.size()); // (both modules, 1x for _id and 1x for name)
      }

      // 0b) Create the more complex bucket

      final EnrichmentControlMetadataBean enrichment_module =
          new EnrichmentControlMetadataBean(
              "test_name",
              Collections.emptyList(),
              true,
              null,
              Arrays.asList("test_tech_id_stream", "test_module_id"),
              null,
              new LinkedHashMap<>());

      final DataBucketBean bucket2 =
          BeanTemplateUtils.clone(bucket)
              .with(DataBucketBean::streaming_enrichment_topology, enrichment_module)
              .done();

      // 1) Normal operation

      CompletableFuture<
              Validation<BasicMessageBean, Map<String, Tuple2<SharedLibraryBean, String>>>>
          reply_structure =
              DataBucketChangeActor.cacheJars(
                  bucket2,
                  _service_context.getCoreManagementDbService(),
                  _service_context.getGlobalProperties(),
                  _service_context.getStorageService(),
                  _service_context,
                  "test1_source",
                  "test1_command");

      if (reply_structure.get().isFail()) {
        fail("About to crash with: " + reply_structure.get().fail().message());
      }
      assertTrue("cacheJars should return valid reply", reply_structure.get().isSuccess());

      final Map<String, Tuple2<SharedLibraryBean, String>> reply_map =
          reply_structure.get().success();

      assertEquals(
          "Should have 4 beans: " + reply_map.toString(),
          4L,
          reply_map.size()); // (both modules, 1x for _id and 1x for name)

      // 3) Couple of error cases:

      final EnrichmentControlMetadataBean enrichment_module2 =
          new EnrichmentControlMetadataBean(
              "test_name",
              Collections.emptyList(),
              true,
              null,
              Arrays.asList("test_tech_id_stream", "test_module_id", "failtest"),
              null,
              new LinkedHashMap<>());

      final DataBucketBean bucket3 =
          BeanTemplateUtils.clone(bucket)
              .with(DataBucketBean::streaming_enrichment_topology, enrichment_module2)
              .done();

      CompletableFuture<
              Validation<BasicMessageBean, Map<String, Tuple2<SharedLibraryBean, String>>>>
          reply_structure3 =
              DataBucketChangeActor.cacheJars(
                  bucket3,
                  _service_context.getCoreManagementDbService(),
                  _service_context.getGlobalProperties(),
                  _service_context.getStorageService(),
                  _service_context,
                  "test2_source",
                  "test2_command");

      assertTrue("cacheJars should return error", reply_structure3.get().isFail());
    } catch (Exception e) {
      System.out.println(ErrorUtils.getLongForm("guice? {0}", e));
      throw e;
    }
  }