@Test
public void test_getStreamingTopology()
throws UnsupportedFileSystemException, InterruptedException, ExecutionException {
final DataBucketBean bucket = createBucket("test_tech_id_stream");
final String pathname1 =
System.getProperty("user.dir") + "/misc_test_assets/simple-topology-example.jar";
final Path path1 = FileContext.getLocalFSFileContext().makeQualified(new Path(pathname1));
final String pathname2 =
System.getProperty("user.dir") + "/misc_test_assets/simple-harvest-example2.jar";
final Path path2 = FileContext.getLocalFSFileContext().makeQualified(new Path(pathname2));
List<SharedLibraryBean> lib_elements = createSharedLibraryBeans(path1, path2);
//////////////////////////////////////////////////////
// 1) Check - if called with an error, then just passes that error along
final BasicMessageBean error =
SharedErrorUtils.buildErrorMessage("test_source", "test_message", "test_error");
final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test1 =
DataBucketChangeActor.getStreamingTopology(
bucket,
new BucketActionMessage.BucketActionOfferMessage(bucket),
"test_source2",
Validation.fail(error));
assertTrue("Got error back", test1.isFail());
assertEquals("test_source", test1.fail().source());
assertEquals("test_message", test1.fail().command());
assertEquals("test_error", test1.fail().message());
//////////////////////////////////////////////////////
// 2) Check the error handling inside getStreamingTopology
final ImmutableMap<String, Tuple2<SharedLibraryBean, String>> test2_input =
ImmutableMap.<String, Tuple2<SharedLibraryBean, String>>builder()
.put("test_tech_id_stream_2b", Tuples._2T(null, null))
.build();
final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test2a =
DataBucketChangeActor.getStreamingTopology(
BeanTemplateUtils.clone(bucket)
.with(
DataBucketBean::streaming_enrichment_topology,
BeanTemplateUtils.build(EnrichmentControlMetadataBean.class)
.with(
EnrichmentControlMetadataBean::library_names_or_ids,
Arrays.asList("test_tech_id_stream_2a"))
.done()
.get())
.done(),
new BucketActionMessage.BucketActionOfferMessage(bucket),
"test_source2a",
Validation.success(test2_input));
assertTrue("Got error back", test2a.isFail());
assertEquals("test_source2a", test2a.fail().source());
assertEquals("BucketActionOfferMessage", test2a.fail().command());
assertEquals(
ErrorUtils.get(
SharedErrorUtils.SHARED_LIBRARY_NAME_NOT_FOUND,
bucket.full_name(),
"(unknown)"), // (cloned bucket above)
test2a.fail().message());
final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test2b =
DataBucketChangeActor.getStreamingTopology(
BeanTemplateUtils.clone(bucket)
.with(
DataBucketBean::streaming_enrichment_topology,
BeanTemplateUtils.build(EnrichmentControlMetadataBean.class)
.with(
EnrichmentControlMetadataBean::library_names_or_ids,
Arrays.asList("test_tech_id_stream_2b"))
.done()
.get())
.done(),
new BucketActionMessage.BucketActionOfferMessage(bucket),
"test_source2b",
Validation.success(test2_input));
assertTrue("Got error back", test2b.isFail());
assertEquals("test_source2b", test2b.fail().source());
assertEquals("BucketActionOfferMessage", test2b.fail().command());
assertEquals(
ErrorUtils.get(
SharedErrorUtils.SHARED_LIBRARY_NAME_NOT_FOUND,
bucket.full_name(),
"(unknown)"), // (cloned bucket above)
test2a.fail().message());
//////////////////////////////////////////////////////
// 3) OK now it will actually do something
final String java_name =
_service_context.getGlobalProperties().local_cached_jar_dir()
+ File.separator
+ "test_tech_id_stream.cache.jar";
_logger.info(
"Needed to delete locally cached file? " + java_name + ": " + new File(java_name).delete());
// Requires that the file has already been cached:
final Validation<BasicMessageBean, String> cached_file =
JarCacheUtils.getCachedJar(
_service_context.getGlobalProperties().local_cached_jar_dir(),
lib_elements.get(0),
_service_context.getStorageService(),
"test3",
"test3")
.get();
if (cached_file.isFail()) {
fail("About to crash with: " + cached_file.fail().message());
}
assertTrue("The cached file exists: " + java_name, new File(java_name).exists());
// OK the setup is done and validated now actually test the underlying call:
final ImmutableMap<String, Tuple2<SharedLibraryBean, String>> test3_input =
ImmutableMap.<String, Tuple2<SharedLibraryBean, String>>builder()
.put("test_tech_id_stream", Tuples._2T(lib_elements.get(0), cached_file.success()))
.build();
final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test3 =
DataBucketChangeActor.getStreamingTopology(
BeanTemplateUtils.clone(bucket)
.with(
DataBucketBean::streaming_enrichment_topology,
BeanTemplateUtils.build(EnrichmentControlMetadataBean.class)
.with(
EnrichmentControlMetadataBean::library_names_or_ids,
Arrays.asList("test_tech_id_stream"))
.done()
.get())
.done(),
new BucketActionMessage.BucketActionOfferMessage(bucket),
"test_source3",
Validation.success(test3_input));
if (test3.isFail()) {
fail("About to crash with: " + test3.fail().message());
}
assertTrue("getStreamingTopology call succeeded", test3.isSuccess());
assertTrue("topology created: ", test3.success() != null);
assertEquals(lib_elements.get(0).misc_entry_point(), test3.success().getClass().getName());
// (Try again but with failing version, due to class not found)
final ImmutableMap<String, Tuple2<SharedLibraryBean, String>> test3a_input =
ImmutableMap.<String, Tuple2<SharedLibraryBean, String>>builder()
.put("test_tech_id_stream_fail", Tuples._2T(lib_elements.get(3), cached_file.success()))
.build();
final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test3a =
DataBucketChangeActor.getStreamingTopology(
BeanTemplateUtils.clone(bucket)
.with(
DataBucketBean::streaming_enrichment_topology,
BeanTemplateUtils.build(EnrichmentControlMetadataBean.class)
.with(
EnrichmentControlMetadataBean::library_names_or_ids,
Arrays.asList("test_tech_id_stream_fail"))
.done()
.get())
.done(),
new BucketActionMessage.BucketActionOfferMessage(bucket),
"test_source3",
Validation.success(test3a_input));
assertTrue("Got error back", test3a.isFail());
assertTrue(
"Right error: " + test3a.fail().message(),
test3a.fail().message().contains("com.ikanow.aleph2.test.example.ExampleStreamTopology"));
// Now check with the "not just the harvest tech" flag set
final String java_name2 =
_service_context.getGlobalProperties().local_cached_jar_dir()
+ File.separator
+ "test_module_id.cache.jar";
_logger.info(
"Needed to delete locally cached file? "
+ java_name2
+ ": "
+ new File(java_name2).delete());
// Requires that the file has already been cached:
final Validation<BasicMessageBean, String> cached_file2 =
JarCacheUtils.getCachedJar(
_service_context.getGlobalProperties().local_cached_jar_dir(),
lib_elements.get(1),
_service_context.getStorageService(),
"test3b",
"test3b")
.get();
if (cached_file2.isFail()) {
fail("About to crash with: " + cached_file2.fail().message());
}
assertTrue("The cached file exists: " + java_name, new File(java_name2).exists());
final ImmutableMap<String, Tuple2<SharedLibraryBean, String>> test3b_input =
ImmutableMap.<String, Tuple2<SharedLibraryBean, String>>builder()
.put("test_tech_id_stream", Tuples._2T(lib_elements.get(0), cached_file.success()))
.put("test_module_id", Tuples._2T(lib_elements.get(1), cached_file.success()))
.build();
final EnrichmentControlMetadataBean enrichment_module =
new EnrichmentControlMetadataBean(
"test_tech_name",
Collections.emptyList(),
true,
null,
Arrays.asList("test_tech_id_stream", "test_module_id"),
null,
null);
final Validation<BasicMessageBean, IEnrichmentStreamingTopology> test3b =
DataBucketChangeActor.getStreamingTopology(
BeanTemplateUtils.clone(bucket)
.with(DataBucketBean::streaming_enrichment_topology, enrichment_module)
.done(),
new BucketActionMessage.BucketActionOfferMessage(bucket),
"test_source3b",
Validation.success(test3b_input));
if (test3b.isFail()) {
fail("About to crash with: " + test3b.fail().message());
}
assertTrue("getStreamingTopology call succeeded", test3b.isSuccess());
assertTrue("topology created: ", test3b.success() != null);
assertEquals(lib_elements.get(0).misc_entry_point(), test3b.success().getClass().getName());
// TODO add a test for disabled streaming but config given (should default to passthrough top
// and
// ignore given topology
}
@Test
public void test_cacheJars()
throws UnsupportedFileSystemException, InterruptedException, ExecutionException {
try {
// Preamble:
// 0) Insert 2 library beans into the management db
final DataBucketBean bucket = createBucket("test_tech_id_stream");
final String pathname1 =
System.getProperty("user.dir") + "/misc_test_assets/simple-harvest-example.jar";
final Path path1 = FileContext.getLocalFSFileContext().makeQualified(new Path(pathname1));
final String pathname2 =
System.getProperty("user.dir") + "/misc_test_assets/simple-harvest-example2.jar";
final Path path2 = FileContext.getLocalFSFileContext().makeQualified(new Path(pathname2));
List<SharedLibraryBean> lib_elements = createSharedLibraryBeans(path1, path2);
final IManagementDbService underlying_db =
_service_context.getService(IManagementDbService.class, Optional.empty()).get();
final IManagementCrudService<SharedLibraryBean> library_crud =
underlying_db.getSharedLibraryStore();
library_crud.deleteDatastore();
assertEquals("Cleansed library store", 0L, (long) library_crud.countObjects().get());
library_crud.storeObjects(lib_elements).get();
assertEquals("Should have 4 library beans", 4L, (long) library_crud.countObjects().get());
// 0a) Check with no streaming, gets nothing
{
CompletableFuture<
Validation<BasicMessageBean, Map<String, Tuple2<SharedLibraryBean, String>>>>
reply_structure =
DataBucketChangeActor.cacheJars(
bucket,
_service_context.getCoreManagementDbService(),
_service_context.getGlobalProperties(),
_service_context.getStorageService(),
_service_context,
"test1_source",
"test1_command");
if (reply_structure.get().isFail()) {
fail("About to crash with: " + reply_structure.get().fail().message());
}
assertTrue("cacheJars should return valid reply", reply_structure.get().isSuccess());
final Map<String, Tuple2<SharedLibraryBean, String>> reply_map =
reply_structure.get().success();
assertEquals(0L, reply_map.size()); // (both modules, 1x for _id and 1x for name)
}
// 0b) Create the more complex bucket
final EnrichmentControlMetadataBean enrichment_module =
new EnrichmentControlMetadataBean(
"test_name",
Collections.emptyList(),
true,
null,
Arrays.asList("test_tech_id_stream", "test_module_id"),
null,
new LinkedHashMap<>());
final DataBucketBean bucket2 =
BeanTemplateUtils.clone(bucket)
.with(DataBucketBean::streaming_enrichment_topology, enrichment_module)
.done();
// 1) Normal operation
CompletableFuture<
Validation<BasicMessageBean, Map<String, Tuple2<SharedLibraryBean, String>>>>
reply_structure =
DataBucketChangeActor.cacheJars(
bucket2,
_service_context.getCoreManagementDbService(),
_service_context.getGlobalProperties(),
_service_context.getStorageService(),
_service_context,
"test1_source",
"test1_command");
if (reply_structure.get().isFail()) {
fail("About to crash with: " + reply_structure.get().fail().message());
}
assertTrue("cacheJars should return valid reply", reply_structure.get().isSuccess());
final Map<String, Tuple2<SharedLibraryBean, String>> reply_map =
reply_structure.get().success();
assertEquals(
"Should have 4 beans: " + reply_map.toString(),
4L,
reply_map.size()); // (both modules, 1x for _id and 1x for name)
// 3) Couple of error cases:
final EnrichmentControlMetadataBean enrichment_module2 =
new EnrichmentControlMetadataBean(
"test_name",
Collections.emptyList(),
true,
null,
Arrays.asList("test_tech_id_stream", "test_module_id", "failtest"),
null,
new LinkedHashMap<>());
final DataBucketBean bucket3 =
BeanTemplateUtils.clone(bucket)
.with(DataBucketBean::streaming_enrichment_topology, enrichment_module2)
.done();
CompletableFuture<
Validation<BasicMessageBean, Map<String, Tuple2<SharedLibraryBean, String>>>>
reply_structure3 =
DataBucketChangeActor.cacheJars(
bucket3,
_service_context.getCoreManagementDbService(),
_service_context.getGlobalProperties(),
_service_context.getStorageService(),
_service_context,
"test2_source",
"test2_command");
assertTrue("cacheJars should return error", reply_structure3.get().isFail());
} catch (Exception e) {
System.out.println(ErrorUtils.getLongForm("guice? {0}", e));
throw e;
}
}