protected static CompletableFuture<BucketActionReplyMessage> talkToStream(
final IStormController storm_controller,
final DataBucketBean bucket,
final BucketActionMessage m,
final Validation<BasicMessageBean, IEnrichmentStreamingTopology> err_or_user_topology,
final Validation<BasicMessageBean, Map<String, Tuple2<SharedLibraryBean, String>>> err_or_map,
final String source,
final StreamingEnrichmentContext context,
final String yarn_config_dir,
final String cached_jars_dir) {
try {
// handle getting the user libs
final List<String> user_lib_paths =
err_or_map.<List<String>>validation(
fail -> Collections.emptyList() // (going to die soon anyway)
,
success ->
success
.values()
.stream()
.map(tuple -> tuple._2.replaceFirst("file:", ""))
.collect(Collectors.toList()));
return err_or_user_topology.<CompletableFuture<BucketActionReplyMessage>>validation(
// ERROR getting enrichment topology
error -> {
return CompletableFuture.completedFuture(new BucketActionHandlerMessage(source, error));
},
// NORMAL grab enrichment topology
enrichment_topology -> {
final String entry_point = enrichment_topology.getClass().getName();
context.setBucket(bucket);
context.setUserTopologyEntryPoint(entry_point);
// also set the library bean - note if here then must have been set, else
// IHarvestTechnologyModule wouldn't exist
err_or_map.forEach(
map -> {
context.setLibraryConfig(
map.values()
.stream()
.map(t2 -> t2._1())
.filter(
lib ->
entry_point.equals(lib.misc_entry_point())
|| entry_point.equals(lib.streaming_enrichment_entry_point()))
.findFirst()
.orElse(BeanTemplateUtils.build(SharedLibraryBean.class).done().get()));
// (else this is a passthrough topology, so just use a dummy library bean)
});
_logger.info(
"Set active class="
+ enrichment_topology.getClass()
+ " message="
+ m.getClass().getSimpleName()
+ " bucket="
+ bucket.full_name());
return Patterns.match(m)
.<CompletableFuture<BucketActionReplyMessage>>andReturn()
.when(
BucketActionMessage.DeleteBucketActionMessage.class,
msg -> {
return StormControllerUtil.stopJob(storm_controller, bucket);
})
.when(
BucketActionMessage.NewBucketActionMessage.class,
msg -> {
if (!msg.is_suspended())
return StormControllerUtil.startJob(
storm_controller,
bucket,
context,
user_lib_paths,
enrichment_topology,
cached_jars_dir);
else
return StormControllerUtil.stopJob(
storm_controller,
bucket); // (nothing to do but just do this to return something
// sensible)
})
.when(
BucketActionMessage.UpdateBucketActionMessage.class,
msg -> {
if (msg.is_enabled())
return StormControllerUtil.restartJob(
storm_controller,
bucket,
context,
user_lib_paths,
enrichment_topology,
cached_jars_dir);
else return StormControllerUtil.stopJob(storm_controller, bucket);
})
.when(
BucketActionMessage.TestBucketActionMessage.class,
msg -> {
// TODO (ALEPH-25): in the future run this test with local storm rather than
// remote storm_controller
return StormControllerUtil.restartJob(
storm_controller,
bucket,
context,
user_lib_paths,
enrichment_topology,
cached_jars_dir);
})
.otherwise(
msg -> {
return CompletableFuture.completedFuture(
new BucketActionHandlerMessage(
source,
new BasicMessageBean(
new Date(),
false,
null,
"Unknown message",
0,
"Unknown message",
null)));
});
});
} catch (Throwable e) { // (trying to use Validation to avoid this, but just in case...)
return CompletableFuture.completedFuture(
new BucketActionHandlerMessage(
source,
new BasicMessageBean(
new Date(),
false,
null,
ErrorUtils.getLongForm("Error loading streaming class: {0}", e),
0,
ErrorUtils.getLongForm("Error loading streaming class: {0}", e),
null)));
}
}
/* (non-Javadoc)
* @see akka.actor.AbstractActor#receive()
*/
@Override
public PartialFunction<Object, BoxedUnit> receive() {
return ReceiveBuilder.match(
BucketActionMessage.class,
m ->
!m.handling_clients().isEmpty()
&& !m.handling_clients()
.contains(_context.getInformationService().getHostname()),
__ -> {}) // (do nothing if it's not for me)
.match(
BucketActionOfferMessage.class,
m -> {
_logger.info(
ErrorUtils.get(
"Actor {0} received message {1} from {2} bucket {3}",
this.self(),
m.getClass().getSimpleName(),
this.sender(),
m.bucket().full_name()));
final ActorRef closing_sender = this.sender();
final ActorRef closing_self = this.self();
final String hostname = _context.getInformationService().getHostname();
// (this isn't async so doesn't require any futures)
final boolean accept_or_ignore =
new File(_globals.local_yarn_config_dir() + File.separator + "storm.yaml")
.exists();
final BucketActionReplyMessage reply =
accept_or_ignore
? new BucketActionReplyMessage.BucketActionWillAcceptMessage(hostname)
: new BucketActionReplyMessage.BucketActionIgnoredMessage(hostname);
closing_sender.tell(reply, closing_self);
})
.match(
BucketActionMessage.class,
m -> {
_logger.info(
ErrorUtils.get(
"Actor {0} received message {1} from {2} bucket={3}",
this.self(),
m.getClass().getSimpleName(),
this.sender(),
m.bucket().full_name()));
final ActorRef closing_sender = this.sender();
final ActorRef closing_self = this.self();
final String hostname = _context.getInformationService().getHostname();
// (cacheJars can't throw checked or unchecked in this thread, only from within
// exceptions)
cacheJars(
m.bucket(),
_management_db,
_globals,
_fs,
_context.getServiceContext(),
hostname,
m)
.thenComposeAsync(
err_or_map -> {
final StreamingEnrichmentContext e_context =
_context.getNewStreamingEnrichmentContext();
final Validation<BasicMessageBean, IEnrichmentStreamingTopology>
err_or_tech_module =
getStreamingTopology(m.bucket(), m, hostname, err_or_map);
final CompletableFuture<BucketActionReplyMessage> ret =
talkToStream(
_storm_controller,
m.bucket(),
m,
err_or_tech_module,
err_or_map,
hostname,
e_context,
_globals.local_yarn_config_dir(),
_globals.local_cached_jar_dir());
return ret;
})
.thenAccept(
reply -> { // (reply can contain an error or successful reply, they're the
// same bean type)
// Some information logging:
Patterns.match(reply)
.andAct()
.when(
BucketActionHandlerMessage.class,
msg ->
_logger.info(
ErrorUtils.get(
"Standard reply to message={0}, bucket={1}, success={2}",
m.getClass().getSimpleName(),
m.bucket().full_name(),
msg.reply().success())))
.when(
BucketActionReplyMessage.BucketActionWillAcceptMessage.class,
msg ->
_logger.info(
ErrorUtils.get(
"Standard reply to message={0}, bucket={1}",
m.getClass().getSimpleName(), m.bucket().full_name())))
.otherwise(
msg ->
_logger.info(
ErrorUtils.get(
"Unusual reply to message={0}, type={2}, bucket={1}",
m.getClass().getSimpleName(),
m.bucket().full_name(),
msg.getClass().getSimpleName())));
closing_sender.tell(reply, closing_self);
})
.exceptionally(
e -> { // another bit of error handling that shouldn't ever be called but is a
// useful backstop
// Some information logging:
_logger.warn(
"Unexpected error replying to '{0}': error = {1}, bucket={2}",
BeanTemplateUtils.toJson(m).toString(),
ErrorUtils.getLongForm("{0}", e),
m.bucket().full_name());
final BasicMessageBean error_bean =
SharedErrorUtils.buildErrorMessage(
hostname,
m,
ErrorUtils.getLongForm(
StreamErrorUtils.STREAM_UNKNOWN_ERROR,
e,
m.bucket().full_name()));
closing_sender.tell(
new BucketActionHandlerMessage(hostname, error_bean), closing_self);
return null;
});
})
.build();
}