Ejemplo n.º 1
0
  protected static CompletableFuture<BucketActionReplyMessage> talkToStream(
      final IStormController storm_controller,
      final DataBucketBean bucket,
      final BucketActionMessage m,
      final Validation<BasicMessageBean, IEnrichmentStreamingTopology> err_or_user_topology,
      final Validation<BasicMessageBean, Map<String, Tuple2<SharedLibraryBean, String>>> err_or_map,
      final String source,
      final StreamingEnrichmentContext context,
      final String yarn_config_dir,
      final String cached_jars_dir) {
    try {
      // handle getting the user libs
      final List<String> user_lib_paths =
          err_or_map.<List<String>>validation(
              fail -> Collections.emptyList() // (going to die soon anyway)
              ,
              success ->
                  success
                      .values()
                      .stream()
                      .map(tuple -> tuple._2.replaceFirst("file:", ""))
                      .collect(Collectors.toList()));

      return err_or_user_topology.<CompletableFuture<BucketActionReplyMessage>>validation(
          // ERROR getting enrichment topology
          error -> {
            return CompletableFuture.completedFuture(new BucketActionHandlerMessage(source, error));
          },
          // NORMAL grab enrichment topology
          enrichment_topology -> {
            final String entry_point = enrichment_topology.getClass().getName();
            context.setBucket(bucket);
            context.setUserTopologyEntryPoint(entry_point);
            // also set the library bean - note if here then must have been set, else
            // IHarvestTechnologyModule wouldn't exist
            err_or_map.forEach(
                map -> {
                  context.setLibraryConfig(
                      map.values()
                          .stream()
                          .map(t2 -> t2._1())
                          .filter(
                              lib ->
                                  entry_point.equals(lib.misc_entry_point())
                                      || entry_point.equals(lib.streaming_enrichment_entry_point()))
                          .findFirst()
                          .orElse(BeanTemplateUtils.build(SharedLibraryBean.class).done().get()));
                  // (else this is a passthrough topology, so just use a dummy library bean)
                });

            _logger.info(
                "Set active class="
                    + enrichment_topology.getClass()
                    + " message="
                    + m.getClass().getSimpleName()
                    + " bucket="
                    + bucket.full_name());

            return Patterns.match(m)
                .<CompletableFuture<BucketActionReplyMessage>>andReturn()
                .when(
                    BucketActionMessage.DeleteBucketActionMessage.class,
                    msg -> {
                      return StormControllerUtil.stopJob(storm_controller, bucket);
                    })
                .when(
                    BucketActionMessage.NewBucketActionMessage.class,
                    msg -> {
                      if (!msg.is_suspended())
                        return StormControllerUtil.startJob(
                            storm_controller,
                            bucket,
                            context,
                            user_lib_paths,
                            enrichment_topology,
                            cached_jars_dir);
                      else
                        return StormControllerUtil.stopJob(
                            storm_controller,
                            bucket); // (nothing to do but just do this to return something
                      // sensible)
                    })
                .when(
                    BucketActionMessage.UpdateBucketActionMessage.class,
                    msg -> {
                      if (msg.is_enabled())
                        return StormControllerUtil.restartJob(
                            storm_controller,
                            bucket,
                            context,
                            user_lib_paths,
                            enrichment_topology,
                            cached_jars_dir);
                      else return StormControllerUtil.stopJob(storm_controller, bucket);
                    })
                .when(
                    BucketActionMessage.TestBucketActionMessage.class,
                    msg -> {
                      // TODO (ALEPH-25): in the future run this test with local storm rather than
                      // remote storm_controller
                      return StormControllerUtil.restartJob(
                          storm_controller,
                          bucket,
                          context,
                          user_lib_paths,
                          enrichment_topology,
                          cached_jars_dir);
                    })
                .otherwise(
                    msg -> {
                      return CompletableFuture.completedFuture(
                          new BucketActionHandlerMessage(
                              source,
                              new BasicMessageBean(
                                  new Date(),
                                  false,
                                  null,
                                  "Unknown message",
                                  0,
                                  "Unknown message",
                                  null)));
                    });
          });
    } catch (Throwable e) { // (trying to use Validation to avoid this, but just in case...)
      return CompletableFuture.completedFuture(
          new BucketActionHandlerMessage(
              source,
              new BasicMessageBean(
                  new Date(),
                  false,
                  null,
                  ErrorUtils.getLongForm("Error loading streaming class: {0}", e),
                  0,
                  ErrorUtils.getLongForm("Error loading streaming class: {0}", e),
                  null)));
    }
  }
Ejemplo n.º 2
0
  /* (non-Javadoc)
   * @see akka.actor.AbstractActor#receive()
   */
  @Override
  public PartialFunction<Object, BoxedUnit> receive() {
    return ReceiveBuilder.match(
            BucketActionMessage.class,
            m ->
                !m.handling_clients().isEmpty()
                    && !m.handling_clients()
                        .contains(_context.getInformationService().getHostname()),
            __ -> {}) // (do nothing if it's not for me)
        .match(
            BucketActionOfferMessage.class,
            m -> {
              _logger.info(
                  ErrorUtils.get(
                      "Actor {0} received message {1} from {2} bucket {3}",
                      this.self(),
                      m.getClass().getSimpleName(),
                      this.sender(),
                      m.bucket().full_name()));

              final ActorRef closing_sender = this.sender();
              final ActorRef closing_self = this.self();

              final String hostname = _context.getInformationService().getHostname();

              // (this isn't async so doesn't require any futures)

              final boolean accept_or_ignore =
                  new File(_globals.local_yarn_config_dir() + File.separator + "storm.yaml")
                      .exists();

              final BucketActionReplyMessage reply =
                  accept_or_ignore
                      ? new BucketActionReplyMessage.BucketActionWillAcceptMessage(hostname)
                      : new BucketActionReplyMessage.BucketActionIgnoredMessage(hostname);

              closing_sender.tell(reply, closing_self);
            })
        .match(
            BucketActionMessage.class,
            m -> {
              _logger.info(
                  ErrorUtils.get(
                      "Actor {0} received message {1} from {2} bucket={3}",
                      this.self(),
                      m.getClass().getSimpleName(),
                      this.sender(),
                      m.bucket().full_name()));

              final ActorRef closing_sender = this.sender();
              final ActorRef closing_self = this.self();

              final String hostname = _context.getInformationService().getHostname();

              // (cacheJars can't throw checked or unchecked in this thread, only from within
              // exceptions)
              cacheJars(
                      m.bucket(),
                      _management_db,
                      _globals,
                      _fs,
                      _context.getServiceContext(),
                      hostname,
                      m)
                  .thenComposeAsync(
                      err_or_map -> {
                        final StreamingEnrichmentContext e_context =
                            _context.getNewStreamingEnrichmentContext();

                        final Validation<BasicMessageBean, IEnrichmentStreamingTopology>
                            err_or_tech_module =
                                getStreamingTopology(m.bucket(), m, hostname, err_or_map);

                        final CompletableFuture<BucketActionReplyMessage> ret =
                            talkToStream(
                                _storm_controller,
                                m.bucket(),
                                m,
                                err_or_tech_module,
                                err_or_map,
                                hostname,
                                e_context,
                                _globals.local_yarn_config_dir(),
                                _globals.local_cached_jar_dir());
                        return ret;
                      })
                  .thenAccept(
                      reply -> { // (reply can contain an error or successful reply, they're the
                        // same bean type)
                        // Some information logging:
                        Patterns.match(reply)
                            .andAct()
                            .when(
                                BucketActionHandlerMessage.class,
                                msg ->
                                    _logger.info(
                                        ErrorUtils.get(
                                            "Standard reply to message={0}, bucket={1}, success={2}",
                                            m.getClass().getSimpleName(),
                                            m.bucket().full_name(),
                                            msg.reply().success())))
                            .when(
                                BucketActionReplyMessage.BucketActionWillAcceptMessage.class,
                                msg ->
                                    _logger.info(
                                        ErrorUtils.get(
                                            "Standard reply to message={0}, bucket={1}",
                                            m.getClass().getSimpleName(), m.bucket().full_name())))
                            .otherwise(
                                msg ->
                                    _logger.info(
                                        ErrorUtils.get(
                                            "Unusual reply to message={0}, type={2}, bucket={1}",
                                            m.getClass().getSimpleName(),
                                            m.bucket().full_name(),
                                            msg.getClass().getSimpleName())));

                        closing_sender.tell(reply, closing_self);
                      })
                  .exceptionally(
                      e -> { // another bit of error handling that shouldn't ever be called but is a
                        // useful backstop
                        // Some information logging:
                        _logger.warn(
                            "Unexpected error replying to '{0}': error = {1}, bucket={2}",
                            BeanTemplateUtils.toJson(m).toString(),
                            ErrorUtils.getLongForm("{0}", e),
                            m.bucket().full_name());

                        final BasicMessageBean error_bean =
                            SharedErrorUtils.buildErrorMessage(
                                hostname,
                                m,
                                ErrorUtils.getLongForm(
                                    StreamErrorUtils.STREAM_UNKNOWN_ERROR,
                                    e,
                                    m.bucket().full_name()));
                        closing_sender.tell(
                            new BucketActionHandlerMessage(hostname, error_bean), closing_self);
                        return null;
                      });
            })
        .build();
  }