@Test
  public void usePatternsAskPipe() {
    ActorSystem system = ActorSystem.create("MySystem");
    ActorRef actorA = system.actorOf(new Props(MyUntypedActor.class));
    ActorRef actorB = system.actorOf(new Props(MyUntypedActor.class));
    ActorRef actorC = system.actorOf(new Props(MyUntypedActor.class));
    // #ask-pipe
    final Timeout t = new Timeout(Duration.create(5, TimeUnit.SECONDS));

    final ArrayList<Future<Object>> futures = new ArrayList<Future<Object>>();
    futures.add(ask(actorA, "request", 1000)); // using 1000ms timeout
    futures.add(ask(actorB, "another request", t)); // using timeout from above

    final Future<Iterable<Object>> aggregate = Futures.sequence(futures, system.dispatcher());

    final Future<Result> transformed =
        aggregate.map(
            new Mapper<Iterable<Object>, Result>() {
              public Result apply(Iterable<Object> coll) {
                final Iterator<Object> it = coll.iterator();
                final String s = (String) it.next();
                final int x = (Integer) it.next();
                return new Result(x, s);
              }
            },
            system.dispatcher());

    pipe(transformed, system.dispatcher()).to(actorC);
    // #ask-pipe
    system.shutdown();
  }
  @Override
  public F.Promise<Void> store(Path path, String key) {
    Promise<Void> promise = Futures.promise();

    TransferManager transferManager = new TransferManager(credentials);
    Upload upload = transferManager.upload(bucketName, key, path.toFile());

    upload.addProgressListener(
        (ProgressListener)
            progressEvent -> {
              if (progressEvent.getEventType().isTransferEvent()) {
                if (progressEvent
                    .getEventType()
                    .equals(ProgressEventType.TRANSFER_COMPLETED_EVENT)) {
                  transferManager.shutdownNow();
                  promise.success(null);
                } else if (progressEvent
                    .getEventType()
                    .equals(ProgressEventType.TRANSFER_FAILED_EVENT)) {
                  transferManager.shutdownNow();
                  promise.failure(new Exception("Upload failed"));
                }
              }
            });

    return F.Promise.wrap(promise.future());
  }
Beispiel #3
0
  /**
   * Trains the network on this mini batch and returns a list of futures for each training job
   *
   * @param trainingBatch the trees to iterate on
   */
  public List<Future<Object>> fitAsync(final List<Tree> trainingBatch) {
    int count = 0;

    List<Future<Object>> futureBatch = new ArrayList<>();

    for (final Tree t : trainingBatch) {
      log.info("Working mini batch " + count++);
      futureBatch.add(
          Futures.future(
              new Callable<Object>() {
                @Override
                public Object call() throws Exception {
                  forwardPropagateTree(t);
                  try {
                    INDArray params = getParameters();
                    INDArray gradient = getValueGradient(trainingBatch);
                    if (params.length() != gradient.length())
                      throw new IllegalStateException("Params not equal to gradient!");
                    setParams(params.subi(gradient));
                  } catch (NegativeArraySizeException e) {
                    log.warn(
                        "Couldnt compute parameters due to negative array size...for trees " + t);
                  }

                  return null;
                }
              },
              rnTnActorSystem.dispatcher()));
    }
    return futureBatch;
  }
  @Override
  protected Future<MetadataItem> next() {
    log.debug("next");

    Entry<String, byte[]> entry = t.next();
    return Futures.successful(new MetadataItem(entry.getKey(), entry.getValue()));
  }
Beispiel #5
0
 @Test
 public void usePipeWithActorSelection() throws Exception {
   TestProbe probe = new TestProbe(system);
   ActorSelection selection = system.actorSelection(probe.ref().path());
   pipe(Futures.successful("hi!"), system.dispatcher()).to(selection);
   probe.expectMsg("hi!");
 }
  /**
   * Entry point. All recent finished azkaban jobs' lineage. Will write to database stagging table
   *
   * @param timeFrame in minutes
   * @param endTimeStamp in millisecond
   * @throws Exception
   */
  public void run(int timeFrame, long endTimeStamp) throws Exception {
    // get recent finished job
    AzJobChecker azJobChecker = new AzJobChecker(prop);
    List<AzkabanJobExecRecord> jobExecList =
        azJobChecker.getRecentFinishedJobFromFlow(timeFrame, endTimeStamp);
    azJobChecker.close();
    logger.info("Total number of azkaban jobs : {}", jobExecList.size());

    ActorSystem actorSystem = ActorSystem.create("LineageExtractor");
    int numOfActor = Integer.valueOf(prop.getProperty(Constant.LINEAGE_ACTOR_NUM, "50"));
    ActorRef lineageExtractorActor =
        actorSystem.actorOf(
            Props.create(AzLineageExtractorActor.class)
                .withRouter(new SmallestMailboxRouter(numOfActor)),
            "lineageExtractorActor");

    // initialize
    // AzkabanServiceCommunicator asc = new AzkabanServiceCommunicator(prop);
    HadoopNameNodeExtractor hnne = new HadoopNameNodeExtractor(prop);
    AzDbCommunicator adc = new AzDbCommunicator(prop);

    String wherehowsUrl = prop.getProperty(Constant.WH_DB_URL_KEY);
    String wherehowsUserName = prop.getProperty(Constant.WH_DB_USERNAME_KEY);
    String wherehowsPassWord = prop.getProperty(Constant.WH_DB_PASSWORD_KEY);
    String connUrl =
        wherehowsUrl + "?" + "user="******"&password="******"stg_job_execution_data_lineage");

    AzLogParser.initialize(conn);
    PathAnalyzer.initialize(conn);
    int timeout = 30; // default 30 minutes for one job
    if (prop.containsKey(Constant.LINEAGE_ACTOR_TIMEOUT_KEY))
      timeout = Integer.valueOf(prop.getProperty(Constant.LINEAGE_ACTOR_TIMEOUT_KEY));
    List<Future<Object>> result = new ArrayList<>();
    for (AzkabanJobExecRecord aje : jobExecList) {
      AzExecMessage message = new AzExecMessage(aje, prop);
      message.asc = null;
      message.hnne = hnne;
      message.adc = adc;
      message.databaseWriter = databaseWriter;
      message.connection = conn;
      Timeout t = new Timeout(timeout, TimeUnit.SECONDS);
      Future<Object> fut = Patterns.ask(lineageExtractorActor, message, t);
      result.add(fut);
    }

    // join all threads
    Future<Iterable<Object>> seq = Futures.sequence(result, actorSystem.dispatcher());
    try {
      Await.result(seq, Duration.create(timeout + " seconds"));
    } catch (TimeoutException exception) {
      exception.printStackTrace();
    }

    adc.close();
    hnne.close();
    databaseWriter.close();
    logger.info("All job finished lineage collecting!");
  }
Beispiel #7
0
 public static void main(String[] args) throws Exception {
   final ActorSystem system = system();
   final ExecutionContextExecutor dispatcher = system.dispatcher();
   Future<Long> fr = Futures.future(task, dispatcher);
   Future<Long> sc = Futures.future(task, dispatcher);
   Future<Long> th = Futures.future(task, dispatcher);
   Future<Long> fo = Futures.future(task, dispatcher);
   fr.onComplete(complete, dispatcher);
   sc.onComplete(complete, dispatcher);
   th.onComplete(complete, dispatcher);
   fo.onComplete(complete, dispatcher);
   Future<Iterable<Long>> sec = Futures.sequence(Arrays.asList(fr, sc, th, fo), dispatcher);
   Patterns.pipe(sec, dispatcher)
       .to(system.actorOf(Props.create(F.class)))
       .future()
       .ready(Duration.create(20, TimeUnit.SECONDS), null);
   Await.ready(system.terminate(), Duration.Inf());
 }
 public Future<Instance> startInstanceAsync(AWSCredentials credentials) {
   Future<Instance> f =
       circuitBreaker.callWithCircuitBreaker(
           () -> Futures.future(() -> startInstance(credentials), executionContext));
   PartialFunction<Throwable, Future<Instance>> recovery =
       new PFBuilder<Throwable, Future<Instance>>()
           .match(
               AmazonClientException.class,
               ex -> ex.isRetryable(),
               ex -> startInstanceAsync(credentials))
           .build();
   return f.recoverWith(recovery, executionContext);
 }
  private Future<Iterable<Object>> invokeCohorts(Object message) {
    List<Future<Object>> futureList = Lists.newArrayListWithCapacity(cohorts.size());
    for (ActorSelection cohort : cohorts) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Tx {}: Sending {} to cohort {}", transactionId, message, cohort);
      }
      futureList.add(
          actorContext.executeOperationAsync(
              cohort, message, actorContext.getTransactionCommitOperationTimeout()));
    }

    return Futures.sequence(futureList, actorContext.getClientDispatcher());
  }
 public Future<TerminateInstancesResult> terminateInstancesAsync(
     AmazonEC2Client client, Instance... instances) {
   List<String> ids =
       Arrays.stream(instances).map(i -> i.getInstanceId()).collect(Collectors.toList());
   TerminateInstancesRequest request = new TerminateInstancesRequest(ids);
   Future<TerminateInstancesResult> f =
       circuitBreaker.callWithCircuitBreaker(
           () -> Futures.future(() -> client.terminateInstances(request), executionContext));
   PartialFunction<Throwable, Future<TerminateInstancesResult>> recovery =
       new PFBuilder<Throwable, Future<TerminateInstancesResult>>()
           .match(
               AmazonClientException.class,
               ex -> ex.isRetryable(),
               ex -> terminateInstancesAsync(client, instances))
           .build();
   return f.recoverWith(recovery, executionContext);
 }
  public Future<RunInstancesResult> runInstancesAsync(
      RunInstancesRequest request, AmazonEC2Async client) {
    Promise<RunInstancesResult> promise = Futures.promise();
    client.runInstancesAsync(
        request,
        new AsyncHandler<RunInstancesRequest, RunInstancesResult>() {
          @Override
          public void onSuccess(RunInstancesRequest request, RunInstancesResult result) {
            promise.success(result);
          }

          @Override
          public void onError(Exception exception) {
            promise.failure(exception);
          }
        });
    return promise.future();
  }
  private Future<Void> buildCohortList() {

    Future<Iterable<ActorSelection>> combinedFutures =
        Futures.sequence(cohortFutures, actorContext.getClientDispatcher());

    return combinedFutures.transform(
        new AbstractFunction1<Iterable<ActorSelection>, Void>() {
          @Override
          public Void apply(Iterable<ActorSelection> actorSelections) {
            cohorts = Lists.newArrayList(actorSelections);
            if (LOG.isDebugEnabled()) {
              LOG.debug("Tx {} successfully built cohort path list: {}", transactionId, cohorts);
            }
            return null;
          }
        },
        TransactionReadyReplyMapper.SAME_FAILURE_TRANSFORMER,
        actorContext.getClientDispatcher());
  }
Beispiel #13
0
    @Override
    public Tuple2<GraphStageLogic, CompletionStage<A>> createLogicAndMaterializedValue(
        Attributes inheritedAttributes) {
      Promise<A> promise = Futures.promise();

      GraphStageLogic logic =
          new GraphStageLogic(shape) {
            {
              setHandler(
                  in,
                  new AbstractInHandler() {
                    @Override
                    public void onPush() {
                      A elem = grab(in);
                      promise.success(elem);
                      push(out, elem);

                      // replace handler with one just forwarding
                      setHandler(
                          in,
                          new AbstractInHandler() {
                            @Override
                            public void onPush() {
                              push(out, grab(in));
                            }
                          });
                    }
                  });

              setHandler(
                  out,
                  new AbstractOutHandler() {
                    @Override
                    public void onPull() {
                      pull(in);
                    }
                  });
            }
          };

      return new Tuple2(logic, promise.future());
    }
Beispiel #14
0
  @Override
  public F.Promise<Void> delete(String key) {
    Promise<Void> promise = Futures.promise();

    AmazonS3 amazonS3 = new AmazonS3Client(credentials);
    DeleteObjectRequest request = new DeleteObjectRequest(bucketName, key);
    request.withGeneralProgressListener(
        progressEvent -> {
          if (progressEvent.getEventType().isTransferEvent()) {
            if (progressEvent.getEventType().equals(ProgressEventType.TRANSFER_COMPLETED_EVENT)) {
              promise.success(null);
            } else if (progressEvent
                .getEventType()
                .equals(ProgressEventType.TRANSFER_FAILED_EVENT)) {
              promise.failure(new Exception("Delete failed"));
            }
          }
        });
    amazonS3.deleteObject(request);

    return F.Promise.wrap(promise.future());
  }
Beispiel #15
0
 public EmailSenderActor(EmailSender emailSender) {
   if (emailSender == null) {
     throw new IllegalArgumentException("emailSender must be defined.");
   }
   receive(
       ReceiveBuilder.match(
               EmailSenderMsg.class,
               msg -> {
                 try {
                   emailSender.send(
                       msg.to,
                       msg.cc,
                       msg.ci,
                       msg.object,
                       msg.content,
                       msg.htmlContent,
                       msg.attachments);
                 } catch (RuntimeException e) {
                   LOGGER.error(
                       "Unable to sent email to '{}', with subject '{}'",
                       Arrays.toString(msg.to.toArray()),
                       msg.object,
                       e);
                   sender().tell(Futures.failed(e), self());
                 }
                 if (LOGGER.isDebugEnabled()) {
                   LOGGER.debug(
                       "Email sent to '{}' with subject '{}'",
                       Arrays.toString(msg.to.toArray()),
                       msg.object);
                 }
                 getContext().stop(self());
               })
           .matchAny(this::unhandled)
           .build());
 }
 public Future<Integer> square(int i) {
   return Futures.successful(i * i);
 }
Beispiel #17
0
  public static void main(String[] args) throws Exception {

    ApplicationContext context = SpringApplication.run(App.class, args);

    ActorSystem system = context.getBean(ActorSystem.class);

    final LoggingAdapter log = Logging.getLogger(system, App.class.getSimpleName());

    log.info("Starting up");

    SpringExtension ext = context.getBean(SpringExtension.class);

    // Use the Spring Extension to create props for a named actor bean
    ActorRef calculator =
        system.actorOf(ext.props("calculator").withMailbox("akka.priority-mailbox"));

    /*
     * Create a completion service instance to await all futures
     */
    final ExecutionContext ec = system.dispatcher();

    Timeout timeout = new Timeout(120, TimeUnit.SECONDS);
    List<Long> ids = new ArrayList<>();

    ArrayList<Future<Object>> futures = new ArrayList<Future<Object>>();

    for (int i = 1; i <= 10; i++) {
      Future<Object> future = Patterns.ask(calculator, new CompilationRequest(i + " + 5"), timeout);
      future.onSuccess(
          new OnSuccess<Object>() {
            public void onSuccess(Object result) {
              if (result instanceof CompilationResult) {
                synchronized (ids) {
                  log.debug("Compilation result {} ", result.toString());
                  ids.add(((CompilationResult) result).getExpressionId());
                }
              } else {
                log.info("Compilation result is unknown type {} ", result.toString());
              }
            }
          },
          ec);
      futures.add(future);
    }

    Future<Iterable<Object>> seq = Futures.sequence(futures, ec);
    Await.result(seq, Duration.create(30, SECONDS));
    log.info("======================================================");
    log.info("Done waiting for compilations...{} ids", ids.size());
    log.info("======================================================");
    futures.clear();

    long start = System.nanoTime();
    List<Double> results = new ArrayList<>();

    Long count = 1_000_000L;
    for (long i = 1; i <= count; i++) {
      Future<Object> future =
          Patterns.ask(
              calculator,
              new CalculationRequest(i, ids.get((int) (i % ids.size())), null),
              timeout);
      future.onSuccess(
          new OnSuccess<Object>() {
            public void onSuccess(Object result) {
              if (result instanceof CalculationResult) {
                log.debug("Calculation result {} ", result.toString());
                //						synchronized(results)
                //						{
                //							results.add((Double) ((CalculationResult) result).getValue());
                //						}
              } else {
                log.info("Calculation result is unknown type {} ", result.toString());
              }
            }
          },
          ec);
      futures.add(future);
    }
    seq = Futures.sequence(futures, ec);
    Await.result(seq, Duration.create(600, SECONDS));
    calculator.tell(PoisonPill.getInstance(), null);

    while (!calculator.isTerminated()) {
      Thread.sleep(100);
    }

    long end = System.nanoTime();
    //        //int count = context.getBean(JdbcTemplate.class).queryForObject("SELECT COUNT(*) FROM
    // tasks", Integer.class);
    Long elapsed = TimeUnit.MILLISECONDS.convert((end - start), TimeUnit.NANOSECONDS);
    Double tps = count.doubleValue() / (elapsed.doubleValue() / Double.parseDouble("1000"));
    log.info("{} calculations in {}ms {}tps", count, elapsed, tps);
    log.info("Shutting down ------------------------------> {}", results.size());
    Thread.sleep(10000);
    system.shutdown();
    system.awaitTermination();
  }
Beispiel #18
0
 @Test
 public void usePipe() throws Exception {
   TestProbe probe = new TestProbe(system);
   pipe(Futures.successful("ho!"), system.dispatcher()).to(probe.ref());
   probe.expectMsg("ho!");
 }
Beispiel #19
0
 @Override
 public Future<Integer> square(int i) {
   return Futures.successful(cal(i));
 }
Beispiel #20
0
  public static void main(String[] args) {

    Outlet<Integer> outlet = null;

    Outlet<Integer> outlet1 = null;
    Outlet<Integer> outlet2 = null;

    Inlet<Integer> inlet = null;

    Inlet<Integer> inlet1 = null;
    Inlet<Integer> inlet2 = null;

    Flow<Integer, Integer, BoxedUnit> flow = Flow.of(Integer.class);
    Flow<Integer, Integer, BoxedUnit> flow1 = Flow.of(Integer.class);
    Flow<Integer, Integer, BoxedUnit> flow2 = Flow.of(Integer.class);

    Promise<Option<Integer>> promise = null;

    {
      Graph<SourceShape<Integer>, BoxedUnit> graphSource = null;
      Graph<SinkShape<Integer>, BoxedUnit> graphSink = null;
      Graph<FlowShape<Integer, Integer>, BoxedUnit> graphFlow = null;

      // #flow-wrap
      Source<Integer, BoxedUnit> source = Source.fromGraph(graphSource);
      Sink<Integer, BoxedUnit> sink = Sink.fromGraph(graphSink);
      Flow<Integer, Integer, BoxedUnit> aflow = Flow.fromGraph(graphFlow);
      Flow.fromSinkAndSource(Sink.<Integer>head(), Source.single(0));
      Flow.fromSinkAndSourceMat(Sink.<Integer>head(), Source.single(0), Keep.left());
      // #flow-wrap

      Graph<BidiShape<Integer, Integer, Integer, Integer>, BoxedUnit> bidiGraph = null;

      // #bidi-wrap
      BidiFlow<Integer, Integer, Integer, Integer, BoxedUnit> bidiFlow =
          BidiFlow.fromGraph(bidiGraph);
      BidiFlow.fromFlows(flow1, flow2);
      BidiFlow.fromFlowsMat(flow1, flow2, Keep.both());
      // #bidi-wrap

    }

    {
      // #graph-create
      GraphDSL.create(
          builder -> {
            // ...
            return ClosedShape.getInstance();
          });

      GraphDSL.create(
          builder -> {
            // ...
            return new FlowShape<>(inlet, outlet);
          });
      // #graph-create
    }

    {
      // #graph-create-2
      GraphDSL.create(
          builder -> {
            // ...
            return SourceShape.of(outlet);
          });

      GraphDSL.create(
          builder -> {
            // ...
            return SinkShape.of(inlet);
          });

      GraphDSL.create(
          builder -> {
            // ...
            return FlowShape.of(inlet, outlet);
          });

      GraphDSL.create(
          builder -> {
            // ...
            return BidiShape.of(inlet1, outlet1, inlet2, outlet2);
          });
      // #graph-create-2
    }

    {
      // #graph-builder
      GraphDSL.create(
          builder -> {
            builder.from(outlet).toInlet(inlet);
            builder.from(outlet).via(builder.add(flow)).toInlet(inlet);
            builder.from(builder.add(Source.single(0))).to(builder.add(Sink.head()));
            // ...
            return ClosedShape.getInstance();
          });
      // #graph-builder
    }

    // #source-creators
    Source<Integer, Promise<Option<Integer>>> src = Source.<Integer>maybe();
    // Complete the promise with an empty option to emulate the old lazyEmpty
    promise.trySuccess(scala.Option.empty());

    final Source<String, Cancellable> ticks =
        Source.tick(
            FiniteDuration.create(0, TimeUnit.MILLISECONDS),
            FiniteDuration.create(200, TimeUnit.MILLISECONDS),
            "tick");

    final Source<Integer, BoxedUnit> pubSource =
        Source.fromPublisher(TestPublisher.<Integer>manualProbe(true, sys));

    final Source<Integer, BoxedUnit> futSource = Source.fromFuture(Futures.successful(42));

    final Source<Integer, Subscriber<Integer>> subSource = Source.<Integer>asSubscriber();
    // #source-creators

    // #sink-creators
    final Sink<Integer, BoxedUnit> subSink =
        Sink.fromSubscriber(TestSubscriber.<Integer>manualProbe(sys));
    // #sink-creators

    // #sink-as-publisher
    final Sink<Integer, Publisher<Integer>> pubSink = Sink.<Integer>asPublisher(false);

    final Sink<Integer, Publisher<Integer>> pubSinkFanout = Sink.<Integer>asPublisher(true);
    // #sink-as-publisher

    // #empty-flow
    Flow<Integer, Integer, BoxedUnit> emptyFlow = Flow.<Integer>create();
    // or
    Flow<Integer, Integer, BoxedUnit> emptyFlow2 = Flow.of(Integer.class);
    // #empty-flow

    // #flatMapConcat
    Flow.<Source<Integer, BoxedUnit>>create()
        .<Integer, BoxedUnit>flatMapConcat(
            new Function<Source<Integer, BoxedUnit>, Source<Integer, BoxedUnit>>() {
              @Override
              public Source<Integer, BoxedUnit> apply(Source<Integer, BoxedUnit> param)
                  throws Exception {
                return param;
              }
            });
    // #flatMapConcat

    Uri uri = null;
    // #raw-query
    final akka.japi.Option<String> theRawQueryString = uri.rawQueryString();
    // #raw-query

    // #query-param
    final akka.japi.Option<String> aQueryParam = uri.query().get("a");
    // #query-param

    // #file-source-sink
    final Source<ByteString, Future<Long>> fileSrc = FileIO.fromFile(new File("."));

    final Source<ByteString, Future<Long>> otherFileSrc = FileIO.fromFile(new File("."), 1024);

    final Sink<ByteString, Future<Long>> fileSink = FileIO.toFile(new File("."));
    // #file-source-sink

    // #input-output-stream-source-sink
    final Source<ByteString, Future<java.lang.Long>> inputStreamSrc =
        StreamConverters.fromInputStream(
            new Creator<InputStream>() {
              public InputStream create() {
                return new SomeInputStream();
              }
            });

    final Source<ByteString, Future<java.lang.Long>> otherInputStreamSrc =
        StreamConverters.fromInputStream(
            new Creator<InputStream>() {
              public InputStream create() {
                return new SomeInputStream();
              }
            },
            1024);

    final Sink<ByteString, Future<java.lang.Long>> outputStreamSink =
        StreamConverters.fromOutputStream(
            new Creator<OutputStream>() {
              public OutputStream create() {
                return new SomeOutputStream();
              }
            });
    // #input-output-stream-source-sink

    // #output-input-stream-source-sink
    final FiniteDuration timeout = FiniteDuration.Zero();

    final Source<ByteString, OutputStream> outputStreamSrc = StreamConverters.asOutputStream();

    final Source<ByteString, OutputStream> otherOutputStreamSrc =
        StreamConverters.asOutputStream(timeout);

    final Sink<ByteString, InputStream> someInputStreamSink = StreamConverters.asInputStream();

    final Sink<ByteString, InputStream> someOtherInputStreamSink =
        StreamConverters.asInputStream(timeout);
    // #output-input-stream-source-sink

  }