/**
   * Entry point. All recent finished azkaban jobs' lineage. Will write to database stagging table
   *
   * @param timeFrame in minutes
   * @param endTimeStamp in millisecond
   * @throws Exception
   */
  public void run(int timeFrame, long endTimeStamp) throws Exception {
    // get recent finished job
    AzJobChecker azJobChecker = new AzJobChecker(prop);
    List<AzkabanJobExecRecord> jobExecList =
        azJobChecker.getRecentFinishedJobFromFlow(timeFrame, endTimeStamp);
    azJobChecker.close();
    logger.info("Total number of azkaban jobs : {}", jobExecList.size());

    ActorSystem actorSystem = ActorSystem.create("LineageExtractor");
    int numOfActor = Integer.valueOf(prop.getProperty(Constant.LINEAGE_ACTOR_NUM, "50"));
    ActorRef lineageExtractorActor =
        actorSystem.actorOf(
            Props.create(AzLineageExtractorActor.class)
                .withRouter(new SmallestMailboxRouter(numOfActor)),
            "lineageExtractorActor");

    // initialize
    // AzkabanServiceCommunicator asc = new AzkabanServiceCommunicator(prop);
    HadoopNameNodeExtractor hnne = new HadoopNameNodeExtractor(prop);
    AzDbCommunicator adc = new AzDbCommunicator(prop);

    String wherehowsUrl = prop.getProperty(Constant.WH_DB_URL_KEY);
    String wherehowsUserName = prop.getProperty(Constant.WH_DB_USERNAME_KEY);
    String wherehowsPassWord = prop.getProperty(Constant.WH_DB_PASSWORD_KEY);
    String connUrl =
        wherehowsUrl + "?" + "user="******"&password="******"stg_job_execution_data_lineage");

    AzLogParser.initialize(conn);
    PathAnalyzer.initialize(conn);
    int timeout = 30; // default 30 minutes for one job
    if (prop.containsKey(Constant.LINEAGE_ACTOR_TIMEOUT_KEY))
      timeout = Integer.valueOf(prop.getProperty(Constant.LINEAGE_ACTOR_TIMEOUT_KEY));
    List<Future<Object>> result = new ArrayList<>();
    for (AzkabanJobExecRecord aje : jobExecList) {
      AzExecMessage message = new AzExecMessage(aje, prop);
      message.asc = null;
      message.hnne = hnne;
      message.adc = adc;
      message.databaseWriter = databaseWriter;
      message.connection = conn;
      Timeout t = new Timeout(timeout, TimeUnit.SECONDS);
      Future<Object> fut = Patterns.ask(lineageExtractorActor, message, t);
      result.add(fut);
    }

    // join all threads
    Future<Iterable<Object>> seq = Futures.sequence(result, actorSystem.dispatcher());
    try {
      Await.result(seq, Duration.create(timeout + " seconds"));
    } catch (TimeoutException exception) {
      exception.printStackTrace();
    }

    adc.close();
    hnne.close();
    databaseWriter.close();
    logger.info("All job finished lineage collecting!");
  }
  @Test
  public void usePatternsAskPipe() {
    ActorSystem system = ActorSystem.create("MySystem");
    ActorRef actorA = system.actorOf(new Props(MyUntypedActor.class));
    ActorRef actorB = system.actorOf(new Props(MyUntypedActor.class));
    ActorRef actorC = system.actorOf(new Props(MyUntypedActor.class));
    // #ask-pipe
    final Timeout t = new Timeout(Duration.create(5, TimeUnit.SECONDS));

    final ArrayList<Future<Object>> futures = new ArrayList<Future<Object>>();
    futures.add(ask(actorA, "request", 1000)); // using 1000ms timeout
    futures.add(ask(actorB, "another request", t)); // using timeout from above

    final Future<Iterable<Object>> aggregate = Futures.sequence(futures, system.dispatcher());

    final Future<Result> transformed =
        aggregate.map(
            new Mapper<Iterable<Object>, Result>() {
              public Result apply(Iterable<Object> coll) {
                final Iterator<Object> it = coll.iterator();
                final String s = (String) it.next();
                final int x = (Integer) it.next();
                return new Result(x, s);
              }
            },
            system.dispatcher());

    pipe(transformed, system.dispatcher()).to(actorC);
    // #ask-pipe
    system.shutdown();
  }
  private Future<Iterable<Object>> invokeCohorts(Object message) {
    List<Future<Object>> futureList = Lists.newArrayListWithCapacity(cohorts.size());
    for (ActorSelection cohort : cohorts) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Tx {}: Sending {} to cohort {}", transactionId, message, cohort);
      }
      futureList.add(
          actorContext.executeOperationAsync(
              cohort, message, actorContext.getTransactionCommitOperationTimeout()));
    }

    return Futures.sequence(futureList, actorContext.getClientDispatcher());
  }
Exemple #4
0
 public static void main(String[] args) throws Exception {
   final ActorSystem system = system();
   final ExecutionContextExecutor dispatcher = system.dispatcher();
   Future<Long> fr = Futures.future(task, dispatcher);
   Future<Long> sc = Futures.future(task, dispatcher);
   Future<Long> th = Futures.future(task, dispatcher);
   Future<Long> fo = Futures.future(task, dispatcher);
   fr.onComplete(complete, dispatcher);
   sc.onComplete(complete, dispatcher);
   th.onComplete(complete, dispatcher);
   fo.onComplete(complete, dispatcher);
   Future<Iterable<Long>> sec = Futures.sequence(Arrays.asList(fr, sc, th, fo), dispatcher);
   Patterns.pipe(sec, dispatcher)
       .to(system.actorOf(Props.create(F.class)))
       .future()
       .ready(Duration.create(20, TimeUnit.SECONDS), null);
   Await.ready(system.terminate(), Duration.Inf());
 }
  private Future<Void> buildCohortList() {

    Future<Iterable<ActorSelection>> combinedFutures =
        Futures.sequence(cohortFutures, actorContext.getClientDispatcher());

    return combinedFutures.transform(
        new AbstractFunction1<Iterable<ActorSelection>, Void>() {
          @Override
          public Void apply(Iterable<ActorSelection> actorSelections) {
            cohorts = Lists.newArrayList(actorSelections);
            if (LOG.isDebugEnabled()) {
              LOG.debug("Tx {} successfully built cohort path list: {}", transactionId, cohorts);
            }
            return null;
          }
        },
        TransactionReadyReplyMapper.SAME_FAILURE_TRANSFORMER,
        actorContext.getClientDispatcher());
  }
Exemple #6
0
  public static void main(String[] args) throws Exception {

    ApplicationContext context = SpringApplication.run(App.class, args);

    ActorSystem system = context.getBean(ActorSystem.class);

    final LoggingAdapter log = Logging.getLogger(system, App.class.getSimpleName());

    log.info("Starting up");

    SpringExtension ext = context.getBean(SpringExtension.class);

    // Use the Spring Extension to create props for a named actor bean
    ActorRef calculator =
        system.actorOf(ext.props("calculator").withMailbox("akka.priority-mailbox"));

    /*
     * Create a completion service instance to await all futures
     */
    final ExecutionContext ec = system.dispatcher();

    Timeout timeout = new Timeout(120, TimeUnit.SECONDS);
    List<Long> ids = new ArrayList<>();

    ArrayList<Future<Object>> futures = new ArrayList<Future<Object>>();

    for (int i = 1; i <= 10; i++) {
      Future<Object> future = Patterns.ask(calculator, new CompilationRequest(i + " + 5"), timeout);
      future.onSuccess(
          new OnSuccess<Object>() {
            public void onSuccess(Object result) {
              if (result instanceof CompilationResult) {
                synchronized (ids) {
                  log.debug("Compilation result {} ", result.toString());
                  ids.add(((CompilationResult) result).getExpressionId());
                }
              } else {
                log.info("Compilation result is unknown type {} ", result.toString());
              }
            }
          },
          ec);
      futures.add(future);
    }

    Future<Iterable<Object>> seq = Futures.sequence(futures, ec);
    Await.result(seq, Duration.create(30, SECONDS));
    log.info("======================================================");
    log.info("Done waiting for compilations...{} ids", ids.size());
    log.info("======================================================");
    futures.clear();

    long start = System.nanoTime();
    List<Double> results = new ArrayList<>();

    Long count = 1_000_000L;
    for (long i = 1; i <= count; i++) {
      Future<Object> future =
          Patterns.ask(
              calculator,
              new CalculationRequest(i, ids.get((int) (i % ids.size())), null),
              timeout);
      future.onSuccess(
          new OnSuccess<Object>() {
            public void onSuccess(Object result) {
              if (result instanceof CalculationResult) {
                log.debug("Calculation result {} ", result.toString());
                //						synchronized(results)
                //						{
                //							results.add((Double) ((CalculationResult) result).getValue());
                //						}
              } else {
                log.info("Calculation result is unknown type {} ", result.toString());
              }
            }
          },
          ec);
      futures.add(future);
    }
    seq = Futures.sequence(futures, ec);
    Await.result(seq, Duration.create(600, SECONDS));
    calculator.tell(PoisonPill.getInstance(), null);

    while (!calculator.isTerminated()) {
      Thread.sleep(100);
    }

    long end = System.nanoTime();
    //        //int count = context.getBean(JdbcTemplate.class).queryForObject("SELECT COUNT(*) FROM
    // tasks", Integer.class);
    Long elapsed = TimeUnit.MILLISECONDS.convert((end - start), TimeUnit.NANOSECONDS);
    Double tps = count.doubleValue() / (elapsed.doubleValue() / Double.parseDouble("1000"));
    log.info("{} calculations in {}ms {}tps", count, elapsed, tps);
    log.info("Shutting down ------------------------------> {}", results.size());
    Thread.sleep(10000);
    system.shutdown();
    system.awaitTermination();
  }