/** * Entry point. All recent finished azkaban jobs' lineage. Will write to database stagging table * * @param timeFrame in minutes * @param endTimeStamp in millisecond * @throws Exception */ public void run(int timeFrame, long endTimeStamp) throws Exception { // get recent finished job AzJobChecker azJobChecker = new AzJobChecker(prop); List<AzkabanJobExecRecord> jobExecList = azJobChecker.getRecentFinishedJobFromFlow(timeFrame, endTimeStamp); azJobChecker.close(); logger.info("Total number of azkaban jobs : {}", jobExecList.size()); ActorSystem actorSystem = ActorSystem.create("LineageExtractor"); int numOfActor = Integer.valueOf(prop.getProperty(Constant.LINEAGE_ACTOR_NUM, "50")); ActorRef lineageExtractorActor = actorSystem.actorOf( Props.create(AzLineageExtractorActor.class) .withRouter(new SmallestMailboxRouter(numOfActor)), "lineageExtractorActor"); // initialize // AzkabanServiceCommunicator asc = new AzkabanServiceCommunicator(prop); HadoopNameNodeExtractor hnne = new HadoopNameNodeExtractor(prop); AzDbCommunicator adc = new AzDbCommunicator(prop); String wherehowsUrl = prop.getProperty(Constant.WH_DB_URL_KEY); String wherehowsUserName = prop.getProperty(Constant.WH_DB_USERNAME_KEY); String wherehowsPassWord = prop.getProperty(Constant.WH_DB_PASSWORD_KEY); String connUrl = wherehowsUrl + "?" + "user="******"&password="******"stg_job_execution_data_lineage"); AzLogParser.initialize(conn); PathAnalyzer.initialize(conn); int timeout = 30; // default 30 minutes for one job if (prop.containsKey(Constant.LINEAGE_ACTOR_TIMEOUT_KEY)) timeout = Integer.valueOf(prop.getProperty(Constant.LINEAGE_ACTOR_TIMEOUT_KEY)); List<Future<Object>> result = new ArrayList<>(); for (AzkabanJobExecRecord aje : jobExecList) { AzExecMessage message = new AzExecMessage(aje, prop); message.asc = null; message.hnne = hnne; message.adc = adc; message.databaseWriter = databaseWriter; message.connection = conn; Timeout t = new Timeout(timeout, TimeUnit.SECONDS); Future<Object> fut = Patterns.ask(lineageExtractorActor, message, t); result.add(fut); } // join all threads Future<Iterable<Object>> seq = Futures.sequence(result, actorSystem.dispatcher()); try { Await.result(seq, Duration.create(timeout + " seconds")); } catch (TimeoutException exception) { exception.printStackTrace(); } adc.close(); hnne.close(); databaseWriter.close(); logger.info("All job finished lineage collecting!"); }
@Test public void usePatternsAskPipe() { ActorSystem system = ActorSystem.create("MySystem"); ActorRef actorA = system.actorOf(new Props(MyUntypedActor.class)); ActorRef actorB = system.actorOf(new Props(MyUntypedActor.class)); ActorRef actorC = system.actorOf(new Props(MyUntypedActor.class)); // #ask-pipe final Timeout t = new Timeout(Duration.create(5, TimeUnit.SECONDS)); final ArrayList<Future<Object>> futures = new ArrayList<Future<Object>>(); futures.add(ask(actorA, "request", 1000)); // using 1000ms timeout futures.add(ask(actorB, "another request", t)); // using timeout from above final Future<Iterable<Object>> aggregate = Futures.sequence(futures, system.dispatcher()); final Future<Result> transformed = aggregate.map( new Mapper<Iterable<Object>, Result>() { public Result apply(Iterable<Object> coll) { final Iterator<Object> it = coll.iterator(); final String s = (String) it.next(); final int x = (Integer) it.next(); return new Result(x, s); } }, system.dispatcher()); pipe(transformed, system.dispatcher()).to(actorC); // #ask-pipe system.shutdown(); }
private Future<Iterable<Object>> invokeCohorts(Object message) { List<Future<Object>> futureList = Lists.newArrayListWithCapacity(cohorts.size()); for (ActorSelection cohort : cohorts) { if (LOG.isDebugEnabled()) { LOG.debug("Tx {}: Sending {} to cohort {}", transactionId, message, cohort); } futureList.add( actorContext.executeOperationAsync( cohort, message, actorContext.getTransactionCommitOperationTimeout())); } return Futures.sequence(futureList, actorContext.getClientDispatcher()); }
public static void main(String[] args) throws Exception { final ActorSystem system = system(); final ExecutionContextExecutor dispatcher = system.dispatcher(); Future<Long> fr = Futures.future(task, dispatcher); Future<Long> sc = Futures.future(task, dispatcher); Future<Long> th = Futures.future(task, dispatcher); Future<Long> fo = Futures.future(task, dispatcher); fr.onComplete(complete, dispatcher); sc.onComplete(complete, dispatcher); th.onComplete(complete, dispatcher); fo.onComplete(complete, dispatcher); Future<Iterable<Long>> sec = Futures.sequence(Arrays.asList(fr, sc, th, fo), dispatcher); Patterns.pipe(sec, dispatcher) .to(system.actorOf(Props.create(F.class))) .future() .ready(Duration.create(20, TimeUnit.SECONDS), null); Await.ready(system.terminate(), Duration.Inf()); }
private Future<Void> buildCohortList() { Future<Iterable<ActorSelection>> combinedFutures = Futures.sequence(cohortFutures, actorContext.getClientDispatcher()); return combinedFutures.transform( new AbstractFunction1<Iterable<ActorSelection>, Void>() { @Override public Void apply(Iterable<ActorSelection> actorSelections) { cohorts = Lists.newArrayList(actorSelections); if (LOG.isDebugEnabled()) { LOG.debug("Tx {} successfully built cohort path list: {}", transactionId, cohorts); } return null; } }, TransactionReadyReplyMapper.SAME_FAILURE_TRANSFORMER, actorContext.getClientDispatcher()); }
public static void main(String[] args) throws Exception { ApplicationContext context = SpringApplication.run(App.class, args); ActorSystem system = context.getBean(ActorSystem.class); final LoggingAdapter log = Logging.getLogger(system, App.class.getSimpleName()); log.info("Starting up"); SpringExtension ext = context.getBean(SpringExtension.class); // Use the Spring Extension to create props for a named actor bean ActorRef calculator = system.actorOf(ext.props("calculator").withMailbox("akka.priority-mailbox")); /* * Create a completion service instance to await all futures */ final ExecutionContext ec = system.dispatcher(); Timeout timeout = new Timeout(120, TimeUnit.SECONDS); List<Long> ids = new ArrayList<>(); ArrayList<Future<Object>> futures = new ArrayList<Future<Object>>(); for (int i = 1; i <= 10; i++) { Future<Object> future = Patterns.ask(calculator, new CompilationRequest(i + " + 5"), timeout); future.onSuccess( new OnSuccess<Object>() { public void onSuccess(Object result) { if (result instanceof CompilationResult) { synchronized (ids) { log.debug("Compilation result {} ", result.toString()); ids.add(((CompilationResult) result).getExpressionId()); } } else { log.info("Compilation result is unknown type {} ", result.toString()); } } }, ec); futures.add(future); } Future<Iterable<Object>> seq = Futures.sequence(futures, ec); Await.result(seq, Duration.create(30, SECONDS)); log.info("======================================================"); log.info("Done waiting for compilations...{} ids", ids.size()); log.info("======================================================"); futures.clear(); long start = System.nanoTime(); List<Double> results = new ArrayList<>(); Long count = 1_000_000L; for (long i = 1; i <= count; i++) { Future<Object> future = Patterns.ask( calculator, new CalculationRequest(i, ids.get((int) (i % ids.size())), null), timeout); future.onSuccess( new OnSuccess<Object>() { public void onSuccess(Object result) { if (result instanceof CalculationResult) { log.debug("Calculation result {} ", result.toString()); // synchronized(results) // { // results.add((Double) ((CalculationResult) result).getValue()); // } } else { log.info("Calculation result is unknown type {} ", result.toString()); } } }, ec); futures.add(future); } seq = Futures.sequence(futures, ec); Await.result(seq, Duration.create(600, SECONDS)); calculator.tell(PoisonPill.getInstance(), null); while (!calculator.isTerminated()) { Thread.sleep(100); } long end = System.nanoTime(); // //int count = context.getBean(JdbcTemplate.class).queryForObject("SELECT COUNT(*) FROM // tasks", Integer.class); Long elapsed = TimeUnit.MILLISECONDS.convert((end - start), TimeUnit.NANOSECONDS); Double tps = count.doubleValue() / (elapsed.doubleValue() / Double.parseDouble("1000")); log.info("{} calculations in {}ms {}tps", count, elapsed, tps); log.info("Shutting down ------------------------------> {}", results.size()); Thread.sleep(10000); system.shutdown(); system.awaitTermination(); }