/**
* Entry point. All recent finished azkaban jobs' lineage. Will write to database stagging table
*
* @param timeFrame in minutes
* @param endTimeStamp in millisecond
* @throws Exception
*/
public void run(int timeFrame, long endTimeStamp) throws Exception {
// get recent finished job
AzJobChecker azJobChecker = new AzJobChecker(prop);
List<AzkabanJobExecRecord> jobExecList =
azJobChecker.getRecentFinishedJobFromFlow(timeFrame, endTimeStamp);
azJobChecker.close();
logger.info("Total number of azkaban jobs : {}", jobExecList.size());
ActorSystem actorSystem = ActorSystem.create("LineageExtractor");
int numOfActor = Integer.valueOf(prop.getProperty(Constant.LINEAGE_ACTOR_NUM, "50"));
ActorRef lineageExtractorActor =
actorSystem.actorOf(
Props.create(AzLineageExtractorActor.class)
.withRouter(new SmallestMailboxRouter(numOfActor)),
"lineageExtractorActor");
// initialize
// AzkabanServiceCommunicator asc = new AzkabanServiceCommunicator(prop);
HadoopNameNodeExtractor hnne = new HadoopNameNodeExtractor(prop);
AzDbCommunicator adc = new AzDbCommunicator(prop);
String wherehowsUrl = prop.getProperty(Constant.WH_DB_URL_KEY);
String wherehowsUserName = prop.getProperty(Constant.WH_DB_USERNAME_KEY);
String wherehowsPassWord = prop.getProperty(Constant.WH_DB_PASSWORD_KEY);
String connUrl =
wherehowsUrl + "?" + "user="******"&password="******"stg_job_execution_data_lineage");
AzLogParser.initialize(conn);
PathAnalyzer.initialize(conn);
int timeout = 30; // default 30 minutes for one job
if (prop.containsKey(Constant.LINEAGE_ACTOR_TIMEOUT_KEY))
timeout = Integer.valueOf(prop.getProperty(Constant.LINEAGE_ACTOR_TIMEOUT_KEY));
List<Future<Object>> result = new ArrayList<>();
for (AzkabanJobExecRecord aje : jobExecList) {
AzExecMessage message = new AzExecMessage(aje, prop);
message.asc = null;
message.hnne = hnne;
message.adc = adc;
message.databaseWriter = databaseWriter;
message.connection = conn;
Timeout t = new Timeout(timeout, TimeUnit.SECONDS);
Future<Object> fut = Patterns.ask(lineageExtractorActor, message, t);
result.add(fut);
}
// join all threads
Future<Iterable<Object>> seq = Futures.sequence(result, actorSystem.dispatcher());
try {
Await.result(seq, Duration.create(timeout + " seconds"));
} catch (TimeoutException exception) {
exception.printStackTrace();
}
adc.close();
hnne.close();
databaseWriter.close();
logger.info("All job finished lineage collecting!");
}
@Test
public void usePatternsAskPipe() {
ActorSystem system = ActorSystem.create("MySystem");
ActorRef actorA = system.actorOf(new Props(MyUntypedActor.class));
ActorRef actorB = system.actorOf(new Props(MyUntypedActor.class));
ActorRef actorC = system.actorOf(new Props(MyUntypedActor.class));
// #ask-pipe
final Timeout t = new Timeout(Duration.create(5, TimeUnit.SECONDS));
final ArrayList<Future<Object>> futures = new ArrayList<Future<Object>>();
futures.add(ask(actorA, "request", 1000)); // using 1000ms timeout
futures.add(ask(actorB, "another request", t)); // using timeout from above
final Future<Iterable<Object>> aggregate = Futures.sequence(futures, system.dispatcher());
final Future<Result> transformed =
aggregate.map(
new Mapper<Iterable<Object>, Result>() {
public Result apply(Iterable<Object> coll) {
final Iterator<Object> it = coll.iterator();
final String s = (String) it.next();
final int x = (Integer) it.next();
return new Result(x, s);
}
},
system.dispatcher());
pipe(transformed, system.dispatcher()).to(actorC);
// #ask-pipe
system.shutdown();
}
private Future<Iterable<Object>> invokeCohorts(Object message) {
List<Future<Object>> futureList = Lists.newArrayListWithCapacity(cohorts.size());
for (ActorSelection cohort : cohorts) {
if (LOG.isDebugEnabled()) {
LOG.debug("Tx {}: Sending {} to cohort {}", transactionId, message, cohort);
}
futureList.add(
actorContext.executeOperationAsync(
cohort, message, actorContext.getTransactionCommitOperationTimeout()));
}
return Futures.sequence(futureList, actorContext.getClientDispatcher());
}
public static void main(String[] args) throws Exception {
final ActorSystem system = system();
final ExecutionContextExecutor dispatcher = system.dispatcher();
Future<Long> fr = Futures.future(task, dispatcher);
Future<Long> sc = Futures.future(task, dispatcher);
Future<Long> th = Futures.future(task, dispatcher);
Future<Long> fo = Futures.future(task, dispatcher);
fr.onComplete(complete, dispatcher);
sc.onComplete(complete, dispatcher);
th.onComplete(complete, dispatcher);
fo.onComplete(complete, dispatcher);
Future<Iterable<Long>> sec = Futures.sequence(Arrays.asList(fr, sc, th, fo), dispatcher);
Patterns.pipe(sec, dispatcher)
.to(system.actorOf(Props.create(F.class)))
.future()
.ready(Duration.create(20, TimeUnit.SECONDS), null);
Await.ready(system.terminate(), Duration.Inf());
}
private Future<Void> buildCohortList() {
Future<Iterable<ActorSelection>> combinedFutures =
Futures.sequence(cohortFutures, actorContext.getClientDispatcher());
return combinedFutures.transform(
new AbstractFunction1<Iterable<ActorSelection>, Void>() {
@Override
public Void apply(Iterable<ActorSelection> actorSelections) {
cohorts = Lists.newArrayList(actorSelections);
if (LOG.isDebugEnabled()) {
LOG.debug("Tx {} successfully built cohort path list: {}", transactionId, cohorts);
}
return null;
}
},
TransactionReadyReplyMapper.SAME_FAILURE_TRANSFORMER,
actorContext.getClientDispatcher());
}
public static void main(String[] args) throws Exception {
ApplicationContext context = SpringApplication.run(App.class, args);
ActorSystem system = context.getBean(ActorSystem.class);
final LoggingAdapter log = Logging.getLogger(system, App.class.getSimpleName());
log.info("Starting up");
SpringExtension ext = context.getBean(SpringExtension.class);
// Use the Spring Extension to create props for a named actor bean
ActorRef calculator =
system.actorOf(ext.props("calculator").withMailbox("akka.priority-mailbox"));
/*
* Create a completion service instance to await all futures
*/
final ExecutionContext ec = system.dispatcher();
Timeout timeout = new Timeout(120, TimeUnit.SECONDS);
List<Long> ids = new ArrayList<>();
ArrayList<Future<Object>> futures = new ArrayList<Future<Object>>();
for (int i = 1; i <= 10; i++) {
Future<Object> future = Patterns.ask(calculator, new CompilationRequest(i + " + 5"), timeout);
future.onSuccess(
new OnSuccess<Object>() {
public void onSuccess(Object result) {
if (result instanceof CompilationResult) {
synchronized (ids) {
log.debug("Compilation result {} ", result.toString());
ids.add(((CompilationResult) result).getExpressionId());
}
} else {
log.info("Compilation result is unknown type {} ", result.toString());
}
}
},
ec);
futures.add(future);
}
Future<Iterable<Object>> seq = Futures.sequence(futures, ec);
Await.result(seq, Duration.create(30, SECONDS));
log.info("======================================================");
log.info("Done waiting for compilations...{} ids", ids.size());
log.info("======================================================");
futures.clear();
long start = System.nanoTime();
List<Double> results = new ArrayList<>();
Long count = 1_000_000L;
for (long i = 1; i <= count; i++) {
Future<Object> future =
Patterns.ask(
calculator,
new CalculationRequest(i, ids.get((int) (i % ids.size())), null),
timeout);
future.onSuccess(
new OnSuccess<Object>() {
public void onSuccess(Object result) {
if (result instanceof CalculationResult) {
log.debug("Calculation result {} ", result.toString());
// synchronized(results)
// {
// results.add((Double) ((CalculationResult) result).getValue());
// }
} else {
log.info("Calculation result is unknown type {} ", result.toString());
}
}
},
ec);
futures.add(future);
}
seq = Futures.sequence(futures, ec);
Await.result(seq, Duration.create(600, SECONDS));
calculator.tell(PoisonPill.getInstance(), null);
while (!calculator.isTerminated()) {
Thread.sleep(100);
}
long end = System.nanoTime();
// //int count = context.getBean(JdbcTemplate.class).queryForObject("SELECT COUNT(*) FROM
// tasks", Integer.class);
Long elapsed = TimeUnit.MILLISECONDS.convert((end - start), TimeUnit.NANOSECONDS);
Double tps = count.doubleValue() / (elapsed.doubleValue() / Double.parseDouble("1000"));
log.info("{} calculations in {}ms {}tps", count, elapsed, tps);
log.info("Shutting down ------------------------------> {}", results.size());
Thread.sleep(10000);
system.shutdown();
system.awaitTermination();
}
