public DAG buildDAG(TezPlanContainerNode tezPlanNode, Map<String, LocalResource> localResources)
throws IOException, YarnException {
DAG tezDag = DAG.create(tezPlanNode.getOperatorKey().toString());
tezDag.setCredentials(tezPlanNode.getTezOperPlan().getCredentials());
TezDagBuilder dagBuilder =
new TezDagBuilder(pigContext, tezPlanNode.getTezOperPlan(), tezDag, localResources);
dagBuilder.visit();
dagBuilder.avoidContainerReuseIfInputSplitInDisk();
return tezDag;
}
private TezJob getJob(TezPlanContainerNode tezPlanNode, TezPlanContainer planContainer)
throws JobCreationException {
try {
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
localResources.putAll(planContainer.getLocalResources());
TezOperPlan tezPlan = tezPlanNode.getTezOperPlan();
localResources.putAll(tezPlan.getExtraResources());
String shipFiles = pigContext.getProperties().getProperty("pig.streaming.ship.files");
if (shipFiles != null) {
for (String file : shipFiles.split(",")) {
TezResourceManager.getInstance().addTezResource(new File(file.trim()).toURI());
}
}
String cacheFiles = pigContext.getProperties().getProperty("pig.streaming.cache.files");
if (cacheFiles != null) {
addCacheResources(cacheFiles.split(","));
}
for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) {
log.info("Local resource: " + entry.getKey());
}
DAG tezDag = buildDAG(tezPlanNode, localResources);
tezDag.setDAGInfo(createDagInfo(TezScriptState.get().getScript()));
// set Tez caller context
// Reflection for the following code since it is only available since tez 0.8.1:
// CallerContext context = CallerContext.create(ATSService.CallerContext,
// ATSService.getPigAuditId(pigContext),
// ATSService.EntityType, "");
// tezDag.setCallerContext(context);
Class callerContextClass = null;
try {
callerContextClass = Class.forName("org.apache.tez.client.CallerContext");
} catch (ClassNotFoundException e) {
// If pre-Tez 0.8.1, skip setting CallerContext
}
if (callerContextClass != null) {
Method builderBuildMethod =
callerContextClass.getMethod(
"create", String.class, String.class, String.class, String.class);
Object context =
builderBuildMethod.invoke(
null,
PigATSClient.CALLER_CONTEXT,
PigATSClient.getPigAuditId(pigContext),
PigATSClient.ENTITY_TYPE,
"");
Method dagSetCallerContext =
tezDag.getClass().getMethod("setCallerContext", context.getClass());
dagSetCallerContext.invoke(tezDag, context);
}
log.info("Total estimated parallelism is " + tezPlan.getEstimatedTotalParallelism());
return new TezJob(tezConf, tezDag, localResources, tezPlan);
} catch (Exception e) {
int errCode = 2017;
String msg = "Internal error creating job configuration.";
throw new JobCreationException(msg, errCode, PigException.BUG, e);
}
}
