public DAG buildDAG(TezPlanContainerNode tezPlanNode, Map<String, LocalResource> localResources) throws IOException, YarnException { DAG tezDag = DAG.create(tezPlanNode.getOperatorKey().toString()); tezDag.setCredentials(tezPlanNode.getTezOperPlan().getCredentials()); TezDagBuilder dagBuilder = new TezDagBuilder(pigContext, tezPlanNode.getTezOperPlan(), tezDag, localResources); dagBuilder.visit(); dagBuilder.avoidContainerReuseIfInputSplitInDisk(); return tezDag; }
private TezJob getJob(TezPlanContainerNode tezPlanNode, TezPlanContainer planContainer) throws JobCreationException { try { Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); localResources.putAll(planContainer.getLocalResources()); TezOperPlan tezPlan = tezPlanNode.getTezOperPlan(); localResources.putAll(tezPlan.getExtraResources()); String shipFiles = pigContext.getProperties().getProperty("pig.streaming.ship.files"); if (shipFiles != null) { for (String file : shipFiles.split(",")) { TezResourceManager.getInstance().addTezResource(new File(file.trim()).toURI()); } } String cacheFiles = pigContext.getProperties().getProperty("pig.streaming.cache.files"); if (cacheFiles != null) { addCacheResources(cacheFiles.split(",")); } for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) { log.info("Local resource: " + entry.getKey()); } DAG tezDag = buildDAG(tezPlanNode, localResources); tezDag.setDAGInfo(createDagInfo(TezScriptState.get().getScript())); // set Tez caller context // Reflection for the following code since it is only available since tez 0.8.1: // CallerContext context = CallerContext.create(ATSService.CallerContext, // ATSService.getPigAuditId(pigContext), // ATSService.EntityType, ""); // tezDag.setCallerContext(context); Class callerContextClass = null; try { callerContextClass = Class.forName("org.apache.tez.client.CallerContext"); } catch (ClassNotFoundException e) { // If pre-Tez 0.8.1, skip setting CallerContext } if (callerContextClass != null) { Method builderBuildMethod = callerContextClass.getMethod( "create", String.class, String.class, String.class, String.class); Object context = builderBuildMethod.invoke( null, PigATSClient.CALLER_CONTEXT, PigATSClient.getPigAuditId(pigContext), PigATSClient.ENTITY_TYPE, ""); Method dagSetCallerContext = tezDag.getClass().getMethod("setCallerContext", context.getClass()); dagSetCallerContext.invoke(tezDag, context); } log.info("Total estimated parallelism is " + tezPlan.getEstimatedTotalParallelism()); return new TezJob(tezConf, tezDag, localResources, tezPlan); } catch (Exception e) { int errCode = 2017; String msg = "Internal error creating job configuration."; throw new JobCreationException(msg, errCode, PigException.BUG, e); } }