private DAGPlan createDAG() {
// Create a plan with 3 vertices: A, B, C. Group(A,B)->C
Configuration conf = new Configuration(false);
int dummyTaskCount = 1;
Resource dummyTaskResource = Resource.newInstance(1, 1);
org.apache.tez.dag.api.Vertex v1 =
new org.apache.tez.dag.api.Vertex(
"vertex1",
new ProcessorDescriptor("Processor").setHistoryText("vertex1 Processor HistoryText"),
dummyTaskCount,
dummyTaskResource);
v1.addInput(
"input1", new InputDescriptor("input.class").setHistoryText("input HistoryText"), null);
org.apache.tez.dag.api.Vertex v2 =
new org.apache.tez.dag.api.Vertex(
"vertex2",
new ProcessorDescriptor("Processor").setHistoryText("vertex2 Processor HistoryText"),
dummyTaskCount,
dummyTaskResource);
org.apache.tez.dag.api.Vertex v3 =
new org.apache.tez.dag.api.Vertex(
"vertex3",
new ProcessorDescriptor("Processor").setHistoryText("vertex3 Processor HistoryText"),
dummyTaskCount,
dummyTaskResource);
DAG dag = new DAG("testDag");
String groupName1 = "uv12";
org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
OutputDescriptor outDesc =
new OutputDescriptor("output.class").setHistoryText("uvOut HistoryText");
uv12.addOutput("uvOut", outDesc, OutputCommitter.class);
v3.addOutput("uvOut", outDesc, OutputCommitter.class);
GroupInputEdge e1 =
new GroupInputEdge(
uv12,
v3,
new EdgeProperty(
DataMovementType.SCATTER_GATHER,
DataSourceType.PERSISTED,
SchedulingType.SEQUENTIAL,
new OutputDescriptor("dummy output class").setHistoryText("Dummy History Text"),
new InputDescriptor("dummy input class").setHistoryText("Dummy History Text")),
new InputDescriptor("merge.class").setHistoryText("Merge HistoryText"));
dag.addVertex(v1);
dag.addVertex(v2);
dag.addVertex(v3);
dag.addEdge(e1);
return dag.createDag(conf);
}
/**
* Submit a DAG to a Tez Session. Blocks until either the DAG is submitted to the session or
* configured timeout period expires. Cleans up session if the submission timed out.
*
* @param dag DAG to be submitted to Session
* @return DAGClient to monitor the DAG
* @throws TezException
* @throws IOException
* @throws SessionNotRunning if session is not alive
* @throws DAGSubmissionTimedOut if submission timed out
*/
public synchronized DAGClient submitDAG(DAG dag) throws TezException, IOException {
if (!sessionStarted) {
throw new TezUncheckedException("Session not started");
} else if (sessionStopped) {
throw new TezUncheckedException("Session stopped");
}
String dagId = null;
LOG.info(
"Submitting dag to TezSession"
+ ", sessionName="
+ sessionName
+ ", applicationId="
+ applicationId);
// Add tez jars to vertices too
for (Vertex v : dag.getVertices()) {
v.getTaskLocalResources().putAll(tezJarResources);
if (null != tezConfPBLRsrc) {
v.getTaskLocalResources().put(TezConfiguration.TEZ_PB_BINARY_CONF_NAME, tezConfPBLRsrc);
}
}
DAGPlan dagPlan = dag.createDag(sessionConfig.getTezConfiguration());
SubmitDAGRequestProto requestProto =
SubmitDAGRequestProto.newBuilder().setDAGPlan(dagPlan).build();
DAGClientAMProtocolBlockingPB proxy;
long startTime = System.currentTimeMillis();
int timeout =
sessionConfig
.getTezConfiguration()
.getInt(
TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS,
TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT);
long endTime = startTime + (timeout * 1000);
while (true) {
// FIXME implement a max time to wait for submit
proxy =
TezClientUtils.getSessionAMProxy(
yarnClient, sessionConfig.getYarnConfiguration(), applicationId);
if (proxy != null) {
break;
}
try {
Thread.sleep(100l);
} catch (InterruptedException e) {
// Ignore
}
if (System.currentTimeMillis() > endTime) {
try {
LOG.warn("DAG submission to session timed out, stopping session");
stop();
} catch (Throwable t) {
LOG.info("Got an exception when trying to stop session", t);
}
throw new DAGSubmissionTimedOut(
"Could not submit DAG to Tez Session" + ", timed out after " + timeout + " seconds");
}
}
try {
dagId = proxy.submitDAG(null, requestProto).getDagId();
} catch (ServiceException e) {
throw new TezException(e);
}
LOG.info(
"Submitted dag to TezSession"
+ ", sessionName="
+ sessionName
+ ", applicationId="
+ applicationId
+ ", dagId="
+ dagId);
return new DAGClientRPCImpl(applicationId, dagId, sessionConfig.getTezConfiguration());
}