private DAGPlan createDAG() { // Create a plan with 3 vertices: A, B, C. Group(A,B)->C Configuration conf = new Configuration(false); int dummyTaskCount = 1; Resource dummyTaskResource = Resource.newInstance(1, 1); org.apache.tez.dag.api.Vertex v1 = new org.apache.tez.dag.api.Vertex( "vertex1", new ProcessorDescriptor("Processor").setHistoryText("vertex1 Processor HistoryText"), dummyTaskCount, dummyTaskResource); v1.addInput( "input1", new InputDescriptor("input.class").setHistoryText("input HistoryText"), null); org.apache.tez.dag.api.Vertex v2 = new org.apache.tez.dag.api.Vertex( "vertex2", new ProcessorDescriptor("Processor").setHistoryText("vertex2 Processor HistoryText"), dummyTaskCount, dummyTaskResource); org.apache.tez.dag.api.Vertex v3 = new org.apache.tez.dag.api.Vertex( "vertex3", new ProcessorDescriptor("Processor").setHistoryText("vertex3 Processor HistoryText"), dummyTaskCount, dummyTaskResource); DAG dag = new DAG("testDag"); String groupName1 = "uv12"; org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2); OutputDescriptor outDesc = new OutputDescriptor("output.class").setHistoryText("uvOut HistoryText"); uv12.addOutput("uvOut", outDesc, OutputCommitter.class); v3.addOutput("uvOut", outDesc, OutputCommitter.class); GroupInputEdge e1 = new GroupInputEdge( uv12, v3, new EdgeProperty( DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, new OutputDescriptor("dummy output class").setHistoryText("Dummy History Text"), new InputDescriptor("dummy input class").setHistoryText("Dummy History Text")), new InputDescriptor("merge.class").setHistoryText("Merge HistoryText")); dag.addVertex(v1); dag.addVertex(v2); dag.addVertex(v3); dag.addEdge(e1); return dag.createDag(conf); }
/** * Submit a DAG to a Tez Session. Blocks until either the DAG is submitted to the session or * configured timeout period expires. Cleans up session if the submission timed out. * * @param dag DAG to be submitted to Session * @return DAGClient to monitor the DAG * @throws TezException * @throws IOException * @throws SessionNotRunning if session is not alive * @throws DAGSubmissionTimedOut if submission timed out */ public synchronized DAGClient submitDAG(DAG dag) throws TezException, IOException { if (!sessionStarted) { throw new TezUncheckedException("Session not started"); } else if (sessionStopped) { throw new TezUncheckedException("Session stopped"); } String dagId = null; LOG.info( "Submitting dag to TezSession" + ", sessionName=" + sessionName + ", applicationId=" + applicationId); // Add tez jars to vertices too for (Vertex v : dag.getVertices()) { v.getTaskLocalResources().putAll(tezJarResources); if (null != tezConfPBLRsrc) { v.getTaskLocalResources().put(TezConfiguration.TEZ_PB_BINARY_CONF_NAME, tezConfPBLRsrc); } } DAGPlan dagPlan = dag.createDag(sessionConfig.getTezConfiguration()); SubmitDAGRequestProto requestProto = SubmitDAGRequestProto.newBuilder().setDAGPlan(dagPlan).build(); DAGClientAMProtocolBlockingPB proxy; long startTime = System.currentTimeMillis(); int timeout = sessionConfig .getTezConfiguration() .getInt( TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS, TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT); long endTime = startTime + (timeout * 1000); while (true) { // FIXME implement a max time to wait for submit proxy = TezClientUtils.getSessionAMProxy( yarnClient, sessionConfig.getYarnConfiguration(), applicationId); if (proxy != null) { break; } try { Thread.sleep(100l); } catch (InterruptedException e) { // Ignore } if (System.currentTimeMillis() > endTime) { try { LOG.warn("DAG submission to session timed out, stopping session"); stop(); } catch (Throwable t) { LOG.info("Got an exception when trying to stop session", t); } throw new DAGSubmissionTimedOut( "Could not submit DAG to Tez Session" + ", timed out after " + timeout + " seconds"); } } try { dagId = proxy.submitDAG(null, requestProto).getDagId(); } catch (ServiceException e) { throw new TezException(e); } LOG.info( "Submitted dag to TezSession" + ", sessionName=" + sessionName + ", applicationId=" + applicationId + ", dagId=" + dagId); return new DAGClientRPCImpl(applicationId, dagId, sessionConfig.getTezConfiguration()); }