public VertexManagerPluginDescriptor build() { VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create(ShuffleVertexManager.class.getName()); try { return desc.setUserPayload(TezUtils.createUserPayloadFromConf(this.conf)); } catch (IOException e) { throw new TezUncheckedException(e); } }
/** * Given two vertices and their respective configuration objects createEdge will create an Edge * object that connects the two. * * @param vConf JobConf of the first vertex * @param v The first vertex (source) * @param wConf JobConf of the second vertex * @param w The second vertex (sink) * @return */ public Edge createEdge(JobConf vConf, Vertex v, JobConf wConf, Vertex w, TezEdgeProperty edgeProp) throws IOException { updateConfigurationForEdge(vConf, v, wConf, w); if (edgeProp.getEdgeType() == EdgeType.CUSTOM_EDGE) { int numBuckets = edgeProp.getNumBuckets(); byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array(); VertexManagerPluginDescriptor desc = new VertexManagerPluginDescriptor(CustomPartitionVertex.class.getName()); desc.setUserPayload(userPayload); w.setVertexManagerPlugin(desc); } return new Edge(v, w, createEdgeProperty(edgeProp)); }
/** * Given a Vertex group and a vertex createEdge will create an Edge between them. * * @param group The parent VertexGroup * @param wConf The job conf of the child vertex * @param w The child vertex * @param edgeProp the edge property of connection between the two endpoints. */ public GroupInputEdge createEdge( VertexGroup group, JobConf wConf, Vertex w, TezEdgeProperty edgeProp) throws IOException { Class mergeInputClass; LOG.info("Creating Edge between " + group.getGroupName() + " and " + w.getVertexName()); w.getProcessorDescriptor().setUserPayload(MRHelpers.createUserPayloadFromConf(wConf)); EdgeType edgeType = edgeProp.getEdgeType(); switch (edgeType) { case BROADCAST_EDGE: mergeInputClass = ConcatenatedMergedKeyValueInput.class; break; case CUSTOM_EDGE: mergeInputClass = ConcatenatedMergedKeyValueInput.class; int numBuckets = edgeProp.getNumBuckets(); VertexManagerPluginDescriptor desc = new VertexManagerPluginDescriptor(CustomPartitionVertex.class.getName()); byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array(); desc.setUserPayload(userPayload); w.setVertexManagerPlugin(desc); break; case CUSTOM_SIMPLE_EDGE: mergeInputClass = ConcatenatedMergedKeyValueInput.class; break; case SIMPLE_EDGE: default: mergeInputClass = TezMergedLogicalInput.class; break; } return new GroupInputEdge( group, w, createEdgeProperty(edgeProp), new InputDescriptor(mergeInputClass.getName())); }