/** * Given two vertices and their respective configuration objects createEdge will create an Edge * object that connects the two. * * @param vConf JobConf of the first vertex * @param v The first vertex (source) * @param wConf JobConf of the second vertex * @param w The second vertex (sink) * @return */ public Edge createEdge(JobConf vConf, Vertex v, JobConf wConf, Vertex w, TezEdgeProperty edgeProp) throws IOException { updateConfigurationForEdge(vConf, v, wConf, w); if (edgeProp.getEdgeType() == EdgeType.CUSTOM_EDGE) { int numBuckets = edgeProp.getNumBuckets(); byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array(); VertexManagerPluginDescriptor desc = new VertexManagerPluginDescriptor(CustomPartitionVertex.class.getName()); desc.setUserPayload(userPayload); w.setVertexManagerPlugin(desc); } return new Edge(v, w, createEdgeProperty(edgeProp)); }
/** * Given a Vertex group and a vertex createEdge will create an Edge between them. * * @param group The parent VertexGroup * @param wConf The job conf of the child vertex * @param w The child vertex * @param edgeProp the edge property of connection between the two endpoints. */ public GroupInputEdge createEdge( VertexGroup group, JobConf wConf, Vertex w, TezEdgeProperty edgeProp) throws IOException { Class mergeInputClass; LOG.info("Creating Edge between " + group.getGroupName() + " and " + w.getVertexName()); w.getProcessorDescriptor().setUserPayload(MRHelpers.createUserPayloadFromConf(wConf)); EdgeType edgeType = edgeProp.getEdgeType(); switch (edgeType) { case BROADCAST_EDGE: mergeInputClass = ConcatenatedMergedKeyValueInput.class; break; case CUSTOM_EDGE: mergeInputClass = ConcatenatedMergedKeyValueInput.class; int numBuckets = edgeProp.getNumBuckets(); VertexManagerPluginDescriptor desc = new VertexManagerPluginDescriptor(CustomPartitionVertex.class.getName()); byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array(); desc.setUserPayload(userPayload); w.setVertexManagerPlugin(desc); break; case CUSTOM_SIMPLE_EDGE: mergeInputClass = ConcatenatedMergedKeyValueInput.class; break; case SIMPLE_EDGE: default: mergeInputClass = TezMergedLogicalInput.class; break; } return new GroupInputEdge( group, w, createEdgeProperty(edgeProp), new InputDescriptor(mergeInputClass.getName())); }
/* * Helper function to create an edge property from an edge type. */ private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp) throws IOException { DataMovementType dataMovementType; Class logicalInputClass; Class logicalOutputClass; EdgeProperty edgeProperty = null; EdgeType edgeType = edgeProp.getEdgeType(); switch (edgeType) { case BROADCAST_EDGE: dataMovementType = DataMovementType.BROADCAST; logicalOutputClass = OnFileUnorderedKVOutput.class; logicalInputClass = ShuffledUnorderedKVInput.class; break; case CUSTOM_EDGE: dataMovementType = DataMovementType.CUSTOM; logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class; logicalInputClass = ShuffledUnorderedKVInput.class; EdgeManagerDescriptor edgeDesc = new EdgeManagerDescriptor(CustomPartitionEdge.class.getName()); CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null); DataOutputBuffer dob = new DataOutputBuffer(); edgeConf.write(dob); byte[] userPayload = dob.getData(); edgeDesc.setUserPayload(userPayload); edgeProperty = new EdgeProperty( edgeDesc, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, new OutputDescriptor(logicalOutputClass.getName()), new InputDescriptor(logicalInputClass.getName())); break; case CUSTOM_SIMPLE_EDGE: dataMovementType = DataMovementType.SCATTER_GATHER; logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class; logicalInputClass = ShuffledUnorderedKVInput.class; break; case SIMPLE_EDGE: default: dataMovementType = DataMovementType.SCATTER_GATHER; logicalOutputClass = OnFileSortedOutput.class; logicalInputClass = ShuffledMergedInputLegacy.class; break; } if (edgeProperty == null) { edgeProperty = new EdgeProperty( dataMovementType, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, new OutputDescriptor(logicalOutputClass.getName()), new InputDescriptor(logicalInputClass.getName())); } return edgeProperty; }