public VertexManagerPluginDescriptor build() {
VertexManagerPluginDescriptor desc =
VertexManagerPluginDescriptor.create(ShuffleVertexManager.class.getName());
try {
return desc.setUserPayload(TezUtils.createUserPayloadFromConf(this.conf));
} catch (IOException e) {
throw new TezUncheckedException(e);
}
}
/**
* Given two vertices and their respective configuration objects createEdge will create an Edge
* object that connects the two.
*
* @param vConf JobConf of the first vertex
* @param v The first vertex (source)
* @param wConf JobConf of the second vertex
* @param w The second vertex (sink)
* @return
*/
public Edge createEdge(JobConf vConf, Vertex v, JobConf wConf, Vertex w, TezEdgeProperty edgeProp)
throws IOException {
updateConfigurationForEdge(vConf, v, wConf, w);
if (edgeProp.getEdgeType() == EdgeType.CUSTOM_EDGE) {
int numBuckets = edgeProp.getNumBuckets();
byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array();
VertexManagerPluginDescriptor desc =
new VertexManagerPluginDescriptor(CustomPartitionVertex.class.getName());
desc.setUserPayload(userPayload);
w.setVertexManagerPlugin(desc);
}
return new Edge(v, w, createEdgeProperty(edgeProp));
}
/**
* Given a Vertex group and a vertex createEdge will create an Edge between them.
*
* @param group The parent VertexGroup
* @param wConf The job conf of the child vertex
* @param w The child vertex
* @param edgeProp the edge property of connection between the two endpoints.
*/
public GroupInputEdge createEdge(
VertexGroup group, JobConf wConf, Vertex w, TezEdgeProperty edgeProp) throws IOException {
Class mergeInputClass;
LOG.info("Creating Edge between " + group.getGroupName() + " and " + w.getVertexName());
w.getProcessorDescriptor().setUserPayload(MRHelpers.createUserPayloadFromConf(wConf));
EdgeType edgeType = edgeProp.getEdgeType();
switch (edgeType) {
case BROADCAST_EDGE:
mergeInputClass = ConcatenatedMergedKeyValueInput.class;
break;
case CUSTOM_EDGE:
mergeInputClass = ConcatenatedMergedKeyValueInput.class;
int numBuckets = edgeProp.getNumBuckets();
VertexManagerPluginDescriptor desc =
new VertexManagerPluginDescriptor(CustomPartitionVertex.class.getName());
byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array();
desc.setUserPayload(userPayload);
w.setVertexManagerPlugin(desc);
break;
case CUSTOM_SIMPLE_EDGE:
mergeInputClass = ConcatenatedMergedKeyValueInput.class;
break;
case SIMPLE_EDGE:
default:
mergeInputClass = TezMergedLogicalInput.class;
break;
}
return new GroupInputEdge(
group, w, createEdgeProperty(edgeProp), new InputDescriptor(mergeInputClass.getName()));
}