/**
* Given two vertices and their respective configuration objects createEdge will create an Edge
* object that connects the two.
*
* @param vConf JobConf of the first vertex
* @param v The first vertex (source)
* @param wConf JobConf of the second vertex
* @param w The second vertex (sink)
* @return
*/
public Edge createEdge(JobConf vConf, Vertex v, JobConf wConf, Vertex w, TezEdgeProperty edgeProp)
throws IOException {
updateConfigurationForEdge(vConf, v, wConf, w);
if (edgeProp.getEdgeType() == EdgeType.CUSTOM_EDGE) {
int numBuckets = edgeProp.getNumBuckets();
byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array();
VertexManagerPluginDescriptor desc =
new VertexManagerPluginDescriptor(CustomPartitionVertex.class.getName());
desc.setUserPayload(userPayload);
w.setVertexManagerPlugin(desc);
}
return new Edge(v, w, createEdgeProperty(edgeProp));
}
/**
* Given a Vertex group and a vertex createEdge will create an Edge between them.
*
* @param group The parent VertexGroup
* @param wConf The job conf of the child vertex
* @param w The child vertex
* @param edgeProp the edge property of connection between the two endpoints.
*/
public GroupInputEdge createEdge(
VertexGroup group, JobConf wConf, Vertex w, TezEdgeProperty edgeProp) throws IOException {
Class mergeInputClass;
LOG.info("Creating Edge between " + group.getGroupName() + " and " + w.getVertexName());
w.getProcessorDescriptor().setUserPayload(MRHelpers.createUserPayloadFromConf(wConf));
EdgeType edgeType = edgeProp.getEdgeType();
switch (edgeType) {
case BROADCAST_EDGE:
mergeInputClass = ConcatenatedMergedKeyValueInput.class;
break;
case CUSTOM_EDGE:
mergeInputClass = ConcatenatedMergedKeyValueInput.class;
int numBuckets = edgeProp.getNumBuckets();
VertexManagerPluginDescriptor desc =
new VertexManagerPluginDescriptor(CustomPartitionVertex.class.getName());
byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array();
desc.setUserPayload(userPayload);
w.setVertexManagerPlugin(desc);
break;
case CUSTOM_SIMPLE_EDGE:
mergeInputClass = ConcatenatedMergedKeyValueInput.class;
break;
case SIMPLE_EDGE:
default:
mergeInputClass = TezMergedLogicalInput.class;
break;
}
return new GroupInputEdge(
group, w, createEdgeProperty(edgeProp), new InputDescriptor(mergeInputClass.getName()));
}
/*
* Helper function to create an edge property from an edge type.
*/
private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp) throws IOException {
DataMovementType dataMovementType;
Class logicalInputClass;
Class logicalOutputClass;
EdgeProperty edgeProperty = null;
EdgeType edgeType = edgeProp.getEdgeType();
switch (edgeType) {
case BROADCAST_EDGE:
dataMovementType = DataMovementType.BROADCAST;
logicalOutputClass = OnFileUnorderedKVOutput.class;
logicalInputClass = ShuffledUnorderedKVInput.class;
break;
case CUSTOM_EDGE:
dataMovementType = DataMovementType.CUSTOM;
logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class;
logicalInputClass = ShuffledUnorderedKVInput.class;
EdgeManagerDescriptor edgeDesc =
new EdgeManagerDescriptor(CustomPartitionEdge.class.getName());
CustomEdgeConfiguration edgeConf =
new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null);
DataOutputBuffer dob = new DataOutputBuffer();
edgeConf.write(dob);
byte[] userPayload = dob.getData();
edgeDesc.setUserPayload(userPayload);
edgeProperty =
new EdgeProperty(
edgeDesc,
DataSourceType.PERSISTED,
SchedulingType.SEQUENTIAL,
new OutputDescriptor(logicalOutputClass.getName()),
new InputDescriptor(logicalInputClass.getName()));
break;
case CUSTOM_SIMPLE_EDGE:
dataMovementType = DataMovementType.SCATTER_GATHER;
logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class;
logicalInputClass = ShuffledUnorderedKVInput.class;
break;
case SIMPLE_EDGE:
default:
dataMovementType = DataMovementType.SCATTER_GATHER;
logicalOutputClass = OnFileSortedOutput.class;
logicalInputClass = ShuffledMergedInputLegacy.class;
break;
}
if (edgeProperty == null) {
edgeProperty =
new EdgeProperty(
dataMovementType,
DataSourceType.PERSISTED,
SchedulingType.SEQUENTIAL,
new OutputDescriptor(logicalOutputClass.getName()),
new InputDescriptor(logicalInputClass.getName()));
}
return edgeProperty;
}