Пример #1
0
  /**
   * Given two vertices and their respective configuration objects createEdge will create an Edge
   * object that connects the two.
   *
   * @param vConf JobConf of the first vertex
   * @param v The first vertex (source)
   * @param wConf JobConf of the second vertex
   * @param w The second vertex (sink)
   * @return
   */
  public Edge createEdge(JobConf vConf, Vertex v, JobConf wConf, Vertex w, TezEdgeProperty edgeProp)
      throws IOException {

    updateConfigurationForEdge(vConf, v, wConf, w);

    if (edgeProp.getEdgeType() == EdgeType.CUSTOM_EDGE) {
      int numBuckets = edgeProp.getNumBuckets();
      byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array();
      VertexManagerPluginDescriptor desc =
          new VertexManagerPluginDescriptor(CustomPartitionVertex.class.getName());
      desc.setUserPayload(userPayload);
      w.setVertexManagerPlugin(desc);
    }

    return new Edge(v, w, createEdgeProperty(edgeProp));
  }
Пример #2
0
  /**
   * Given a Vertex group and a vertex createEdge will create an Edge between them.
   *
   * @param group The parent VertexGroup
   * @param wConf The job conf of the child vertex
   * @param w The child vertex
   * @param edgeProp the edge property of connection between the two endpoints.
   */
  public GroupInputEdge createEdge(
      VertexGroup group, JobConf wConf, Vertex w, TezEdgeProperty edgeProp) throws IOException {

    Class mergeInputClass;

    LOG.info("Creating Edge between " + group.getGroupName() + " and " + w.getVertexName());
    w.getProcessorDescriptor().setUserPayload(MRHelpers.createUserPayloadFromConf(wConf));

    EdgeType edgeType = edgeProp.getEdgeType();
    switch (edgeType) {
      case BROADCAST_EDGE:
        mergeInputClass = ConcatenatedMergedKeyValueInput.class;
        break;
      case CUSTOM_EDGE:
        mergeInputClass = ConcatenatedMergedKeyValueInput.class;
        int numBuckets = edgeProp.getNumBuckets();
        VertexManagerPluginDescriptor desc =
            new VertexManagerPluginDescriptor(CustomPartitionVertex.class.getName());
        byte[] userPayload = ByteBuffer.allocate(4).putInt(numBuckets).array();
        desc.setUserPayload(userPayload);
        w.setVertexManagerPlugin(desc);
        break;

      case CUSTOM_SIMPLE_EDGE:
        mergeInputClass = ConcatenatedMergedKeyValueInput.class;
        break;

      case SIMPLE_EDGE:
      default:
        mergeInputClass = TezMergedLogicalInput.class;
        break;
    }

    return new GroupInputEdge(
        group, w, createEdgeProperty(edgeProp), new InputDescriptor(mergeInputClass.getName()));
  }
Пример #3
0
  /*
   * Helper function to create an edge property from an edge type.
   */
  private EdgeProperty createEdgeProperty(TezEdgeProperty edgeProp) throws IOException {
    DataMovementType dataMovementType;
    Class logicalInputClass;
    Class logicalOutputClass;

    EdgeProperty edgeProperty = null;
    EdgeType edgeType = edgeProp.getEdgeType();
    switch (edgeType) {
      case BROADCAST_EDGE:
        dataMovementType = DataMovementType.BROADCAST;
        logicalOutputClass = OnFileUnorderedKVOutput.class;
        logicalInputClass = ShuffledUnorderedKVInput.class;
        break;

      case CUSTOM_EDGE:
        dataMovementType = DataMovementType.CUSTOM;
        logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class;
        logicalInputClass = ShuffledUnorderedKVInput.class;
        EdgeManagerDescriptor edgeDesc =
            new EdgeManagerDescriptor(CustomPartitionEdge.class.getName());
        CustomEdgeConfiguration edgeConf =
            new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null);
        DataOutputBuffer dob = new DataOutputBuffer();
        edgeConf.write(dob);
        byte[] userPayload = dob.getData();
        edgeDesc.setUserPayload(userPayload);
        edgeProperty =
            new EdgeProperty(
                edgeDesc,
                DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                new OutputDescriptor(logicalOutputClass.getName()),
                new InputDescriptor(logicalInputClass.getName()));
        break;

      case CUSTOM_SIMPLE_EDGE:
        dataMovementType = DataMovementType.SCATTER_GATHER;
        logicalOutputClass = OnFileUnorderedPartitionedKVOutput.class;
        logicalInputClass = ShuffledUnorderedKVInput.class;
        break;

      case SIMPLE_EDGE:
      default:
        dataMovementType = DataMovementType.SCATTER_GATHER;
        logicalOutputClass = OnFileSortedOutput.class;
        logicalInputClass = ShuffledMergedInputLegacy.class;
        break;
    }

    if (edgeProperty == null) {
      edgeProperty =
          new EdgeProperty(
              dataMovementType,
              DataSourceType.PERSISTED,
              SchedulingType.SEQUENTIAL,
              new OutputDescriptor(logicalOutputClass.getName()),
              new InputDescriptor(logicalInputClass.getName()));
    }

    return edgeProperty;
  }