コード例 #1
0
  @Override
  public synchronized void onVertexManagerEventReceived(VertexManagerEvent vmEvent) {
    // currently events from multiple attempts of the same task can be ignored because
    // their output will be the same. However, with pipelined events that may not hold.
    TaskIdentifier producerTask = vmEvent.getProducerAttemptIdentifier().getTaskIdentifier();
    if (!taskWithVmEvents.add(producerTask)) {
      LOG.info("Ignoring vertex manager event from: " + producerTask);
      return;
    }

    numVertexManagerEventsReceived++;

    long sourceTaskOutputSize = 0;
    if (vmEvent.getUserPayload() != null) {
      // save output size
      VertexManagerEventPayloadProto proto;
      try {
        proto =
            VertexManagerEventPayloadProto.parseFrom(ByteString.copyFrom(vmEvent.getUserPayload()));
      } catch (InvalidProtocolBufferException e) {
        throw new TezUncheckedException(e);
      }
      sourceTaskOutputSize = proto.getOutputSize();

      if (proto.hasPartitionStats()) {
        try {
          RoaringBitmap partitionStats = new RoaringBitmap();
          ByteString compressedPartitionStats = proto.getPartitionStats();
          byte[] rawData = TezCommonUtils.decompressByteStringToByteArray(compressedPartitionStats);
          ByteArrayInputStream bin = new ByteArrayInputStream(rawData);
          partitionStats.deserialize(new DataInputStream(bin));

          parsePartitionStats(partitionStats);
        } catch (IOException e) {
          throw new TezUncheckedException(e);
        }
      }
      completedSourceTasksOutputSize += sourceTaskOutputSize;
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug(
          "Received info of output size: "
              + sourceTaskOutputSize
              + " numInfoReceived: "
              + numVertexManagerEventsReceived
              + " total output size: "
              + completedSourceTasksOutputSize);
    }
  }
コード例 #2
0
 @Override
 public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) {
   // TODO handle duplicates from retries
   if (enableAutoParallelism) {
     // save output size
     VertexManagerEventPayloadProto proto;
     try {
       proto = VertexManagerEventPayloadProto.parseFrom(vmEvent.getUserPayload());
     } catch (InvalidProtocolBufferException e) {
       throw new TezUncheckedException(e);
     }
     long sourceTaskOutputSize = proto.getOutputSize();
     numVertexManagerEventsReceived++;
     completedSourceTasksOutputSize += sourceTaskOutputSize;
     if (LOG.isDebugEnabled()) {
       LOG.debug(
           "Received info of output size: "
               + sourceTaskOutputSize
               + " numInfoReceived: "
               + numVertexManagerEventsReceived
               + " total output size: "
               + completedSourceTasksOutputSize);
     }
   }
 }