Exemplo n.º 1
0
  @Override
  public synchronized void onVertexManagerEventReceived(VertexManagerEvent vmEvent) {
    // currently events from multiple attempts of the same task can be ignored because
    // their output will be the same. However, with pipelined events that may not hold.
    TaskIdentifier producerTask = vmEvent.getProducerAttemptIdentifier().getTaskIdentifier();
    if (!taskWithVmEvents.add(producerTask)) {
      LOG.info("Ignoring vertex manager event from: " + producerTask);
      return;
    }

    numVertexManagerEventsReceived++;

    long sourceTaskOutputSize = 0;
    if (vmEvent.getUserPayload() != null) {
      // save output size
      VertexManagerEventPayloadProto proto;
      try {
        proto =
            VertexManagerEventPayloadProto.parseFrom(ByteString.copyFrom(vmEvent.getUserPayload()));
      } catch (InvalidProtocolBufferException e) {
        throw new TezUncheckedException(e);
      }
      sourceTaskOutputSize = proto.getOutputSize();

      if (proto.hasPartitionStats()) {
        try {
          RoaringBitmap partitionStats = new RoaringBitmap();
          ByteString compressedPartitionStats = proto.getPartitionStats();
          byte[] rawData = TezCommonUtils.decompressByteStringToByteArray(compressedPartitionStats);
          ByteArrayInputStream bin = new ByteArrayInputStream(rawData);
          partitionStats.deserialize(new DataInputStream(bin));

          parsePartitionStats(partitionStats);
        } catch (IOException e) {
          throw new TezUncheckedException(e);
        }
      }
      completedSourceTasksOutputSize += sourceTaskOutputSize;
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug(
          "Received info of output size: "
              + sourceTaskOutputSize
              + " numInfoReceived: "
              + numVertexManagerEventsReceived
              + " total output size: "
              + completedSourceTasksOutputSize);
    }
  }
Exemplo n.º 2
0
  @Override
  protected TaskRunner2Result callInternal() throws Exception {
    isStarted.set(true);

    this.startTime = System.currentTimeMillis();
    this.threadName = Thread.currentThread().getName();
    if (LOG.isDebugEnabled()) {
      LOG.debug("canFinish: " + taskSpec.getTaskAttemptID() + ": " + canFinish());
    }

    // Unregister from the AMReporter, since the task is now running.
    this.amReporter.unregisterTask(request.getAmHost(), request.getAmPort());

    synchronized (this) {
      if (!shouldRunTask) {
        LOG.info("Not starting task {} since it was killed earlier", taskSpec.getTaskAttemptID());
        return new TaskRunner2Result(EndReason.KILL_REQUESTED, null, false);
      }
    }

    // TODO This executor seems unnecessary. Here and TezChild
    ExecutorService executorReal =
        Executors.newFixedThreadPool(
            1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("TezTaskRunner").build());
    executor = MoreExecutors.listeningDecorator(executorReal);

    // TODO Consolidate this code with TezChild.
    runtimeWatch.start();
    UserGroupInformation taskUgi = UserGroupInformation.createRemoteUser(request.getUser());
    taskUgi.addCredentials(credentials);

    Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<>();
    serviceConsumerMetadata.put(
        TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID,
        TezCommonUtils.convertJobTokenToBytes(jobToken));
    Multimap<String, String> startedInputsMap = createStartedInputMap(request.getFragmentSpec());

    UserGroupInformation taskOwner =
        UserGroupInformation.createRemoteUser(request.getTokenIdentifier());
    final InetSocketAddress address =
        NetUtils.createSocketAddrForHost(request.getAmHost(), request.getAmPort());
    SecurityUtil.setTokenService(jobToken, address);
    taskOwner.addToken(jobToken);
    umbilical =
        taskOwner.doAs(
            new PrivilegedExceptionAction<LlapTaskUmbilicalProtocol>() {
              @Override
              public LlapTaskUmbilicalProtocol run() throws Exception {
                return RPC.getProxy(
                    LlapTaskUmbilicalProtocol.class,
                    LlapTaskUmbilicalProtocol.versionID,
                    address,
                    conf);
              }
            });

    taskReporter =
        new LlapTaskReporter(
            umbilical,
            confParams.amHeartbeatIntervalMsMax,
            confParams.amCounterHeartbeatInterval,
            confParams.amMaxEventsPerHeartbeat,
            new AtomicLong(0),
            request.getContainerIdString());

    String attemptId = fragmentInfo.getFragmentIdentifierString();
    IOContextMap.setThreadAttemptId(attemptId);
    try {
      synchronized (this) {
        if (shouldRunTask) {
          taskRunner =
              new TezTaskRunner2(
                  conf,
                  taskUgi,
                  fragmentInfo.getLocalDirs(),
                  taskSpec,
                  request.getAppAttemptNumber(),
                  serviceConsumerMetadata,
                  envMap,
                  startedInputsMap,
                  taskReporter,
                  executor,
                  objectRegistry,
                  pid,
                  executionContext,
                  memoryAvailable,
                  false);
        }
      }
      if (taskRunner == null) {
        LOG.info("Not starting task {} since it was killed earlier", taskSpec.getTaskAttemptID());
        return new TaskRunner2Result(EndReason.KILL_REQUESTED, null, false);
      }

      try {
        TaskRunner2Result result = taskRunner.run();
        if (result.isContainerShutdownRequested()) {
          LOG.warn("Unexpected container shutdown requested while running task. Ignoring");
        }
        isCompleted.set(true);
        return result;
      } finally {
        FileSystem.closeAllForUGI(taskUgi);
        LOG.info(
            "ExecutionTime for Container: "
                + request.getContainerIdString()
                + "="
                + runtimeWatch.stop().elapsedMillis());
        if (LOG.isDebugEnabled()) {
          LOG.debug(
              "canFinish post completion: " + taskSpec.getTaskAttemptID() + ": " + canFinish());
        }
      }
    } finally {
      IOContextMap.clearThreadAttempt(attemptId);
    }
  }