@Override
public synchronized void onVertexManagerEventReceived(VertexManagerEvent vmEvent) {
// currently events from multiple attempts of the same task can be ignored because
// their output will be the same. However, with pipelined events that may not hold.
TaskIdentifier producerTask = vmEvent.getProducerAttemptIdentifier().getTaskIdentifier();
if (!taskWithVmEvents.add(producerTask)) {
LOG.info("Ignoring vertex manager event from: " + producerTask);
return;
}
numVertexManagerEventsReceived++;
long sourceTaskOutputSize = 0;
if (vmEvent.getUserPayload() != null) {
// save output size
VertexManagerEventPayloadProto proto;
try {
proto =
VertexManagerEventPayloadProto.parseFrom(ByteString.copyFrom(vmEvent.getUserPayload()));
} catch (InvalidProtocolBufferException e) {
throw new TezUncheckedException(e);
}
sourceTaskOutputSize = proto.getOutputSize();
if (proto.hasPartitionStats()) {
try {
RoaringBitmap partitionStats = new RoaringBitmap();
ByteString compressedPartitionStats = proto.getPartitionStats();
byte[] rawData = TezCommonUtils.decompressByteStringToByteArray(compressedPartitionStats);
ByteArrayInputStream bin = new ByteArrayInputStream(rawData);
partitionStats.deserialize(new DataInputStream(bin));
parsePartitionStats(partitionStats);
} catch (IOException e) {
throw new TezUncheckedException(e);
}
}
completedSourceTasksOutputSize += sourceTaskOutputSize;
}
if (LOG.isDebugEnabled()) {
LOG.debug(
"Received info of output size: "
+ sourceTaskOutputSize
+ " numInfoReceived: "
+ numVertexManagerEventsReceived
+ " total output size: "
+ completedSourceTasksOutputSize);
}
}
@Override
protected TaskRunner2Result callInternal() throws Exception {
isStarted.set(true);
this.startTime = System.currentTimeMillis();
this.threadName = Thread.currentThread().getName();
if (LOG.isDebugEnabled()) {
LOG.debug("canFinish: " + taskSpec.getTaskAttemptID() + ": " + canFinish());
}
// Unregister from the AMReporter, since the task is now running.
this.amReporter.unregisterTask(request.getAmHost(), request.getAmPort());
synchronized (this) {
if (!shouldRunTask) {
LOG.info("Not starting task {} since it was killed earlier", taskSpec.getTaskAttemptID());
return new TaskRunner2Result(EndReason.KILL_REQUESTED, null, false);
}
}
// TODO This executor seems unnecessary. Here and TezChild
ExecutorService executorReal =
Executors.newFixedThreadPool(
1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("TezTaskRunner").build());
executor = MoreExecutors.listeningDecorator(executorReal);
// TODO Consolidate this code with TezChild.
runtimeWatch.start();
UserGroupInformation taskUgi = UserGroupInformation.createRemoteUser(request.getUser());
taskUgi.addCredentials(credentials);
Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<>();
serviceConsumerMetadata.put(
TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID,
TezCommonUtils.convertJobTokenToBytes(jobToken));
Multimap<String, String> startedInputsMap = createStartedInputMap(request.getFragmentSpec());
UserGroupInformation taskOwner =
UserGroupInformation.createRemoteUser(request.getTokenIdentifier());
final InetSocketAddress address =
NetUtils.createSocketAddrForHost(request.getAmHost(), request.getAmPort());
SecurityUtil.setTokenService(jobToken, address);
taskOwner.addToken(jobToken);
umbilical =
taskOwner.doAs(
new PrivilegedExceptionAction<LlapTaskUmbilicalProtocol>() {
@Override
public LlapTaskUmbilicalProtocol run() throws Exception {
return RPC.getProxy(
LlapTaskUmbilicalProtocol.class,
LlapTaskUmbilicalProtocol.versionID,
address,
conf);
}
});
taskReporter =
new LlapTaskReporter(
umbilical,
confParams.amHeartbeatIntervalMsMax,
confParams.amCounterHeartbeatInterval,
confParams.amMaxEventsPerHeartbeat,
new AtomicLong(0),
request.getContainerIdString());
String attemptId = fragmentInfo.getFragmentIdentifierString();
IOContextMap.setThreadAttemptId(attemptId);
try {
synchronized (this) {
if (shouldRunTask) {
taskRunner =
new TezTaskRunner2(
conf,
taskUgi,
fragmentInfo.getLocalDirs(),
taskSpec,
request.getAppAttemptNumber(),
serviceConsumerMetadata,
envMap,
startedInputsMap,
taskReporter,
executor,
objectRegistry,
pid,
executionContext,
memoryAvailable,
false);
}
}
if (taskRunner == null) {
LOG.info("Not starting task {} since it was killed earlier", taskSpec.getTaskAttemptID());
return new TaskRunner2Result(EndReason.KILL_REQUESTED, null, false);
}
try {
TaskRunner2Result result = taskRunner.run();
if (result.isContainerShutdownRequested()) {
LOG.warn("Unexpected container shutdown requested while running task. Ignoring");
}
isCompleted.set(true);
return result;
} finally {
FileSystem.closeAllForUGI(taskUgi);
LOG.info(
"ExecutionTime for Container: "
+ request.getContainerIdString()
+ "="
+ runtimeWatch.stop().elapsedMillis());
if (LOG.isDebugEnabled()) {
LOG.debug(
"canFinish post completion: " + taskSpec.getTaskAttemptID() + ": " + canFinish());
}
}
} finally {
IOContextMap.clearThreadAttempt(attemptId);
}
}