@Override
public synchronized void onVertexManagerEventReceived(VertexManagerEvent vmEvent) {
// currently events from multiple attempts of the same task can be ignored because
// their output will be the same. However, with pipelined events that may not hold.
TaskIdentifier producerTask = vmEvent.getProducerAttemptIdentifier().getTaskIdentifier();
if (!taskWithVmEvents.add(producerTask)) {
LOG.info("Ignoring vertex manager event from: " + producerTask);
return;
}
numVertexManagerEventsReceived++;
long sourceTaskOutputSize = 0;
if (vmEvent.getUserPayload() != null) {
// save output size
VertexManagerEventPayloadProto proto;
try {
proto =
VertexManagerEventPayloadProto.parseFrom(ByteString.copyFrom(vmEvent.getUserPayload()));
} catch (InvalidProtocolBufferException e) {
throw new TezUncheckedException(e);
}
sourceTaskOutputSize = proto.getOutputSize();
if (proto.hasPartitionStats()) {
try {
RoaringBitmap partitionStats = new RoaringBitmap();
ByteString compressedPartitionStats = proto.getPartitionStats();
byte[] rawData = TezCommonUtils.decompressByteStringToByteArray(compressedPartitionStats);
ByteArrayInputStream bin = new ByteArrayInputStream(rawData);
partitionStats.deserialize(new DataInputStream(bin));
parsePartitionStats(partitionStats);
} catch (IOException e) {
throw new TezUncheckedException(e);
}
}
completedSourceTasksOutputSize += sourceTaskOutputSize;
}
if (LOG.isDebugEnabled()) {
LOG.debug(
"Received info of output size: "
+ sourceTaskOutputSize
+ " numInfoReceived: "
+ numVertexManagerEventsReceived
+ " total output size: "
+ completedSourceTasksOutputSize);
}
}
@Override
public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) {
// TODO handle duplicates from retries
if (enableAutoParallelism) {
// save output size
VertexManagerEventPayloadProto proto;
try {
proto = VertexManagerEventPayloadProto.parseFrom(vmEvent.getUserPayload());
} catch (InvalidProtocolBufferException e) {
throw new TezUncheckedException(e);
}
long sourceTaskOutputSize = proto.getOutputSize();
numVertexManagerEventsReceived++;
completedSourceTasksOutputSize += sourceTaskOutputSize;
if (LOG.isDebugEnabled()) {
LOG.debug(
"Received info of output size: "
+ sourceTaskOutputSize
+ " numInfoReceived: "
+ numVertexManagerEventsReceived
+ " total output size: "
+ completedSourceTasksOutputSize);
}
}
}