public void releaseAllResources() throws IOException {
synchronized (requestLock) {
if (!isReleased) {
try {
LOG.debug("{}: Releasing {}.", owningTaskName, this);
if (retriggerLocalRequestTimer != null) {
retriggerLocalRequestTimer.cancel();
}
for (InputChannel inputChannel : inputChannels.values()) {
try {
inputChannel.releaseAllResources();
} catch (IOException e) {
LOG.warn("Error during release of channel resources: " + e.getMessage(), e);
}
}
// The buffer pool can actually be destroyed immediately after the
// reader received all of the data from the input channels.
if (bufferPool != null) {
bufferPool.lazyDestroy();
}
} finally {
isReleased = true;
}
}
}
}
/** Retriggers a partition request. */
public void retriggerPartitionRequest(IntermediateResultPartitionID partitionId)
throws IOException, InterruptedException {
synchronized (requestLock) {
if (!isReleased) {
final InputChannel ch = inputChannels.get(partitionId);
checkNotNull(ch, "Unknown input channel with ID " + partitionId);
LOG.debug(
"Retriggering partition request {}:{}.", ch.partitionId, consumedSubpartitionIndex);
if (ch.getClass() == RemoteInputChannel.class) {
final RemoteInputChannel rch = (RemoteInputChannel) ch;
rch.retriggerSubpartitionRequest(consumedSubpartitionIndex);
} else if (ch.getClass() == LocalInputChannel.class) {
final LocalInputChannel ich = (LocalInputChannel) ch;
if (retriggerLocalRequestTimer == null) {
retriggerLocalRequestTimer = new Timer(true);
}
ich.retriggerSubpartitionRequest(retriggerLocalRequestTimer, consumedSubpartitionIndex);
} else {
throw new IllegalStateException(
"Unexpected type of channel to retrigger partition: " + ch.getClass());
}
}
}
}
@Test
public void testExponentialNoBackoff() throws Exception {
InputChannel ch = createInputChannel(0, 0);
assertEquals(0, ch.getCurrentBackoff());
assertFalse(ch.increaseBackoff());
assertEquals(0, ch.getCurrentBackoff());
}
@Override
public void sendTaskEvent(TaskEvent event) throws IOException {
synchronized (requestLock) {
for (InputChannel inputChannel : inputChannels.values()) {
inputChannel.sendTaskEvent(event);
}
if (numberOfUninitializedChannels > 0) {
pendingEvents.add(event);
}
}
}
@Override
public boolean isFinished() {
synchronized (requestLock) {
for (InputChannel inputChannel : inputChannels.values()) {
if (!inputChannel.isReleased()) {
return false;
}
}
}
return true;
}
public void updateInputChannel(InputChannelDeploymentDescriptor icdd)
throws IOException, InterruptedException {
synchronized (requestLock) {
if (isReleased) {
// There was a race with a task failure/cancel
return;
}
final IntermediateResultPartitionID partitionId =
icdd.getConsumedPartitionId().getPartitionId();
InputChannel current = inputChannels.get(partitionId);
if (current.getClass() == UnknownInputChannel.class) {
UnknownInputChannel unknownChannel = (UnknownInputChannel) current;
InputChannel newChannel;
ResultPartitionLocation partitionLocation = icdd.getConsumedPartitionLocation();
if (partitionLocation.isLocal()) {
newChannel = unknownChannel.toLocalInputChannel();
} else if (partitionLocation.isRemote()) {
newChannel = unknownChannel.toRemoteInputChannel(partitionLocation.getConnectionId());
} else {
throw new IllegalStateException("Tried to update unknown channel with unknown channel.");
}
LOG.debug("Updated unknown input channel to {}.", newChannel);
inputChannels.put(partitionId, newChannel);
newChannel.requestSubpartition(consumedSubpartitionIndex);
for (TaskEvent event : pendingEvents) {
newChannel.sendTaskEvent(event);
}
if (--numberOfUninitializedChannels == 0) {
pendingEvents.clear();
}
}
}
}
public void setInputChannel(
IntermediateResultPartitionID partitionId, InputChannel inputChannel) {
synchronized (requestLock) {
if (inputChannels.put(checkNotNull(partitionId), checkNotNull(inputChannel)) == null
&& inputChannel.getClass() == UnknownInputChannel.class) {
numberOfUninitializedChannels++;
}
}
}
@Override
public void requestPartitions() throws IOException, InterruptedException {
// Sanity check
if (numberOfInputChannels != inputChannels.size()) {
throw new IllegalStateException(
"Bug in input gate setup logic: mismatch between"
+ "number of total input channels and the currently set number of input "
+ "channels.");
}
if (!requestedPartitionsFlag) {
synchronized (requestLock) {
for (InputChannel inputChannel : inputChannels.values()) {
inputChannel.requestSubpartition(consumedSubpartitionIndex);
}
}
requestedPartitionsFlag = true;
}
}
@Override
public BufferOrEvent getNextBufferOrEvent() throws IOException, InterruptedException {
if (hasReceivedAllEndOfPartitionEvents) {
return null;
}
if (isReleased) {
throw new IllegalStateException("Already released.");
}
requestPartitions();
InputChannel currentChannel = null;
while (currentChannel == null) {
currentChannel = inputChannelsWithData.poll(2, TimeUnit.SECONDS);
}
final Buffer buffer = currentChannel.getNextBuffer();
// Sanity check that notifications only happen when data is available
if (buffer == null) {
throw new IllegalStateException(
"Bug in input gate/channel logic: input gate got"
+ "notified by channel about available data, but none was available.");
}
if (buffer.isBuffer()) {
return new BufferOrEvent(buffer, currentChannel.getChannelIndex());
} else {
final AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
if (event.getClass() == EndOfPartitionEvent.class) {
channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex());
if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) {
hasReceivedAllEndOfPartitionEvents = true;
}
currentChannel.notifySubpartitionConsumed();
currentChannel.releaseAllResources();
}
return new BufferOrEvent(event, currentChannel.getChannelIndex());
}
}
@Test
public void testExponentialBackoffCappedAtMax() throws Exception {
InputChannel ch = createInputChannel(500, 3000);
assertEquals(0, ch.getCurrentBackoff());
assertTrue(ch.increaseBackoff());
assertEquals(500, ch.getCurrentBackoff());
assertTrue(ch.increaseBackoff());
assertEquals(1000, ch.getCurrentBackoff());
assertTrue(ch.increaseBackoff());
assertEquals(2000, ch.getCurrentBackoff());
assertTrue(ch.increaseBackoff());
assertEquals(3000, ch.getCurrentBackoff());
assertFalse(ch.increaseBackoff());
assertEquals(3000, ch.getCurrentBackoff());
}
