private void processNewSources() {
checkLockHeld("Lock must be held to call processNewSources");
// only update if the driver is still alive
if (state.get() != State.ALIVE) {
return;
}
// copy the pending sources
// it is ok to "miss" a source added during the copy as it will be
// handled on the next call to this method
Map<PlanNodeId, TaskSource> sources = new HashMap<>(newSources);
for (Entry<PlanNodeId, TaskSource> entry : sources.entrySet()) {
// Remove the entries we are going to process from the newSources map.
// It is ok if someone already updated the entry; we will catch it on
// the next iteration.
newSources.remove(entry.getKey(), entry.getValue());
processNewSource(entry.getValue());
}
}
private void processNewSource(TaskSource source) {
checkLockHeld("Lock must be held to call processNewSources");
// create new source
Set<ScheduledSplit> newSplits;
TaskSource currentSource = currentSources.get(source.getPlanNodeId());
if (currentSource == null) {
newSplits = source.getSplits();
currentSources.put(source.getPlanNodeId(), source);
} else {
// merge the current source and the specified source
TaskSource newSource = currentSource.update(source);
// if this is not a new source, just return
if (newSource == currentSource) {
return;
}
// find the new splits to add
newSplits = Sets.difference(newSource.getSplits(), currentSource.getSplits());
currentSources.put(source.getPlanNodeId(), newSource);
}
// add new splits
for (ScheduledSplit newSplit : newSplits) {
Split split = newSplit.getSplit();
SourceOperator sourceOperator = sourceOperators.get(source.getPlanNodeId());
if (sourceOperator != null) {
sourceOperator.addSplit(split);
}
}
// set no more splits
if (source.isNoMoreSplits()) {
sourceOperators.get(source.getPlanNodeId()).noMoreSplits();
}
}
private void destroyIfNecessary() {
checkLockHeld("Lock must be held to call destroyIfNecessary");
if (!state.compareAndSet(State.NEED_DESTRUCTION, State.DESTROYED)) {
return;
}
Throwable inFlightException = null;
try {
// call finish on every operator; if error occurs, just bail out; we will still call close
for (Operator operator : operators) {
operator.finish();
}
} catch (Throwable t) {
// record in flight exception so we can add suppressed exceptions below
inFlightException = t;
throw t;
} finally {
// record the current interrupted status (and clear the flag); we'll reset it later
boolean wasInterrupted = Thread.interrupted();
// if we get an error while closing a driver, record it and we will throw it at the end
try {
for (Operator operator : operators) {
if (operator instanceof AutoCloseable) {
try {
((AutoCloseable) operator).close();
} catch (InterruptedException t) {
// don't record the stack
wasInterrupted = true;
} catch (Throwable t) {
inFlightException =
addSuppressedException(
inFlightException,
t,
"Error closing operator %s for task %s",
operator.getOperatorContext().getOperatorId(),
driverContext.getTaskId());
}
}
}
driverContext.finished();
} catch (Throwable t) {
// this shouldn't happen but be safe
inFlightException =
addSuppressedException(
inFlightException,
t,
"Error destroying driver for task %s",
driverContext.getTaskId());
} finally {
// reset the interrupted flag
if (wasInterrupted) {
Thread.currentThread().interrupt();
}
}
if (inFlightException != null) {
// this will always be an Error or Runtime
throw Throwables.propagate(inFlightException);
}
}
}