private void destroyIfNecessary() {
checkLockHeld("Lock must be held to call destroyIfNecessary");
if (!state.compareAndSet(State.NEED_DESTRUCTION, State.DESTROYED)) {
return;
}
Throwable inFlightException = null;
try {
// call finish on every operator; if error occurs, just bail out; we will still call close
for (Operator operator : operators) {
operator.finish();
}
} catch (Throwable t) {
// record in flight exception so we can add suppressed exceptions below
inFlightException = t;
throw t;
} finally {
// record the current interrupted status (and clear the flag); we'll reset it later
boolean wasInterrupted = Thread.interrupted();
// if we get an error while closing a driver, record it and we will throw it at the end
try {
for (Operator operator : operators) {
if (operator instanceof AutoCloseable) {
try {
((AutoCloseable) operator).close();
} catch (InterruptedException t) {
// don't record the stack
wasInterrupted = true;
} catch (Throwable t) {
inFlightException =
addSuppressedException(
inFlightException,
t,
"Error closing operator %s for task %s",
operator.getOperatorContext().getOperatorId(),
driverContext.getTaskId());
}
}
}
driverContext.finished();
} catch (Throwable t) {
// this shouldn't happen but be safe
inFlightException =
addSuppressedException(
inFlightException,
t,
"Error destroying driver for task %s",
driverContext.getTaskId());
} finally {
// reset the interrupted flag
if (wasInterrupted) {
Thread.currentThread().interrupt();
}
}
if (inFlightException != null) {
// this will always be an Error or Runtime
throw Throwables.propagate(inFlightException);
}
}
}
public ListenableFuture<?> process() {
checkLockNotHeld("Can not process while holding the driver lock");
try (DriverLockResult lockResult =
tryLockAndProcessPendingStateChanges(100, TimeUnit.MILLISECONDS)) {
try {
if (!lockResult.wasAcquired()) {
// this is unlikely to happen unless the driver is being
// destroyed and in that case the caller should notice notice
// this state change by calling isFinished
return NOT_BLOCKED;
}
driverContext.start();
if (!newSources.isEmpty()) {
processNewSources();
}
for (int i = 0; i < operators.size() - 1 && !driverContext.isDone(); i++) {
// check if current operator is blocked
Operator current = operators.get(i);
ListenableFuture<?> blocked = current.isBlocked();
if (!blocked.isDone()) {
current.getOperatorContext().recordBlocked(blocked);
return blocked;
}
// check if next operator is blocked
Operator next = operators.get(i + 1);
blocked = next.isBlocked();
if (!blocked.isDone()) {
next.getOperatorContext().recordBlocked(blocked);
return blocked;
}
// if current operator is finished...
if (current.isFinished()) {
// let next operator know there will be no more data
next.getOperatorContext().startIntervalTimer();
next.finish();
next.getOperatorContext().recordFinish();
} else {
// if next operator needs input...
if (next.needsInput()) {
// get an output page from current operator
current.getOperatorContext().startIntervalTimer();
Page page = current.getOutput();
current.getOperatorContext().recordGetOutput(page);
// if we got an output page, add it to the next operator
if (page != null) {
next.getOperatorContext().startIntervalTimer();
next.addInput(page);
next.getOperatorContext().recordAddInput(page);
}
}
}
}
return NOT_BLOCKED;
} catch (Throwable t) {
driverContext.failed(t);
throw t;
}
}
}