public ListenableFuture<?> processFor(Duration duration) {
checkLockNotHeld("Can not process for a duration while holding the driver lock");
checkNotNull(duration, "duration is null");
long maxRuntime = duration.roundTo(TimeUnit.NANOSECONDS);
long start = System.nanoTime();
do {
ListenableFuture<?> future = process();
if (!future.isDone()) {
return future;
}
} while (System.nanoTime() - start < maxRuntime && !isFinished());
return NOT_BLOCKED;
}
public boolean isFinished() {
checkLockNotHeld("Can not check finished status while holding the driver lock");
// if we can get the lock, attempt a clean shutdown; otherwise someone else will shutdown
try (DriverLockResult lockResult =
tryLockAndProcessPendingStateChanges(0, TimeUnit.MILLISECONDS)) {
if (lockResult.wasAcquired()) {
boolean finished =
state.get() != State.ALIVE
|| driverContext.isDone()
|| operators.get(operators.size() - 1).isFinished();
if (finished) {
state.compareAndSet(State.ALIVE, State.NEED_DESTRUCTION);
}
return finished;
} else {
// did not get the lock, so we can't check operators, or destroy
return state.get() != State.ALIVE || driverContext.isDone();
}
}
}
public void updateSource(TaskSource source) {
checkLockNotHeld("Can not update sources while holding the driver lock");
// does this driver have an operator for the specified source?
if (!sourceOperators.containsKey(source.getPlanNodeId())) {
return;
}
// stage the new updates
while (true) {
// attempt to update directly to the new source
TaskSource currentNewSource = newSources.putIfAbsent(source.getPlanNodeId(), source);
// if update succeeded, just break
if (currentNewSource == null) {
break;
}
// merge source into the current new source
TaskSource newSource = currentNewSource.update(source);
// if this is not a new source, just return
if (newSource == currentNewSource) {
break;
}
// attempt to replace the currentNewSource with the new source
if (newSources.replace(source.getPlanNodeId(), currentNewSource, newSource)) {
break;
}
// someone else updated while we were processing
}
// attempt to get the lock and process the updates we staged above
// updates will be processed in close if and only if we got the lock
tryLockAndProcessPendingStateChanges(0, TimeUnit.MILLISECONDS).close();
}
private DriverLockResult tryLockAndProcessPendingStateChanges(int timeout, TimeUnit unit) {
checkLockNotHeld("Can not acquire the driver lock while already holding the driver lock");
return new DriverLockResult(timeout, unit);
}
public ListenableFuture<?> process() {
checkLockNotHeld("Can not process while holding the driver lock");
try (DriverLockResult lockResult =
tryLockAndProcessPendingStateChanges(100, TimeUnit.MILLISECONDS)) {
try {
if (!lockResult.wasAcquired()) {
// this is unlikely to happen unless the driver is being
// destroyed and in that case the caller should notice notice
// this state change by calling isFinished
return NOT_BLOCKED;
}
driverContext.start();
if (!newSources.isEmpty()) {
processNewSources();
}
for (int i = 0; i < operators.size() - 1 && !driverContext.isDone(); i++) {
// check if current operator is blocked
Operator current = operators.get(i);
ListenableFuture<?> blocked = current.isBlocked();
if (!blocked.isDone()) {
current.getOperatorContext().recordBlocked(blocked);
return blocked;
}
// check if next operator is blocked
Operator next = operators.get(i + 1);
blocked = next.isBlocked();
if (!blocked.isDone()) {
next.getOperatorContext().recordBlocked(blocked);
return blocked;
}
// if current operator is finished...
if (current.isFinished()) {
// let next operator know there will be no more data
next.getOperatorContext().startIntervalTimer();
next.finish();
next.getOperatorContext().recordFinish();
} else {
// if next operator needs input...
if (next.needsInput()) {
// get an output page from current operator
current.getOperatorContext().startIntervalTimer();
Page page = current.getOutput();
current.getOperatorContext().recordGetOutput(page);
// if we got an output page, add it to the next operator
if (page != null) {
next.getOperatorContext().startIntervalTimer();
next.addInput(page);
next.getOperatorContext().recordAddInput(page);
}
}
}
}
return NOT_BLOCKED;
} catch (Throwable t) {
driverContext.failed(t);
throw t;
}
}
}