private void processNewSources() { checkLockHeld("Lock must be held to call processNewSources"); // only update if the driver is still alive if (state.get() != State.ALIVE) { return; } // copy the pending sources // it is ok to "miss" a source added during the copy as it will be // handled on the next call to this method Map<PlanNodeId, TaskSource> sources = new HashMap<>(newSources); for (Entry<PlanNodeId, TaskSource> entry : sources.entrySet()) { // Remove the entries we are going to process from the newSources map. // It is ok if someone already updated the entry; we will catch it on // the next iteration. newSources.remove(entry.getKey(), entry.getValue()); processNewSource(entry.getValue()); } }
private void processNewSource(TaskSource source) { checkLockHeld("Lock must be held to call processNewSources"); // create new source Set<ScheduledSplit> newSplits; TaskSource currentSource = currentSources.get(source.getPlanNodeId()); if (currentSource == null) { newSplits = source.getSplits(); currentSources.put(source.getPlanNodeId(), source); } else { // merge the current source and the specified source TaskSource newSource = currentSource.update(source); // if this is not a new source, just return if (newSource == currentSource) { return; } // find the new splits to add newSplits = Sets.difference(newSource.getSplits(), currentSource.getSplits()); currentSources.put(source.getPlanNodeId(), newSource); } // add new splits for (ScheduledSplit newSplit : newSplits) { Split split = newSplit.getSplit(); SourceOperator sourceOperator = sourceOperators.get(source.getPlanNodeId()); if (sourceOperator != null) { sourceOperator.addSplit(split); } } // set no more splits if (source.isNoMoreSplits()) { sourceOperators.get(source.getPlanNodeId()).noMoreSplits(); } }
private void destroyIfNecessary() { checkLockHeld("Lock must be held to call destroyIfNecessary"); if (!state.compareAndSet(State.NEED_DESTRUCTION, State.DESTROYED)) { return; } Throwable inFlightException = null; try { // call finish on every operator; if error occurs, just bail out; we will still call close for (Operator operator : operators) { operator.finish(); } } catch (Throwable t) { // record in flight exception so we can add suppressed exceptions below inFlightException = t; throw t; } finally { // record the current interrupted status (and clear the flag); we'll reset it later boolean wasInterrupted = Thread.interrupted(); // if we get an error while closing a driver, record it and we will throw it at the end try { for (Operator operator : operators) { if (operator instanceof AutoCloseable) { try { ((AutoCloseable) operator).close(); } catch (InterruptedException t) { // don't record the stack wasInterrupted = true; } catch (Throwable t) { inFlightException = addSuppressedException( inFlightException, t, "Error closing operator %s for task %s", operator.getOperatorContext().getOperatorId(), driverContext.getTaskId()); } } } driverContext.finished(); } catch (Throwable t) { // this shouldn't happen but be safe inFlightException = addSuppressedException( inFlightException, t, "Error destroying driver for task %s", driverContext.getTaskId()); } finally { // reset the interrupted flag if (wasInterrupted) { Thread.currentThread().interrupt(); } } if (inFlightException != null) { // this will always be an Error or Runtime throw Throwables.propagate(inFlightException); } } }