Beispiel #1
0
  private void processNewSources() {
    checkLockHeld("Lock must be held to call processNewSources");

    // only update if the driver is still alive
    if (state.get() != State.ALIVE) {
      return;
    }

    // copy the pending sources
    // it is ok to "miss" a source added during the copy as it will be
    // handled on the next call to this method
    Map<PlanNodeId, TaskSource> sources = new HashMap<>(newSources);
    for (Entry<PlanNodeId, TaskSource> entry : sources.entrySet()) {
      // Remove the entries we are going to process from the newSources map.
      // It is ok if someone already updated the entry; we will catch it on
      // the next iteration.
      newSources.remove(entry.getKey(), entry.getValue());

      processNewSource(entry.getValue());
    }
  }
Beispiel #2
0
  private void processNewSource(TaskSource source) {
    checkLockHeld("Lock must be held to call processNewSources");

    // create new source
    Set<ScheduledSplit> newSplits;
    TaskSource currentSource = currentSources.get(source.getPlanNodeId());
    if (currentSource == null) {
      newSplits = source.getSplits();
      currentSources.put(source.getPlanNodeId(), source);
    } else {
      // merge the current source and the specified source
      TaskSource newSource = currentSource.update(source);

      // if this is not a new source, just return
      if (newSource == currentSource) {
        return;
      }

      // find the new splits to add
      newSplits = Sets.difference(newSource.getSplits(), currentSource.getSplits());
      currentSources.put(source.getPlanNodeId(), newSource);
    }

    // add new splits
    for (ScheduledSplit newSplit : newSplits) {
      Split split = newSplit.getSplit();

      SourceOperator sourceOperator = sourceOperators.get(source.getPlanNodeId());
      if (sourceOperator != null) {
        sourceOperator.addSplit(split);
      }
    }

    // set no more splits
    if (source.isNoMoreSplits()) {
      sourceOperators.get(source.getPlanNodeId()).noMoreSplits();
    }
  }
Beispiel #3
0
  private void destroyIfNecessary() {
    checkLockHeld("Lock must be held to call destroyIfNecessary");

    if (!state.compareAndSet(State.NEED_DESTRUCTION, State.DESTROYED)) {
      return;
    }

    Throwable inFlightException = null;
    try {
      // call finish on every operator; if error occurs, just bail out; we will still call close
      for (Operator operator : operators) {
        operator.finish();
      }
    } catch (Throwable t) {
      // record in flight exception so we can add suppressed exceptions below
      inFlightException = t;
      throw t;
    } finally {
      // record the current interrupted status (and clear the flag); we'll reset it later
      boolean wasInterrupted = Thread.interrupted();

      // if we get an error while closing a driver, record it and we will throw it at the end
      try {
        for (Operator operator : operators) {
          if (operator instanceof AutoCloseable) {
            try {
              ((AutoCloseable) operator).close();
            } catch (InterruptedException t) {
              // don't record the stack
              wasInterrupted = true;
            } catch (Throwable t) {
              inFlightException =
                  addSuppressedException(
                      inFlightException,
                      t,
                      "Error closing operator %s for task %s",
                      operator.getOperatorContext().getOperatorId(),
                      driverContext.getTaskId());
            }
          }
        }
        driverContext.finished();
      } catch (Throwable t) {
        // this shouldn't happen but be safe
        inFlightException =
            addSuppressedException(
                inFlightException,
                t,
                "Error destroying driver for task %s",
                driverContext.getTaskId());
      } finally {
        // reset the interrupted flag
        if (wasInterrupted) {
          Thread.currentThread().interrupt();
        }
      }

      if (inFlightException != null) {
        // this will always be an Error or Runtime
        throw Throwables.propagate(inFlightException);
      }
    }
  }