Example #1
0
  public void restoreStateLazy() throws Exception {
    if (lazyRestoreState != null) {
      LOG.info("Restoring checkpointed state to task {}", getName());

      try {
        final StreamOperator<?>[] allOperators = operatorChain.getAllOperators();
        final StreamTaskState[] states = lazyRestoreState.getState(userClassLoader);

        // be GC friendly
        lazyRestoreState = null;

        for (int i = 0; i < states.length; i++) {
          StreamTaskState state = states[i];
          StreamOperator<?> operator = allOperators[i];

          if (state != null && operator != null) {
            LOG.debug("Task {} in chain ({}) has checkpointed state", i, getName());
            operator.restoreState(state);
          } else if (operator != null) {
            LOG.debug("Task {} in chain ({}) does not have checkpointed state", i, getName());
          }
        }
      } catch (Exception e) {
        throw new Exception("Could not restore checkpointed state to operators and functions", e);
      }
    }
  }
Example #2
0
  @Override
  public void triggerCheckpoint(long checkpointId, long timestamp) throws Exception {
    LOG.debug("Starting checkpoint {} on task {}", checkpointId, getName());

    synchronized (lock) {
      if (isRunning) {

        // since both state checkpointing and downstream barrier emission occurs in this
        // lock scope, they are an atomic operation regardless of the order in which they occur
        // we immediately emit the checkpoint barriers, so the downstream operators can start
        // their checkpoint work as soon as possible
        operatorChain.broadcastCheckpointBarrier(checkpointId, timestamp);

        // now draw the state snapshot
        try {
          final StreamOperator<?>[] allOperators = operatorChain.getAllOperators();
          final StreamTaskState[] states = new StreamTaskState[allOperators.length];

          for (int i = 0; i < states.length; i++) {
            StreamOperator<?> operator = allOperators[i];
            if (operator != null) {
              StreamTaskState state = operator.snapshotOperatorState(checkpointId, timestamp);
              states[i] = state.isEmpty() ? null : state;
            }
          }

          StreamTaskStateList allStates = new StreamTaskStateList(states);
          if (allStates.isEmpty()) {
            getEnvironment().acknowledgeCheckpoint(checkpointId);
          } else {
            getEnvironment().acknowledgeCheckpoint(checkpointId, allStates);
          }
        } catch (Exception e) {
          if (isRunning) {
            throw e;
          }
        }
      }
    }
  }
Example #3
0
  @Override
  @SuppressWarnings("unchecked,rawtypes")
  public boolean triggerCheckpoint(final long checkpointId, final long timestamp) throws Exception {
    LOG.debug("Starting checkpoint {} on task {}", checkpointId, getName());

    synchronized (lock) {
      if (isRunning) {

        // since both state checkpointing and downstream barrier emission occurs in this
        // lock scope, they are an atomic operation regardless of the order in which they occur
        // we immediately emit the checkpoint barriers, so the downstream operators can start
        // their checkpoint work as soon as possible
        operatorChain.broadcastCheckpointBarrier(checkpointId, timestamp);

        // now draw the state snapshot
        try {
          final StreamOperator<?>[] allOperators = operatorChain.getAllOperators();
          final StreamTaskState[] states = new StreamTaskState[allOperators.length];

          boolean hasAsyncStates = false;

          for (int i = 0; i < states.length; i++) {
            StreamOperator<?> operator = allOperators[i];
            if (operator != null) {
              StreamTaskState state = operator.snapshotOperatorState(checkpointId, timestamp);
              if (state.getOperatorState() instanceof AsynchronousStateHandle) {
                hasAsyncStates = true;
              }
              if (state.getFunctionState() instanceof AsynchronousStateHandle) {
                hasAsyncStates = true;
              }
              states[i] = state.isEmpty() ? null : state;
            }
          }

          StreamTaskStateList allStates = new StreamTaskStateList(states);

          if (allStates.isEmpty()) {
            getEnvironment().acknowledgeCheckpoint(checkpointId);
          } else if (!hasAsyncStates) {
            getEnvironment().acknowledgeCheckpoint(checkpointId, allStates);
          } else {
            // start a Thread that does the asynchronous materialization and
            // then sends the checkpoint acknowledge

            Thread checkpointThread =
                new Thread() {
                  @Override
                  public void run() {
                    try {
                      for (StreamTaskState state : states) {
                        if (state != null) {
                          if (state.getFunctionState() instanceof AsynchronousStateHandle) {
                            AsynchronousStateHandle<?> asyncState =
                                (AsynchronousStateHandle<?>) state.getFunctionState();
                            state.setFunctionState((StateHandle) asyncState.materialize());
                          }
                          if (state.getOperatorState() instanceof AsynchronousStateHandle) {
                            AsynchronousStateHandle<?> asyncState =
                                (AsynchronousStateHandle<?>) state.getOperatorState();
                            state.setOperatorState((StateHandle) asyncState.materialize());
                          }
                        }
                      }
                      StreamTaskStateList allStates = new StreamTaskStateList(states);
                      getEnvironment().acknowledgeCheckpoint(checkpointId, allStates);
                    } catch (Exception e) {
                      LOG.error(
                          "Caught exception while materializing asynchronous checkpoints.", e);
                      if (asyncException == null) {
                        asyncException = new AsynchronousException(e);
                      }
                    }
                    asyncCheckpointThreads.remove(this);
                    LOG.debug(
                        "Finished asynchronous checkpoints for checkpoint {} on task {}",
                        checkpointId,
                        getName());
                  }
                };

            asyncCheckpointThreads.add(checkpointThread);
            checkpointThread.start();
          }
        } catch (Exception e) {
          if (isRunning) {
            throw e;
          }
        }
        return true;
      } else {
        return false;
      }
    }
  }