private void tryDisposeAllOperators() throws Exception {
for (StreamOperator<?> operator : operatorChain.getAllOperators()) {
if (operator != null) {
operator.dispose();
}
}
}
public void restoreStateLazy() throws Exception {
if (lazyRestoreState != null) {
LOG.info("Restoring checkpointed state to task {}", getName());
try {
final StreamOperator<?>[] allOperators = operatorChain.getAllOperators();
final StreamTaskState[] states = lazyRestoreState.getState(userClassLoader);
// be GC friendly
lazyRestoreState = null;
for (int i = 0; i < states.length; i++) {
StreamTaskState state = states[i];
StreamOperator<?> operator = allOperators[i];
if (state != null && operator != null) {
LOG.debug("Task {} in chain ({}) has checkpointed state", i, getName());
operator.restoreState(state);
} else if (operator != null) {
LOG.debug("Task {} in chain ({}) does not have checkpointed state", i, getName());
}
}
} catch (Exception e) {
throw new Exception("Could not restore checkpointed state to operators and functions", e);
}
}
}
private void openAllOperators() throws Exception {
for (StreamOperator<?> operator : operatorChain.getAllOperators()) {
if (operator != null) {
operator.open();
}
}
}
private void disposeAllOperators() {
for (StreamOperator<?> operator : operatorChain.getAllOperators()) {
try {
if (operator != null) {
operator.dispose();
}
} catch (Throwable t) {
LOG.error("Error during disposal of stream operator.", t);
}
}
}
private void closeAllOperators() throws Exception {
// We need to close them first to last, since upstream operators in the chain might emit
// elements in their close methods.
StreamOperator<?>[] allOperators = operatorChain.getAllOperators();
for (int i = allOperators.length - 1; i >= 0; i--) {
StreamOperator<?> operator = allOperators[i];
if (operator != null) {
operator.close();
}
}
}
@Override
public void notifyCheckpointComplete(long checkpointId) throws Exception {
synchronized (lock) {
if (isRunning) {
LOG.debug("Notification of complete checkpoint for task {}", getName());
for (StreamOperator<?> operator : operatorChain.getAllOperators()) {
if (operator != null) {
operator.notifyOfCompletedCheckpoint(checkpointId);
}
}
} else {
LOG.debug(
"Ignoring notification of complete checkpoint for not-running task {}", getName());
}
}
}
@Override
public void triggerCheckpoint(long checkpointId, long timestamp) throws Exception {
LOG.debug("Starting checkpoint {} on task {}", checkpointId, getName());
synchronized (lock) {
if (isRunning) {
// since both state checkpointing and downstream barrier emission occurs in this
// lock scope, they are an atomic operation regardless of the order in which they occur
// we immediately emit the checkpoint barriers, so the downstream operators can start
// their checkpoint work as soon as possible
operatorChain.broadcastCheckpointBarrier(checkpointId, timestamp);
// now draw the state snapshot
try {
final StreamOperator<?>[] allOperators = operatorChain.getAllOperators();
final StreamTaskState[] states = new StreamTaskState[allOperators.length];
for (int i = 0; i < states.length; i++) {
StreamOperator<?> operator = allOperators[i];
if (operator != null) {
StreamTaskState state = operator.snapshotOperatorState(checkpointId, timestamp);
states[i] = state.isEmpty() ? null : state;
}
}
StreamTaskStateList allStates = new StreamTaskStateList(states);
if (allStates.isEmpty()) {
getEnvironment().acknowledgeCheckpoint(checkpointId);
} else {
getEnvironment().acknowledgeCheckpoint(checkpointId, allStates);
}
} catch (Exception e) {
if (isRunning) {
throw e;
}
}
}
}
}
@Override
public void notifyCheckpointComplete(long checkpointId) throws Exception {
synchronized (lock) {
if (isRunning) {
LOG.debug("Notification of complete checkpoint for task {}", getName());
// We first notify the state backend if necessary
if (stateBackend instanceof CheckpointNotifier) {
((CheckpointNotifier) stateBackend).notifyCheckpointComplete(checkpointId);
}
for (StreamOperator<?> operator : operatorChain.getAllOperators()) {
if (operator != null) {
operator.notifyOfCompletedCheckpoint(checkpointId);
}
}
} else {
LOG.debug(
"Ignoring notification of complete checkpoint for not-running task {}", getName());
}
}
}
@Override
@SuppressWarnings("unchecked,rawtypes")
public boolean triggerCheckpoint(final long checkpointId, final long timestamp) throws Exception {
LOG.debug("Starting checkpoint {} on task {}", checkpointId, getName());
synchronized (lock) {
if (isRunning) {
// since both state checkpointing and downstream barrier emission occurs in this
// lock scope, they are an atomic operation regardless of the order in which they occur
// we immediately emit the checkpoint barriers, so the downstream operators can start
// their checkpoint work as soon as possible
operatorChain.broadcastCheckpointBarrier(checkpointId, timestamp);
// now draw the state snapshot
try {
final StreamOperator<?>[] allOperators = operatorChain.getAllOperators();
final StreamTaskState[] states = new StreamTaskState[allOperators.length];
boolean hasAsyncStates = false;
for (int i = 0; i < states.length; i++) {
StreamOperator<?> operator = allOperators[i];
if (operator != null) {
StreamTaskState state = operator.snapshotOperatorState(checkpointId, timestamp);
if (state.getOperatorState() instanceof AsynchronousStateHandle) {
hasAsyncStates = true;
}
if (state.getFunctionState() instanceof AsynchronousStateHandle) {
hasAsyncStates = true;
}
states[i] = state.isEmpty() ? null : state;
}
}
StreamTaskStateList allStates = new StreamTaskStateList(states);
if (allStates.isEmpty()) {
getEnvironment().acknowledgeCheckpoint(checkpointId);
} else if (!hasAsyncStates) {
getEnvironment().acknowledgeCheckpoint(checkpointId, allStates);
} else {
// start a Thread that does the asynchronous materialization and
// then sends the checkpoint acknowledge
Thread checkpointThread =
new Thread() {
@Override
public void run() {
try {
for (StreamTaskState state : states) {
if (state != null) {
if (state.getFunctionState() instanceof AsynchronousStateHandle) {
AsynchronousStateHandle<?> asyncState =
(AsynchronousStateHandle<?>) state.getFunctionState();
state.setFunctionState((StateHandle) asyncState.materialize());
}
if (state.getOperatorState() instanceof AsynchronousStateHandle) {
AsynchronousStateHandle<?> asyncState =
(AsynchronousStateHandle<?>) state.getOperatorState();
state.setOperatorState((StateHandle) asyncState.materialize());
}
}
}
StreamTaskStateList allStates = new StreamTaskStateList(states);
getEnvironment().acknowledgeCheckpoint(checkpointId, allStates);
} catch (Exception e) {
LOG.error(
"Caught exception while materializing asynchronous checkpoints.", e);
if (asyncException == null) {
asyncException = new AsynchronousException(e);
}
}
asyncCheckpointThreads.remove(this);
LOG.debug(
"Finished asynchronous checkpoints for checkpoint {} on task {}",
checkpointId,
getName());
}
};
asyncCheckpointThreads.add(checkpointThread);
checkpointThread.start();
}
} catch (Exception e) {
if (isRunning) {
throw e;
}
}
return true;
} else {
return false;
}
}
}
