/** Advance the watermark to the specified time, firing any timers that should fire. */
public void advanceWatermark(Instant newWatermark) throws Exception {
Preconditions.checkState(
!newWatermark.isBefore(watermark),
"Cannot move watermark time backwards from %s to %s",
watermark.getMillis(),
newWatermark.getMillis());
logInteraction("Advancing watermark to %d", newWatermark.getMillis());
watermark = newWatermark;
timerInternals.advanceWatermark(runner, newWatermark);
}
/** Advance the processing time to the specified time, firing any timers that should fire. */
public void advanceProcessingTime(Instant newProcessingTime) throws Exception {
Preconditions.checkState(
!newProcessingTime.isBefore(processingTime),
"Cannot move processing time backwards from %s to %s",
processingTime.getMillis(),
newProcessingTime.getMillis());
logInteraction("Advancing processing time to %d", newProcessingTime.getMillis());
processingTime = newProcessingTime;
timerInternals.advanceProcessingTime(runner, newProcessingTime);
}
public Instant minimumWatermarkHold() {
Instant minimum = null;
for (State storage : inMemoryState.values()) {
if (storage instanceof WatermarkStateInternal) {
Instant hold = ((WatermarkStateInternal) storage).get().read();
if (minimum == null || (hold != null && hold.isBefore(minimum))) {
minimum = hold;
}
}
}
return minimum;
}
protected void checkTimestamp(WindowedValue<IN> ref, Instant timestamp) {
if (timestamp.isBefore(ref.getTimestamp().minus(doFn.getAllowedTimestampSkew()))) {
throw new IllegalArgumentException(
String.format(
"Cannot output with timestamp %s. Output timestamps must be no earlier than the "
+ "timestamp of the current input (%s) minus the allowed skew (%s). See the "
+ "DoFn#getAllowedTimestmapSkew() Javadoc for details on changing the allowed skew.",
timestamp,
ref.getTimestamp(),
PeriodFormat.getDefault().print(doFn.getAllowedTimestampSkew().toPeriod())));
}
}
private <W> PaneInfo describePane(
Object key, Instant windowMaxTimestamp, PaneInfo previousPane, boolean isFinal) {
boolean isFirst = previousPane == null;
Timing previousTiming = isFirst ? null : previousPane.getTiming();
long index = isFirst ? 0 : previousPane.getIndex() + 1;
long nonSpeculativeIndex = isFirst ? 0 : previousPane.getNonSpeculativeIndex() + 1;
Instant outputWM = timerInternals.currentOutputWatermarkTime();
Instant inputWM = timerInternals.currentInputWatermarkTime();
// True if it is not possible to assign the element representing this pane a timestamp
// which will make an ON_TIME pane for any following computation.
// Ie true if the element's latest possible timestamp is before the current output watermark.
boolean isLateForOutput = outputWM != null && windowMaxTimestamp.isBefore(outputWM);
// True if all emitted panes (if any) were EARLY panes.
// Once the ON_TIME pane has fired, all following panes must be considered LATE even
// if the output watermark is behind the end of the window.
boolean onlyEarlyPanesSoFar = previousTiming == null || previousTiming == Timing.EARLY;
// True is the input watermark hasn't passed the window's max timestamp.
boolean isEarlyForInput = !inputWM.isAfter(windowMaxTimestamp);
Timing timing;
if (isLateForOutput || !onlyEarlyPanesSoFar) {
// The output watermark has already passed the end of this window, or we have already
// emitted a non-EARLY pane. Irrespective of how this pane was triggered we must
// consider this pane LATE.
timing = Timing.LATE;
} else if (isEarlyForInput) {
// This is an EARLY firing.
timing = Timing.EARLY;
nonSpeculativeIndex = -1;
} else {
// This is the unique ON_TIME firing for the window.
timing = Timing.ON_TIME;
}
WindowTracing.debug(
"describePane: {} pane (prev was {}) for key:{}; windowMaxTimestamp:{}; "
+ "inputWatermark:{}; outputWatermark:{}; isLateForOutput:{}",
timing,
previousTiming,
key,
windowMaxTimestamp,
inputWM,
outputWM,
isLateForOutput);
if (previousPane != null) {
// Timing transitions should follow EARLY* ON_TIME? LATE*
switch (previousTiming) {
case EARLY:
checkState(
timing == Timing.EARLY || timing == Timing.ON_TIME || timing == Timing.LATE,
"EARLY cannot transition to %s",
timing);
break;
case ON_TIME:
checkState(timing == Timing.LATE, "ON_TIME cannot transition to %s", timing);
break;
case LATE:
checkState(timing == Timing.LATE, "LATE cannot transtion to %s", timing);
break;
case UNKNOWN:
break;
}
checkState(!previousPane.isLast(), "Last pane was not last after all.");
}
return PaneInfo.createPane(isFirst, isFinal, timing, index, nonSpeculativeIndex);
}
