예제 #1
0
 /** Advance the watermark to the specified time, firing any timers that should fire. */
 public void advanceWatermark(Instant newWatermark) throws Exception {
   Preconditions.checkState(
       !newWatermark.isBefore(watermark),
       "Cannot move watermark time backwards from %s to %s",
       watermark.getMillis(),
       newWatermark.getMillis());
   logInteraction("Advancing watermark to %d", newWatermark.getMillis());
   watermark = newWatermark;
   timerInternals.advanceWatermark(runner, newWatermark);
 }
예제 #2
0
 /** Advance the processing time to the specified time, firing any timers that should fire. */
 public void advanceProcessingTime(Instant newProcessingTime) throws Exception {
   Preconditions.checkState(
       !newProcessingTime.isBefore(processingTime),
       "Cannot move processing time backwards from %s to %s",
       processingTime.getMillis(),
       newProcessingTime.getMillis());
   logInteraction("Advancing processing time to %d", newProcessingTime.getMillis());
   processingTime = newProcessingTime;
   timerInternals.advanceProcessingTime(runner, newProcessingTime);
 }
예제 #3
0
 public Instant minimumWatermarkHold() {
   Instant minimum = null;
   for (State storage : inMemoryState.values()) {
     if (storage instanceof WatermarkStateInternal) {
       Instant hold = ((WatermarkStateInternal) storage).get().read();
       if (minimum == null || (hold != null && hold.isBefore(minimum))) {
         minimum = hold;
       }
     }
   }
   return minimum;
 }
 protected void checkTimestamp(WindowedValue<IN> ref, Instant timestamp) {
   if (timestamp.isBefore(ref.getTimestamp().minus(doFn.getAllowedTimestampSkew()))) {
     throw new IllegalArgumentException(
         String.format(
             "Cannot output with timestamp %s. Output timestamps must be no earlier than the "
                 + "timestamp of the current input (%s) minus the allowed skew (%s). See the "
                 + "DoFn#getAllowedTimestmapSkew() Javadoc for details on changing the allowed skew.",
             timestamp,
             ref.getTimestamp(),
             PeriodFormat.getDefault().print(doFn.getAllowedTimestampSkew().toPeriod())));
   }
 }
예제 #5
0
  private <W> PaneInfo describePane(
      Object key, Instant windowMaxTimestamp, PaneInfo previousPane, boolean isFinal) {
    boolean isFirst = previousPane == null;
    Timing previousTiming = isFirst ? null : previousPane.getTiming();
    long index = isFirst ? 0 : previousPane.getIndex() + 1;
    long nonSpeculativeIndex = isFirst ? 0 : previousPane.getNonSpeculativeIndex() + 1;
    Instant outputWM = timerInternals.currentOutputWatermarkTime();
    Instant inputWM = timerInternals.currentInputWatermarkTime();

    // True if it is not possible to assign the element representing this pane a timestamp
    // which will make an ON_TIME pane for any following computation.
    // Ie true if the element's latest possible timestamp is before the current output watermark.
    boolean isLateForOutput = outputWM != null && windowMaxTimestamp.isBefore(outputWM);

    // True if all emitted panes (if any) were EARLY panes.
    // Once the ON_TIME pane has fired, all following panes must be considered LATE even
    // if the output watermark is behind the end of the window.
    boolean onlyEarlyPanesSoFar = previousTiming == null || previousTiming == Timing.EARLY;

    // True is the input watermark hasn't passed the window's max timestamp.
    boolean isEarlyForInput = !inputWM.isAfter(windowMaxTimestamp);

    Timing timing;
    if (isLateForOutput || !onlyEarlyPanesSoFar) {
      // The output watermark has already passed the end of this window, or we have already
      // emitted a non-EARLY pane. Irrespective of how this pane was triggered we must
      // consider this pane LATE.
      timing = Timing.LATE;
    } else if (isEarlyForInput) {
      // This is an EARLY firing.
      timing = Timing.EARLY;
      nonSpeculativeIndex = -1;
    } else {
      // This is the unique ON_TIME firing for the window.
      timing = Timing.ON_TIME;
    }

    WindowTracing.debug(
        "describePane: {} pane (prev was {}) for key:{}; windowMaxTimestamp:{}; "
            + "inputWatermark:{}; outputWatermark:{}; isLateForOutput:{}",
        timing,
        previousTiming,
        key,
        windowMaxTimestamp,
        inputWM,
        outputWM,
        isLateForOutput);

    if (previousPane != null) {
      // Timing transitions should follow EARLY* ON_TIME? LATE*
      switch (previousTiming) {
        case EARLY:
          checkState(
              timing == Timing.EARLY || timing == Timing.ON_TIME || timing == Timing.LATE,
              "EARLY cannot transition to %s",
              timing);
          break;
        case ON_TIME:
          checkState(timing == Timing.LATE, "ON_TIME cannot transition to %s", timing);
          break;
        case LATE:
          checkState(timing == Timing.LATE, "LATE cannot transtion to %s", timing);
          break;
        case UNKNOWN:
          break;
      }
      checkState(!previousPane.isLast(), "Last pane was not last after all.");
    }

    return PaneInfo.createPane(isFirst, isFinal, timing, index, nonSpeculativeIndex);
  }