예제 #1
0
  @Override
  public void trigger(long time) throws Exception {
    boolean fire;

    // Remove information about the triggering task
    processingTimeTimerFutures.remove(time);
    processingTimeTimerTimestamps.remove(time, processingTimeTimerTimestamps.count(time));

    do {
      Timer<K, W> timer = processingTimeTimersQueue.peek();
      if (timer != null && timer.timestamp <= time) {
        fire = true;

        processingTimeTimers.remove(timer);
        processingTimeTimersQueue.remove();

        context.key = timer.key;
        context.window = timer.window;
        setKeyContext(timer.key);

        ListState<StreamRecord<IN>> windowState;
        MergingWindowSet<W> mergingWindows = null;

        if (windowAssigner instanceof MergingWindowAssigner) {
          mergingWindows = getMergingWindowSet();
          W stateWindow = mergingWindows.getStateWindow(context.window);
          windowState = getPartitionedState(stateWindow, windowSerializer, windowStateDescriptor);
        } else {
          windowState =
              getPartitionedState(context.window, windowSerializer, windowStateDescriptor);
        }

        TriggerResult triggerResult = context.onProcessingTime(timer.timestamp);
        fireOrContinue(triggerResult, context.window, windowState);

        if (triggerResult.isPurge()
            || (!windowAssigner.isEventTime() && isCleanupTime(timer.window, timer.timestamp))) {
          cleanup(timer.window, windowState, mergingWindows);
        }

      } else {
        fire = false;
      }
    } while (fire);
  }
예제 #2
0
  @Override
  public void processWatermark(Watermark mark) throws Exception {
    boolean fire;
    do {
      Timer<K, W> timer = watermarkTimersQueue.peek();
      if (timer != null && timer.timestamp <= mark.getTimestamp()) {
        fire = true;

        watermarkTimers.remove(timer);
        watermarkTimersQueue.remove();

        context.key = timer.key;
        context.window = timer.window;
        setKeyContext(timer.key);

        ListState<StreamRecord<IN>> windowState;
        MergingWindowSet<W> mergingWindows = null;

        if (windowAssigner instanceof MergingWindowAssigner) {
          mergingWindows = getMergingWindowSet();
          W stateWindow = mergingWindows.getStateWindow(context.window);
          windowState = getPartitionedState(stateWindow, windowSerializer, windowStateDescriptor);
        } else {
          windowState =
              getPartitionedState(context.window, windowSerializer, windowStateDescriptor);
        }

        TriggerResult triggerResult = context.onEventTime(timer.timestamp);
        fireOrContinue(triggerResult, context.window, windowState);

        if (triggerResult.isPurge()
            || (windowAssigner.isEventTime() && isCleanupTime(timer.window, timer.timestamp))) {
          cleanup(timer.window, windowState, mergingWindows);
        }

      } else {
        fire = false;
      }
    } while (fire);

    output.emitWatermark(mark);

    this.currentWatermark = mark.getTimestamp();
  }
예제 #3
0
  @Override
  @SuppressWarnings("unchecked")
  public void processElement(StreamRecord<IN> element) throws Exception {
    Collection<W> elementWindows =
        windowAssigner.assignWindows(
            element.getValue(), element.getTimestamp(), windowAssignerContext);

    final K key = (K) getStateBackend().getCurrentKey();

    if (windowAssigner instanceof MergingWindowAssigner) {

      MergingWindowSet<W> mergingWindows = getMergingWindowSet();

      for (W window : elementWindows) {
        // If there is a merge, it can only result in a window that contains our new
        // element because we always eagerly merge
        final Tuple1<TriggerResult> mergeTriggerResult = new Tuple1<>(TriggerResult.CONTINUE);

        // adding the new window might result in a merge, in that case the actualWindow
        // is the merged window and we work with that. If we don't merge then
        // actualWindow == window
        W actualWindow =
            mergingWindows.addWindow(
                window,
                new MergingWindowSet.MergeFunction<W>() {
                  @Override
                  public void merge(
                      W mergeResult,
                      Collection<W> mergedWindows,
                      W stateWindowResult,
                      Collection<W> mergedStateWindows)
                      throws Exception {
                    context.key = key;
                    context.window = mergeResult;

                    // store for later use
                    mergeTriggerResult.f0 = context.onMerge(mergedWindows);

                    for (W m : mergedWindows) {
                      context.window = m;
                      context.clear();
                      deleteCleanupTimer(m);
                    }

                    // merge the merged state windows into the newly resulting state window
                    getStateBackend()
                        .mergePartitionedStates(
                            stateWindowResult,
                            mergedStateWindows,
                            windowSerializer,
                            (StateDescriptor<? extends MergingState<?, ?>, ?>)
                                windowStateDescriptor);
                  }
                });

        // check if the window is already inactive
        if (isLate(actualWindow)) {
          LOG.info(
              "Dropped element " + element + " for window " + actualWindow + " due to lateness.");
          continue;
        }

        W stateWindow = mergingWindows.getStateWindow(actualWindow);
        ListState<StreamRecord<IN>> windowState =
            getPartitionedState(stateWindow, windowSerializer, windowStateDescriptor);
        windowState.add(element);

        context.key = key;
        context.window = actualWindow;

        // we might have already fired because of a merge but still call onElement
        // on the (possibly merged) window
        TriggerResult triggerResult = context.onElement(element);
        TriggerResult combinedTriggerResult =
            TriggerResult.merge(triggerResult, mergeTriggerResult.f0);
        fireOrContinue(combinedTriggerResult, actualWindow, windowState);

        if (combinedTriggerResult.isPurge()) {
          cleanup(actualWindow, windowState, mergingWindows);
        } else {
          registerCleanupTimer(actualWindow);
        }
      }

    } else {
      for (W window : elementWindows) {

        // check if the window is already inactive
        if (isLate(window)) {
          LOG.info("Dropped element " + element + " for window " + window + " due to lateness.");
          continue;
        }

        ListState<StreamRecord<IN>> windowState =
            getPartitionedState(window, windowSerializer, windowStateDescriptor);
        windowState.add(element);

        context.key = key;
        context.window = window;

        TriggerResult triggerResult = context.onElement(element);
        fireOrContinue(triggerResult, window, windowState);

        if (triggerResult.isPurge()) {
          cleanup(window, windowState, null);
        } else {
          registerCleanupTimer(window);
        }
      }
    }
  }