@Override
    public void processElement(ProcessContext c) {
      KV<K, Iterable<InputT>> kv = c.element();
      K key = kv.getKey();

      c.output(KV.of(key, this.combineFn.apply(key, kv.getValue())));
    }
 @Override
 public void processElement(ProcessContext c) {
   String row =
       c.element().getKey() + " - " + c.element().getValue() + " @ " + c.timestamp().toString();
   System.out.println(row);
   c.output(row);
 }
    @Override
    public void processElement(ProcessContext c) {
      KV<K, AccumT> kv = c.element();
      K key = kv.getKey();
      OutputT output = this.combineFn.extractOutput(key, kv.getValue());

      c.output(KV.of(key, output));
    }
    @Override
    public void processElement(ProcessContext c) {
      KV<K, Iterable<AccumT>> kv = c.element();
      K key = kv.getKey();
      AccumT accum = this.combineFn.mergeAccumulators(key, kv.getValue());

      c.output(KV.of(key, accum));
    }
    @Override
    public void processElement(ProcessContext c) {
      KV<K, Iterable<InputT>> kv = c.element();
      K key = kv.getKey();
      AccumT accum = this.combineFn.createAccumulator(key);
      for (InputT input : kv.getValue()) {
        accum = this.combineFn.addInput(key, accum, input);
      }

      c.output(KV.of(key, accum));
    }
Esempio n. 6
0
 @Override
 public void processElement(ProcessContext c) throws Exception {
   String dest = c.element();
   logger.info("Saving to " + dest);
   Iterable<GATKRead> reads = c.sideInput(iterableView);
   OutputStream outputStream = BucketUtils.createFile(dest, c.getPipelineOptions());
   try (SAMFileWriter writer =
       new SAMFileWriterFactory().makeBAMWriter(header, false, outputStream)) {
     for (GATKRead r : reads) {
       final SAMRecord sr = r.convertToSAMRecord(header);
       writer.addAlignment(sr);
     }
   }
 }
 @Override
 public void processElement(ProcessContext c) throws IOException {
   PubsubMessage pubsubMessage = new PubsubMessage();
   pubsubMessage.encodeData(c.element().getBytes());
   if (timestampLabelKey != null) {
     Label timestampLabel = new Label();
     timestampLabel.setKey(timestampLabelKey);
     timestampLabel.setNumValue(c.timestamp().getMillis());
     pubsubMessage.setLabel(ImmutableList.of(timestampLabel));
   }
   PublishRequest publishRequest = new PublishRequest();
   publishRequest.setTopic(outputTopic).setMessage(pubsubMessage);
   this.pubsub.topics().publish(publishRequest).execute();
 }
    @Override
    public void processElement(ProcessContext c) {
      if (c.element().trim().isEmpty()) {
        emptyLines.addValue(1L);
      }

      // Split the line into words.
      String[] words = c.element().split("[^a-zA-Z']+");

      // Output each word encountered into the output PCollection.
      for (String word : words) {
        if (!word.isEmpty()) {
          c.output(word);
        }
      }
    }
Esempio n. 9
0
 @ProcessElement
 public void processElement(ProcessContext ctx) throws Exception {
   // Need to copy the document because mongoCollection.insertMany() will mutate it
   // before inserting (will assign an id).
   batch.add(new Document(ctx.element()));
   if (batch.size() >= spec.batchSize()) {
     finishBundle(ctx);
   }
 }
 private void closeWindow(
     K key, W w, Map<W, AccumT> accumulators, Map<W, Instant> minTimestamps, ProcessContext c) {
   AccumT accum = accumulators.remove(w);
   Instant timestamp = minTimestamps.remove(w);
   checkState(accum != null && timestamp != null);
   c.windowingInternals()
       .outputWindowedValue(
           KV.of(key, combineFn.extractOutput(key, accum)),
           timestamp,
           Arrays.asList(w),
           PaneInfo.ON_TIME_AND_ONLY_FIRING);
 }
  @Override
  public void processElement(ProcessContext c) throws Exception {
    KeyedWorkItem<K, InputT> element = c.element();

    K key = c.element().key();
    TimerInternals timerInternals = c.windowingInternals().timerInternals();
    StateInternals<K> stateInternals = stateInternalsFactory.stateInternalsForKey(key);

    ReduceFnRunner<K, InputT, OutputT, W> reduceFnRunner =
        new ReduceFnRunner<>(
            key,
            windowingStrategy,
            ExecutableTriggerStateMachine.create(
                TriggerStateMachines.stateMachineForTrigger(windowingStrategy.getTrigger())),
            stateInternals,
            timerInternals,
            WindowingInternalsAdapters.outputWindowedValue(c.windowingInternals()),
            WindowingInternalsAdapters.sideInputReader(c.windowingInternals()),
            droppedDueToClosedWindow,
            reduceFn,
            c.getPipelineOptions());

    reduceFnRunner.processElements(element.elementsIterable());
    for (TimerData timer : element.timersIterable()) {
      reduceFnRunner.onTimer(timer);
    }
    reduceFnRunner.persist();
  }
 @Override
 public void processElement(ProcessContext c) {
   KV<K, Iterable<RawUnionValue>> e = c.element();
   c.output(KV.of(e.getKey(), new CoGbkResult(schema, e.getValue())));
 }
 @Override
 public void processElement(ProcessContext c) {
   KV<K, ?> e = c.element();
   c.output(KV.of(e.getKey(), new RawUnionValue(index, e.getValue())));
 }
  /**
   * Execute this process.
   *
   * @param conn the database connection
   * @throws ServletException
   */
  public String execute(ConnectionProvider conn) throws ServletException {

    Process process = null;
    try {
      process = bundle.getProcessClass().newInstance();

    } catch (final Exception e) {
      log.error(e.getMessage(), e);
      throw new ServletException(e.getMessage(), e);
    }
    final String requestId = SequenceIdData.getUUID();
    String status = SCHEDULED;

    final ProcessContext ctx = bundle.getContext();
    ProcessRequestData.insert(
        conn,
        ctx.getOrganization(),
        ctx.getClient(),
        ctx.getUser(),
        ctx.getUser(),
        requestId,
        bundle.getProcessId(),
        ctx.getUser(),
        status,
        "Direct",
        ctx.toString(),
        "",
        null,
        null,
        null,
        null);

    final String executionId = SequenceIdData.getUUID();
    final long startTime = System.currentTimeMillis();
    long endTime = startTime;

    status = PROCESSING;
    ProcessRunData.insert(
        conn,
        ctx.getOrganization(),
        ctx.getClient(),
        ctx.getUser(),
        ctx.getUser(),
        executionId,
        status,
        null,
        bundle.getLog(),
        requestId);

    try {
      log.debug("Calling execute on process " + requestId);
      process.execute(bundle);
      endTime = System.currentTimeMillis();
      status = SUCCESS;

    } catch (final Exception e) {
      endTime = System.currentTimeMillis();
      status = ERROR;
      log.error("Process " + requestId + " threw an Exception: " + e.getMessage(), e);
      throw new ServletException(e);
    } finally {
      final String duration = ProcessMonitor.getDuration(endTime - startTime);
      ProcessRequestData.update(conn, COMPLETE, requestId);
      ProcessRunData.update(conn, ctx.getUser(), status, duration, bundle.getLog(), executionId);
    }

    return executionId;
  }
 @Override
 public <T> T sideInput(PCollectionView<T> view) {
   return context.sideInput(view);
 }
 @Override
 protected <AggInputT, AggOutputT> Aggregator<AggInputT, AggOutputT> createAggregatorInternal(
     String name, CombineFn<AggInputT, ?, AggOutputT> combiner) {
   return context.createAggregatorInternal(name, combiner);
 }
 @Override
 public BoundedWindow window() {
   return context.window();
 }
Esempio n. 18
0
 @Override
 protected void preProcess(ProcessContext ctx) {
   adTableId = (Integer) ctx.getRowMap().get("adTableId");
   adTabId = (Integer) ctx.getRowMap().get("adTabId");
 }
 @Override
 public <T> void sideOutputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
   synchronized (MultiThreadedIntraBundleProcessingDoFn.this) {
     context.sideOutputWithTimestamp(tag, output, timestamp);
   }
 }
 @Override
 public Instant timestamp() {
   return context.timestamp();
 }
  @Override
  public void processElement(ProcessContext c) throws Exception {
    final K key = c.element().getKey();
    Iterator<WindowedValue<InputT>> iterator = c.element().getValue().iterator();

    final PriorityQueue<W> liveWindows =
        new PriorityQueue<>(
            11,
            new Comparator<BoundedWindow>() {
              @Override
              public int compare(BoundedWindow w1, BoundedWindow w2) {
                return Long.signum(w1.maxTimestamp().getMillis() - w2.maxTimestamp().getMillis());
              }
            });

    final Map<W, AccumT> accumulators = Maps.newHashMap();
    final Map<W, Instant> minTimestamps = Maps.newHashMap();

    WindowFn<Object, W>.MergeContext mergeContext =
        new CombiningMergeContext() {
          @Override
          public Collection<W> windows() {
            return liveWindows;
          }

          @Override
          public void merge(Collection<W> toBeMerged, W mergeResult) throws Exception {
            List<AccumT> accumsToBeMerged = new ArrayList<>(toBeMerged.size());
            Instant minTimestamp = null;
            for (W window : toBeMerged) {
              accumsToBeMerged.add(accumulators.remove(window));

              Instant timestampToBeMerged = minTimestamps.remove(window);
              if (minTimestamp == null
                  || (timestampToBeMerged != null && timestampToBeMerged.isBefore(minTimestamp))) {
                minTimestamp = timestampToBeMerged;
              }
            }
            liveWindows.removeAll(toBeMerged);

            minTimestamps.put(mergeResult, minTimestamp);
            liveWindows.add(mergeResult);
            accumulators.put(mergeResult, combineFn.mergeAccumulators(key, accumsToBeMerged));
          }
        };

    while (iterator.hasNext()) {
      WindowedValue<InputT> e = iterator.next();

      @SuppressWarnings("unchecked")
      Collection<W> windows = (Collection<W>) e.getWindows();
      for (W w : windows) {
        Instant timestamp = minTimestamps.get(w);
        if (timestamp == null || timestamp.compareTo(e.getTimestamp()) > 0) {
          minTimestamps.put(w, e.getTimestamp());
        } else {
          minTimestamps.put(w, timestamp);
        }

        AccumT accum = accumulators.get(w);
        checkState((timestamp == null && accum == null) || (timestamp != null && accum != null));
        if (accum == null) {
          accum = combineFn.createAccumulator(key);
          liveWindows.add(w);
        }
        accum = combineFn.addInput(key, accum, e.getValue());
        accumulators.put(w, accum);
      }

      windowFn.mergeWindows(mergeContext);

      while (!liveWindows.isEmpty()
          && liveWindows.peek().maxTimestamp().isBefore(e.getTimestamp())) {
        closeWindow(key, liveWindows.poll(), accumulators, minTimestamps, c);
      }
    }

    // To have gotten here, we've either not had any elements added, or we've only run merge
    // and then closed windows. We don't need to retry merging.
    while (!liveWindows.isEmpty()) {
      closeWindow(key, liveWindows.poll(), accumulators, minTimestamps, c);
    }
  }
 @Override
 public void outputWithTimestamp(OutputT output, Instant timestamp) {
   synchronized (MultiThreadedIntraBundleProcessingDoFn.this) {
     context.outputWithTimestamp(output, timestamp);
   }
 }
 @Override
 public void output(OutputT output) {
   synchronized (MultiThreadedIntraBundleProcessingDoFn.this) {
     context.output(output);
   }
 }
 @Override
 public void processElement(ProcessContext c) throws Exception {
   c.output(c.element());
 }
 @Override
 public PaneInfo pane() {
   return context.pane();
 }
 @Override
 public InputT element() {
   return context.element();
 }
 @Override
 public WindowingInternals<InputT, OutputT> windowingInternals() {
   return context.windowingInternals();
 }
 @Override
 public <T> void sideOutput(TupleTag<T> tag, T output) {
   synchronized (MultiThreadedIntraBundleProcessingDoFn.this) {
     context.sideOutput(tag, output);
   }
 }
 @Override
 public PipelineOptions getPipelineOptions() {
   return context.getPipelineOptions();
 }