@Override
public void processElement(ProcessContext c) {
KV<K, Iterable<InputT>> kv = c.element();
K key = kv.getKey();
c.output(KV.of(key, this.combineFn.apply(key, kv.getValue())));
}
@Override
public void processElement(ProcessContext c) {
String row =
c.element().getKey() + " - " + c.element().getValue() + " @ " + c.timestamp().toString();
System.out.println(row);
c.output(row);
}
@Override
public void processElement(ProcessContext c) {
KV<K, AccumT> kv = c.element();
K key = kv.getKey();
OutputT output = this.combineFn.extractOutput(key, kv.getValue());
c.output(KV.of(key, output));
}
@Override
public void processElement(ProcessContext c) {
KV<K, Iterable<AccumT>> kv = c.element();
K key = kv.getKey();
AccumT accum = this.combineFn.mergeAccumulators(key, kv.getValue());
c.output(KV.of(key, accum));
}
@Override
public void processElement(ProcessContext c) {
KV<K, Iterable<InputT>> kv = c.element();
K key = kv.getKey();
AccumT accum = this.combineFn.createAccumulator(key);
for (InputT input : kv.getValue()) {
accum = this.combineFn.addInput(key, accum, input);
}
c.output(KV.of(key, accum));
}
@Override
public void processElement(ProcessContext c) throws Exception {
String dest = c.element();
logger.info("Saving to " + dest);
Iterable<GATKRead> reads = c.sideInput(iterableView);
OutputStream outputStream = BucketUtils.createFile(dest, c.getPipelineOptions());
try (SAMFileWriter writer =
new SAMFileWriterFactory().makeBAMWriter(header, false, outputStream)) {
for (GATKRead r : reads) {
final SAMRecord sr = r.convertToSAMRecord(header);
writer.addAlignment(sr);
}
}
}
@Override
public void processElement(ProcessContext c) throws IOException {
PubsubMessage pubsubMessage = new PubsubMessage();
pubsubMessage.encodeData(c.element().getBytes());
if (timestampLabelKey != null) {
Label timestampLabel = new Label();
timestampLabel.setKey(timestampLabelKey);
timestampLabel.setNumValue(c.timestamp().getMillis());
pubsubMessage.setLabel(ImmutableList.of(timestampLabel));
}
PublishRequest publishRequest = new PublishRequest();
publishRequest.setTopic(outputTopic).setMessage(pubsubMessage);
this.pubsub.topics().publish(publishRequest).execute();
}
@Override
public void processElement(ProcessContext c) {
if (c.element().trim().isEmpty()) {
emptyLines.addValue(1L);
}
// Split the line into words.
String[] words = c.element().split("[^a-zA-Z']+");
// Output each word encountered into the output PCollection.
for (String word : words) {
if (!word.isEmpty()) {
c.output(word);
}
}
}
@ProcessElement
public void processElement(ProcessContext ctx) throws Exception {
// Need to copy the document because mongoCollection.insertMany() will mutate it
// before inserting (will assign an id).
batch.add(new Document(ctx.element()));
if (batch.size() >= spec.batchSize()) {
finishBundle(ctx);
}
}
private void closeWindow(
K key, W w, Map<W, AccumT> accumulators, Map<W, Instant> minTimestamps, ProcessContext c) {
AccumT accum = accumulators.remove(w);
Instant timestamp = minTimestamps.remove(w);
checkState(accum != null && timestamp != null);
c.windowingInternals()
.outputWindowedValue(
KV.of(key, combineFn.extractOutput(key, accum)),
timestamp,
Arrays.asList(w),
PaneInfo.ON_TIME_AND_ONLY_FIRING);
}
@Override
public void processElement(ProcessContext c) throws Exception {
KeyedWorkItem<K, InputT> element = c.element();
K key = c.element().key();
TimerInternals timerInternals = c.windowingInternals().timerInternals();
StateInternals<K> stateInternals = stateInternalsFactory.stateInternalsForKey(key);
ReduceFnRunner<K, InputT, OutputT, W> reduceFnRunner =
new ReduceFnRunner<>(
key,
windowingStrategy,
ExecutableTriggerStateMachine.create(
TriggerStateMachines.stateMachineForTrigger(windowingStrategy.getTrigger())),
stateInternals,
timerInternals,
WindowingInternalsAdapters.outputWindowedValue(c.windowingInternals()),
WindowingInternalsAdapters.sideInputReader(c.windowingInternals()),
droppedDueToClosedWindow,
reduceFn,
c.getPipelineOptions());
reduceFnRunner.processElements(element.elementsIterable());
for (TimerData timer : element.timersIterable()) {
reduceFnRunner.onTimer(timer);
}
reduceFnRunner.persist();
}
@Override
public void processElement(ProcessContext c) {
KV<K, Iterable<RawUnionValue>> e = c.element();
c.output(KV.of(e.getKey(), new CoGbkResult(schema, e.getValue())));
}
@Override
public void processElement(ProcessContext c) {
KV<K, ?> e = c.element();
c.output(KV.of(e.getKey(), new RawUnionValue(index, e.getValue())));
}
/**
* Execute this process.
*
* @param conn the database connection
* @throws ServletException
*/
public String execute(ConnectionProvider conn) throws ServletException {
Process process = null;
try {
process = bundle.getProcessClass().newInstance();
} catch (final Exception e) {
log.error(e.getMessage(), e);
throw new ServletException(e.getMessage(), e);
}
final String requestId = SequenceIdData.getUUID();
String status = SCHEDULED;
final ProcessContext ctx = bundle.getContext();
ProcessRequestData.insert(
conn,
ctx.getOrganization(),
ctx.getClient(),
ctx.getUser(),
ctx.getUser(),
requestId,
bundle.getProcessId(),
ctx.getUser(),
status,
"Direct",
ctx.toString(),
"",
null,
null,
null,
null);
final String executionId = SequenceIdData.getUUID();
final long startTime = System.currentTimeMillis();
long endTime = startTime;
status = PROCESSING;
ProcessRunData.insert(
conn,
ctx.getOrganization(),
ctx.getClient(),
ctx.getUser(),
ctx.getUser(),
executionId,
status,
null,
bundle.getLog(),
requestId);
try {
log.debug("Calling execute on process " + requestId);
process.execute(bundle);
endTime = System.currentTimeMillis();
status = SUCCESS;
} catch (final Exception e) {
endTime = System.currentTimeMillis();
status = ERROR;
log.error("Process " + requestId + " threw an Exception: " + e.getMessage(), e);
throw new ServletException(e);
} finally {
final String duration = ProcessMonitor.getDuration(endTime - startTime);
ProcessRequestData.update(conn, COMPLETE, requestId);
ProcessRunData.update(conn, ctx.getUser(), status, duration, bundle.getLog(), executionId);
}
return executionId;
}
@Override
public <T> T sideInput(PCollectionView<T> view) {
return context.sideInput(view);
}
@Override
protected <AggInputT, AggOutputT> Aggregator<AggInputT, AggOutputT> createAggregatorInternal(
String name, CombineFn<AggInputT, ?, AggOutputT> combiner) {
return context.createAggregatorInternal(name, combiner);
}
@Override
public BoundedWindow window() {
return context.window();
}
@Override
protected void preProcess(ProcessContext ctx) {
adTableId = (Integer) ctx.getRowMap().get("adTableId");
adTabId = (Integer) ctx.getRowMap().get("adTabId");
}
@Override
public <T> void sideOutputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
synchronized (MultiThreadedIntraBundleProcessingDoFn.this) {
context.sideOutputWithTimestamp(tag, output, timestamp);
}
}
@Override
public Instant timestamp() {
return context.timestamp();
}
@Override
public void processElement(ProcessContext c) throws Exception {
final K key = c.element().getKey();
Iterator<WindowedValue<InputT>> iterator = c.element().getValue().iterator();
final PriorityQueue<W> liveWindows =
new PriorityQueue<>(
11,
new Comparator<BoundedWindow>() {
@Override
public int compare(BoundedWindow w1, BoundedWindow w2) {
return Long.signum(w1.maxTimestamp().getMillis() - w2.maxTimestamp().getMillis());
}
});
final Map<W, AccumT> accumulators = Maps.newHashMap();
final Map<W, Instant> minTimestamps = Maps.newHashMap();
WindowFn<Object, W>.MergeContext mergeContext =
new CombiningMergeContext() {
@Override
public Collection<W> windows() {
return liveWindows;
}
@Override
public void merge(Collection<W> toBeMerged, W mergeResult) throws Exception {
List<AccumT> accumsToBeMerged = new ArrayList<>(toBeMerged.size());
Instant minTimestamp = null;
for (W window : toBeMerged) {
accumsToBeMerged.add(accumulators.remove(window));
Instant timestampToBeMerged = minTimestamps.remove(window);
if (minTimestamp == null
|| (timestampToBeMerged != null && timestampToBeMerged.isBefore(minTimestamp))) {
minTimestamp = timestampToBeMerged;
}
}
liveWindows.removeAll(toBeMerged);
minTimestamps.put(mergeResult, minTimestamp);
liveWindows.add(mergeResult);
accumulators.put(mergeResult, combineFn.mergeAccumulators(key, accumsToBeMerged));
}
};
while (iterator.hasNext()) {
WindowedValue<InputT> e = iterator.next();
@SuppressWarnings("unchecked")
Collection<W> windows = (Collection<W>) e.getWindows();
for (W w : windows) {
Instant timestamp = minTimestamps.get(w);
if (timestamp == null || timestamp.compareTo(e.getTimestamp()) > 0) {
minTimestamps.put(w, e.getTimestamp());
} else {
minTimestamps.put(w, timestamp);
}
AccumT accum = accumulators.get(w);
checkState((timestamp == null && accum == null) || (timestamp != null && accum != null));
if (accum == null) {
accum = combineFn.createAccumulator(key);
liveWindows.add(w);
}
accum = combineFn.addInput(key, accum, e.getValue());
accumulators.put(w, accum);
}
windowFn.mergeWindows(mergeContext);
while (!liveWindows.isEmpty()
&& liveWindows.peek().maxTimestamp().isBefore(e.getTimestamp())) {
closeWindow(key, liveWindows.poll(), accumulators, minTimestamps, c);
}
}
// To have gotten here, we've either not had any elements added, or we've only run merge
// and then closed windows. We don't need to retry merging.
while (!liveWindows.isEmpty()) {
closeWindow(key, liveWindows.poll(), accumulators, minTimestamps, c);
}
}
@Override
public void outputWithTimestamp(OutputT output, Instant timestamp) {
synchronized (MultiThreadedIntraBundleProcessingDoFn.this) {
context.outputWithTimestamp(output, timestamp);
}
}
@Override
public void output(OutputT output) {
synchronized (MultiThreadedIntraBundleProcessingDoFn.this) {
context.output(output);
}
}
@Override
public void processElement(ProcessContext c) throws Exception {
c.output(c.element());
}
@Override
public PaneInfo pane() {
return context.pane();
}
@Override
public InputT element() {
return context.element();
}
@Override
public WindowingInternals<InputT, OutputT> windowingInternals() {
return context.windowingInternals();
}
@Override
public <T> void sideOutput(TupleTag<T> tag, T output) {
synchronized (MultiThreadedIntraBundleProcessingDoFn.this) {
context.sideOutput(tag, output);
}
}
@Override
public PipelineOptions getPipelineOptions() {
return context.getPipelineOptions();
}