@Override public void run(SourceContext<Integer> ctx) throws Exception { final Object lock = ctx.getCheckpointLock(); final int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask(); while (running) { if (counter < numberElements) { synchronized (lock) { for (int value = subtaskIndex; value < numberKeys; value += getRuntimeContext().getNumberOfParallelSubtasks()) { ctx.collect(value); } counter++; } } else { if (terminateAfterEmission) { running = false; } else { Thread.sleep(100); } } } }
@Override public void run(SourceContext<String> ctx) throws Exception { final Object lockingObject = ctx.getCheckpointLock(); while (isRunning && index.value() < numElements) { char first = (char) ((index.value() % 40) + 40); stringBuilder.setLength(0); stringBuilder.append(first); String result = randomString(stringBuilder, rnd); synchronized (lockingObject) { index.update(index.value() + step); ctx.collect(result); } } }
@Override public void run(SourceContext<Integer> ctx) throws Exception { final Object lock = ctx.getCheckpointLock(); while (running) { synchronized (lock) { ++counter; ctx.collect(1); } Thread.sleep(2); if (counter == 10) { workStartedLatch.countDown(); } if (counter >= 500) { break; } } }
@Override public void run(SourceContext<Long> ctx) throws Exception { final Object checkpointLock = ctx.getCheckpointLock(); RuntimeContext context = getRuntimeContext(); final long stepSize = context.getNumberOfParallelSubtasks(); final long congruence = start + context.getIndexOfThisSubtask(); final long toCollect = ((end - start + 1) % stepSize > (congruence - start)) ? ((end - start + 1) / stepSize + 1) : ((end - start + 1) / stepSize); while (isRunning && collected < toCollect) { synchronized (checkpointLock) { ctx.collect(collected * stepSize + congruence); collected++; } } }
@Override public void run(SourceContext<OUT> ctx) throws Exception { if (iteratorToRead == null) { throw new IllegalStateException("Kafka iterator not initialized properly."); } final Object checkpointLock = ctx.getCheckpointLock(); while (running && iteratorToRead.hasNext()) { MessageAndMetadata<byte[], byte[]> message = iteratorToRead.next(); if (lastOffsets.getState()[message.partition()] >= message.offset()) { LOG.info( "Skipping message with offset {} from partition {}", message.offset(), message.partition()); continue; } OUT next = deserializationSchema.deserialize(message.message()); if (deserializationSchema.isEndOfStream(next)) { LOG.info("DeserializationSchema signaled end of stream for this source"); break; } // make the state update and the element emission atomic synchronized (checkpointLock) { lastOffsets.getState()[message.partition()] = message.offset(); ctx.collect(next); } if (LOG.isTraceEnabled()) { LOG.trace( "Processed record with offset {} from partition {}", message.offset(), message.partition()); } } }
@Override public void run(SourceContext<String> ctx) throws Exception { while (running) { ctx.collect("hello"); } }