@SuppressWarnings("ConstantConditions")
JavaDStream<WindowedValue<T>> getDStream() {
if (dStream == null) {
WindowedValue.ValueOnlyWindowedValueCoder<T> windowCoder =
WindowedValue.getValueOnlyCoder(coder);
// create the DStream from queue
Queue<JavaRDD<WindowedValue<T>>> rddQueue = new LinkedBlockingQueue<>();
JavaRDD<WindowedValue<T>> lastRDD = null;
for (Iterable<T> v : values) {
Iterable<WindowedValue<T>> windowedValues =
Iterables.transform(v, WindowingHelpers.<T>windowValueFunction());
JavaRDD<WindowedValue<T>> rdd =
jssc.sc()
.parallelize(CoderHelpers.toByteArrays(windowedValues, windowCoder))
.map(CoderHelpers.fromByteFunction(windowCoder));
rddQueue.offer(rdd);
lastRDD = rdd;
}
// create DStream from queue, one at a time,
// with last as default in case batches repeat (graceful stops for example).
// if the stream is empty, avoid creating a default empty RDD.
// mainly for unit test so no reason to have this configurable.
dStream =
lastRDD != null
? jssc.queueStream(rddQueue, true, lastRDD)
: jssc.queueStream(rddQueue, true);
}
return dStream;
}
@SuppressWarnings("ConstantConditions")
public JavaRDD<WindowedValue<T>> getRDD() {
if (rdd == null) {
WindowedValue.ValueOnlyWindowedValueCoder<T> windowCoder =
WindowedValue.getValueOnlyCoder(coder);
rdd =
jsc.parallelize(CoderHelpers.toByteArrays(windowedValues, windowCoder))
.map(CoderHelpers.fromByteFunction(windowCoder));
}
return rdd;
}
