@Override
protected List<OUT> executeOnCollections(
List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
MapFunction<IN, OUT> function = this.userFunction.getUserCodeObject();
FunctionUtils.setFunctionRuntimeContext(function, ctx);
FunctionUtils.openFunction(function, this.parameters);
ArrayList<OUT> result = new ArrayList<OUT>(inputData.size());
TypeSerializer<IN> inSerializer =
getOperatorInfo().getInputType().createSerializer(executionConfig);
TypeSerializer<OUT> outSerializer =
getOperatorInfo().getOutputType().createSerializer(executionConfig);
for (IN element : inputData) {
IN inCopy = inSerializer.copy(element);
OUT out = function.map(inCopy);
result.add(outSerializer.copy(out));
}
FunctionUtils.closeFunction(function);
return result;
}
private void restoreTimers(DataInputView in) throws IOException {
int numWatermarkTimers = in.readInt();
watermarkTimers = new HashSet<>(numWatermarkTimers);
watermarkTimersQueue = new PriorityQueue<>(Math.max(numWatermarkTimers, 1));
for (int i = 0; i < numWatermarkTimers; i++) {
K key = keySerializer.deserialize(in);
W window = windowSerializer.deserialize(in);
long timestamp = in.readLong();
Timer<K, W> timer = new Timer<>(timestamp, key, window);
watermarkTimers.add(timer);
watermarkTimersQueue.add(timer);
}
int numProcessingTimeTimers = in.readInt();
processingTimeTimersQueue = new PriorityQueue<>(Math.max(numProcessingTimeTimers, 1));
processingTimeTimers = new HashSet<>();
for (int i = 0; i < numProcessingTimeTimers; i++) {
K key = keySerializer.deserialize(in);
W window = windowSerializer.deserialize(in);
long timestamp = in.readLong();
Timer<K, W> timer = new Timer<>(timestamp, key, window);
processingTimeTimersQueue.add(timer);
processingTimeTimers.add(timer);
}
int numProcessingTimeTimerTimestamp = in.readInt();
processingTimeTimerTimestamps = HashMultiset.create();
for (int i = 0; i < numProcessingTimeTimerTimestamp; i++) {
long timestamp = in.readLong();
int count = in.readInt();
processingTimeTimerTimestamps.add(timestamp, count);
}
}
@Override
public TaggedUnion<T1, T2> copy(TaggedUnion<T1, T2> from, TaggedUnion<T1, T2> reuse) {
if (from.isOne()) {
return TaggedUnion.one(oneSerializer.copy(from.getOne()));
} else {
return TaggedUnion.two(twoSerializer.copy(from.getTwo()));
}
}
@Override
public TaggedUnion<T1, T2> deserialize(DataInputView source) throws IOException {
byte tag = source.readByte();
if (tag == 1) {
return TaggedUnion.one(oneSerializer.deserialize(source));
} else {
return TaggedUnion.two(twoSerializer.deserialize(source));
}
}
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
byte tag = source.readByte();
target.writeByte(tag);
if (tag == 1) {
oneSerializer.copy(source, target);
} else {
twoSerializer.copy(source, target);
}
}
@Override
public void serialize(TaggedUnion<T1, T2> record, DataOutputView target) throws IOException {
if (record.isOne()) {
target.writeByte(1);
oneSerializer.serialize(record.getOne(), target);
} else {
target.writeByte(2);
twoSerializer.serialize(record.getTwo(), target);
}
}
@Override
@SuppressWarnings("unchecked")
public boolean equals(Object obj) {
if (obj instanceof UnionSerializer) {
UnionSerializer<T1, T2> other = (UnionSerializer<T1, T2>) obj;
return other.canEqual(this)
&& oneSerializer.equals(other.oneSerializer)
&& twoSerializer.equals(other.twoSerializer);
} else {
return false;
}
}
@Override
public StreamRecord<T> deserialize(StreamRecord<T> reuse, DataInputView source)
throws IOException {
T element = typeSerializer.deserialize(reuse.getValue(), source);
reuse.replace(element, 0);
return reuse;
}
@SuppressWarnings("unchecked")
public <X> void setInput2(MutableObjectIterator<X> input, TypeSerializer<X> serializer) {
this.input2 = input;
this.serializer2 =
new RuntimeSerializerFactory<X>(
serializer, (Class<X>) serializer.createInstance().getClass());
}
public ReusingBuildFirstHashMatchIterator(
MutableObjectIterator<V1> firstInput,
MutableObjectIterator<V2> secondInput,
TypeSerializer<V1> serializer1,
TypeComparator<V1> comparator1,
TypeSerializer<V2> serializer2,
TypeComparator<V2> comparator2,
TypePairComparator<V2, V1> pairComparator,
MemoryManager memManager,
IOManager ioManager,
AbstractInvokable ownerTask,
double memoryFraction)
throws MemoryAllocationException {
this.memManager = memManager;
this.firstInput = firstInput;
this.secondInput = secondInput;
this.probeSideSerializer = serializer2;
this.nextBuildSideObject = serializer1.createInstance();
this.tempBuildSideRecord = serializer1.createInstance();
this.hashJoin =
getHashJoin(
serializer1,
comparator1,
serializer2,
comparator2,
pairComparator,
memManager,
ioManager,
ownerTask,
memoryFraction);
}
@Override
public StreamRecord<T> createInstance() {
try {
return new StreamRecord<T>(typeSerializer.createInstance());
} catch (Exception e) {
throw new RuntimeException("Cannot instantiate StreamRecord.", e);
}
}
@Override
public void run() {
final MutableObjectIterator<T> input = this.input;
final TypeSerializer<T> serializer = this.serializer;
final SpillingBuffer buffer = this.buffer;
try {
T record = serializer.createInstance();
while (this.running && ((record = input.next(record)) != null)) {
serializer.serialize(record, buffer);
}
TempBarrier.this.writingDone();
} catch (Throwable t) {
TempBarrier.this.setException(t);
}
}
@Override
public void run() throws Exception {
// cache references on the stack
final MutableObjectIterator<T> input = this.taskContext.getInput(0);
final Collector<T> output = this.taskContext.getOutputCollector();
if (objectReuseEnabled) {
T record = this.taskContext.<T>getInputSerializer(0).getSerializer().createInstance();
while (this.running && ((record = input.next(record)) != null)) {
output.collect(record);
}
} else {
T record;
TypeSerializer<T> serializer = this.taskContext.<T>getInputSerializer(0).getSerializer();
while (this.running && ((record = input.next(serializer.createInstance())) != null)) {
output.collect(record);
}
}
}
private void snapshotTimers(DataOutputView out) throws IOException {
out.writeInt(watermarkTimersQueue.size());
for (Timer<K, W> timer : watermarkTimersQueue) {
keySerializer.serialize(timer.key, out);
windowSerializer.serialize(timer.window, out);
out.writeLong(timer.timestamp);
}
out.writeInt(processingTimeTimers.size());
for (Timer<K, W> timer : processingTimeTimers) {
keySerializer.serialize(timer.key, out);
windowSerializer.serialize(timer.window, out);
out.writeLong(timer.timestamp);
}
out.writeInt(processingTimeTimerTimestamps.entrySet().size());
for (Multiset.Entry<Long> timerTimestampCounts : processingTimeTimerTimestamps.entrySet()) {
out.writeLong(timerTimestampCounts.getElement());
out.writeInt(timerTimestampCounts.getCount());
}
}
@Override
public boolean equals(Object obj) {
if (obj instanceof GenericArraySerializer) {
GenericArraySerializer<?> other = (GenericArraySerializer<?>) obj;
return other.canEqual(this)
&& componentClass == other.componentClass
&& componentSerializer.equals(other.componentSerializer);
} else {
return false;
}
}
public FetchingIterator(
TypeSerializer<T> serializer,
MutableObjectIterator<Tuple> tupleInput,
SeekableFileChannelInputView recordsInputs,
TypeSerializer<Tuple> tupleSerializer,
int pointerPos) {
this.serializer = serializer;
this.tupleInput = tupleInput;
this.recordsInputs = recordsInputs;
this.pointerPos = pointerPos;
this.value = tupleSerializer.createInstance();
}
@Override
public T next() throws IOException {
Tuple value = tupleInput.next(this.value);
if (value != null) {
this.value = value;
long pointer = value.<Long>getField(pointerPos);
recordsInputs.seek(pointer);
return serializer.deserialize(recordsInputs);
} else {
return null;
}
}
@Override
public void serialize(C[] value, DataOutputView target) throws IOException {
target.writeInt(value.length);
for (int i = 0; i < value.length; i++) {
C val = value[i];
if (val == null) {
target.writeBoolean(false);
} else {
target.writeBoolean(true);
componentSerializer.serialize(val, target);
}
}
}
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
int len = source.readInt();
target.writeInt(len);
for (int i = 0; i < len; i++) {
boolean isNonNull = source.readBoolean();
target.writeBoolean(isNonNull);
if (isNonNull) {
componentSerializer.copy(source, target);
}
}
}
public boolean isCompatibleWith(RegisteredBackendStateMetaInfo<?, ?> other) {
if (this == other) {
return true;
}
if (null == other) {
return false;
}
if (!stateType.equals(StateDescriptor.Type.UNKNOWN)
&& !other.stateType.equals(StateDescriptor.Type.UNKNOWN)
&& !stateType.equals(other.stateType)) {
return false;
}
if (!name.equals(other.getName())) {
return false;
}
return namespaceSerializer.isCompatibleWith(other.namespaceSerializer)
&& stateSerializer.isCompatibleWith(other.stateSerializer);
}
@Override
public void run() throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug(
this.taskContext.formatLogString("AllReduce preprocessing done. Running Reducer code."));
}
final GenericReduce<T> stub = this.taskContext.getStub();
final MutableObjectIterator<T> input = this.input;
final TypeSerializer<T> serializer = this.serializer;
T val1 = serializer.createInstance();
if ((val1 = input.next(val1)) == null) {
return;
}
T val2;
while (running && (val2 = input.next(serializer.createInstance())) != null) {
val1 = stub.reduce(val1, val2);
}
this.taskContext.getOutputCollector().collect(val1);
}
@Override
public C[] deserialize(DataInputView source) throws IOException {
int len = source.readInt();
C[] array = create(len);
for (int i = 0; i < len; i++) {
boolean isNonNull = source.readBoolean();
if (isNonNull) {
array[i] = componentSerializer.deserialize(source);
} else {
array[i] = null;
}
}
return array;
}
private SpillingResettableIterator(
Iterator<T> input,
TypeSerializer<T> serializer,
MemoryManager memoryManager,
IOManager ioManager,
List<MemorySegment> memory,
boolean releaseMemOnClose) {
this.memoryManager = memoryManager;
this.input = input;
this.instance = serializer.createInstance();
this.serializer = serializer;
this.memorySegments = memory;
this.releaseMemoryOnClose = releaseMemOnClose;
if (LOG.isDebugEnabled()) {
LOG.debug(
"Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
}
this.buffer =
new SpillingBuffer(
ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
protected IN copy(IN record) {
return objectSerializer.copy(record);
}
protected void writeKeyAndNamespace(DataOutputView out) throws IOException {
keySerializer.serialize(currentKey, out);
out.writeByte(42);
namespaceSerializer.serialize(currentNamespace, out);
}
@Override
public StreamRecord<T> copy(StreamRecord<T> from, StreamRecord<T> reuse) {
reuse.replace(typeSerializer.copy(from.getValue(), reuse.getValue()), 0);
return reuse;
}
@Override
public StreamRecord<T> copy(StreamRecord<T> from) {
return new StreamRecord<T>(typeSerializer.copy(from.getValue()), from.getTimestamp());
}
@Override
public int getLength() {
return typeSerializer.getLength();
}
@Override
public StreamRecordSerializer<T> duplicate() {
TypeSerializer<T> serializerCopy = typeSerializer.duplicate();
return serializerCopy == typeSerializer ? this : new StreamRecordSerializer<T>(serializerCopy);
}
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
typeSerializer.copy(source, target);
}