@Override
public void copy(DataInputView in, DataOutputView target) throws IOException {
int len = in.readUnsignedByte();
target.writeByte(len);
if (len >= HIGH_BIT) {
int shift = 7;
int curr;
len = len & 0x7f;
while ((curr = in.readUnsignedByte()) >= HIGH_BIT) {
len |= (curr & 0x7f) << shift;
shift += 7;
target.writeByte(curr);
}
len |= curr << shift;
target.writeByte(curr);
}
for (int i = 0; i < len; i++) {
int c = in.readUnsignedByte();
target.writeByte(c);
while (c >= HIGH_BIT) {
c = in.readUnsignedByte();
target.writeByte(c);
}
}
}
public static boolean copyBigInteger(DataInputView source, DataOutputView target)
throws IOException {
final int len = source.readInt();
target.writeInt(len);
if (len > 4) {
target.write(source, len - 4);
}
return len == 0; // returns true if the copied record was null
}
@Override
public void write(final DataOutputView out) throws IOException {
out.writeInt(this.inetAddress.getAddress().length);
out.write(this.inetAddress.getAddress());
out.writeInt(this.dataPort);
StringUtils.writeNullableString(fqdnHostName, out);
StringUtils.writeNullableString(hostName, out);
}
@Override
public void serialize(TaggedUnion<T1, T2> record, DataOutputView target) throws IOException {
if (record.isOne()) {
target.writeByte(1);
oneSerializer.serialize(record.getOne(), target);
} else {
target.writeByte(2);
twoSerializer.serialize(record.getTwo(), target);
}
}
private OperatorStateHandle writeAllTestKeyGroups(
OperatorStateCheckpointOutputStream stream, int numPartitions) throws Exception {
DataOutputView dov = new DataOutputViewStreamWrapper(stream);
for (int i = 0; i < numPartitions; ++i) {
Assert.assertEquals(i, stream.getNumberOfPartitions());
stream.startNewPartition();
dov.writeInt(i);
}
return stream.closeAndGetHandle();
}
@Override
public void serialize(C[] value, DataOutputView target) throws IOException {
target.writeInt(value.length);
for (int i = 0; i < value.length; i++) {
C val = value[i];
if (val == null) {
target.writeBoolean(false);
} else {
target.writeBoolean(true);
componentSerializer.serialize(val, target);
}
}
}
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
int len = source.readInt();
target.writeInt(len);
for (int i = 0; i < len; i++) {
boolean isNonNull = source.readBoolean();
target.writeBoolean(isNonNull);
if (isNonNull) {
componentSerializer.copy(source, target);
}
}
}
@Override
public void write(final DataOutputView out) throws IOException {
out.writeInt(this.list.size());
for (final V value : this.list) {
value.write(out);
}
}
@Override
public void write(DataOutputView out) throws IOException {
if (uri == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
StringUtils.writeNullableString(uri.getScheme(), out);
StringUtils.writeNullableString(uri.getUserInfo(), out);
StringUtils.writeNullableString(uri.getHost(), out);
out.writeInt(uri.getPort());
StringUtils.writeNullableString(uri.getPath(), out);
StringUtils.writeNullableString(uri.getQuery(), out);
StringUtils.writeNullableString(uri.getFragment(), out);
}
}
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
byte tag = source.readByte();
target.writeByte(tag);
if (tag == 1) {
oneSerializer.copy(source, target);
} else {
twoSerializer.copy(source, target);
}
}
@Override
public void write(final DataOutputView out) throws IOException {
int len = this.len;
// write the length, variable-length encoded
while (len >= HIGH_BIT) {
out.write(len | HIGH_BIT);
len >>>= 7;
}
out.write(len);
// write the char data, variable length encoded
for (int i = 0; i < this.len; i++) {
int c = this.value[i];
while (c >= HIGH_BIT) {
out.write(c | HIGH_BIT);
c >>>= 7;
}
out.write(c);
}
}
private void snapshotTimers(DataOutputView out) throws IOException {
out.writeInt(watermarkTimersQueue.size());
for (Timer<K, W> timer : watermarkTimersQueue) {
keySerializer.serialize(timer.key, out);
windowSerializer.serialize(timer.window, out);
out.writeLong(timer.timestamp);
}
out.writeInt(processingTimeTimers.size());
for (Timer<K, W> timer : processingTimeTimers) {
keySerializer.serialize(timer.key, out);
windowSerializer.serialize(timer.window, out);
out.writeLong(timer.timestamp);
}
out.writeInt(processingTimeTimerTimestamps.entrySet().size());
for (Multiset.Entry<Long> timerTimestampCounts : processingTimeTimerTimestamps.entrySet()) {
out.writeLong(timerTimestampCounts.getElement());
out.writeInt(timerTimestampCounts.getCount());
}
}
private void writeKVStateMetaData() throws IOException, InterruptedException {
// write number of k/v states
outputView.writeInt(stateBackend.kvStateInformation.size());
int kvStateId = 0;
// iterate all column families, where each column family holds one k/v state, to write the
// metadata
for (Map.Entry<String, Tuple2<ColumnFamilyHandle, StateDescriptor<?, ?>>> column :
stateBackend.kvStateInformation.entrySet()) {
// be cooperative and check for interruption from time to time in the hot loop
checkInterrupted();
// write StateDescriptor for this k/v state
InstantiationUtil.serializeObject(outStream, column.getValue().f1);
// retrieve iterator for this k/v states
readOptions = new ReadOptions();
readOptions.setSnapshot(snapshot);
RocksIterator iterator = stateBackend.db.newIterator(column.getValue().f0, readOptions);
kvStateIterators.add(new Tuple2<>(iterator, kvStateId));
++kvStateId;
}
}
public static void writeBigInteger(BigInteger record, DataOutputView target) throws IOException {
// null value support
if (record == null) {
target.writeInt(0);
return;
}
// fast paths for 0, 1, 10
// only reference equality is checked because equals would be too expensive
else if (record == BigInteger.ZERO) {
target.writeInt(1);
return;
} else if (record == BigInteger.ONE) {
target.writeInt(2);
return;
} else if (record == BigInteger.TEN) {
target.writeInt(3);
return;
}
// default
final byte[] bytes = record.toByteArray();
// the length we write is offset by four, because null and short-paths for ZERO, ONE, and TEN
target.writeInt(bytes.length + 4);
target.write(bytes);
}
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
target.writeChar(source.readChar());
}
@Override
public void serialize(Character record, DataOutputView target) throws IOException {
target.writeChar(record.charValue());
}
protected void writeKeyAndNamespace(DataOutputView out) throws IOException {
keySerializer.serialize(currentKey, out);
out.writeByte(42);
namespaceSerializer.serialize(currentNamespace, out);
}
private void writeKVStateData() throws IOException, InterruptedException {
byte[] previousKey = null;
byte[] previousValue = null;
List<Tuple2<RocksIterator, Integer>> kvStateIteratorsHandover = this.kvStateIterators;
this.kvStateIterators = null;
// Here we transfer ownership of RocksIterators to the RocksDBMergeIterator
try (RocksDBMergeIterator mergeIterator =
new RocksDBMergeIterator(kvStateIteratorsHandover, stateBackend.keyGroupPrefixBytes)) {
// preamble: setup with first key-group as our lookahead
if (mergeIterator.isValid()) {
// begin first key-group by recording the offset
keyGroupRangeOffsets.setKeyGroupOffset(mergeIterator.keyGroup(), outStream.getPos());
// write the k/v-state id as metadata
// TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
outputView.writeShort(mergeIterator.kvStateId());
previousKey = mergeIterator.key();
previousValue = mergeIterator.value();
mergeIterator.next();
}
// main loop: write k/v pairs ordered by (key-group, kv-state), thereby tracking key-group
// offsets.
while (mergeIterator.isValid()) {
assert (!hasMetaDataFollowsFlag(previousKey));
// set signal in first key byte that meta data will follow in the stream after this k/v
// pair
if (mergeIterator.isNewKeyGroup() || mergeIterator.isNewKeyValueState()) {
// be cooperative and check for interruption from time to time in the hot loop
checkInterrupted();
setMetaDataFollowsFlagInKey(previousKey);
}
writeKeyValuePair(previousKey, previousValue);
// write meta data if we have to
if (mergeIterator.isNewKeyGroup()) {
// TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
outputView.writeShort(END_OF_KEY_GROUP_MARK);
// begin new key-group
keyGroupRangeOffsets.setKeyGroupOffset(mergeIterator.keyGroup(), outStream.getPos());
// write the kev-state
// TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
outputView.writeShort(mergeIterator.kvStateId());
} else if (mergeIterator.isNewKeyValueState()) {
// write the k/v-state
// TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
outputView.writeShort(mergeIterator.kvStateId());
}
// request next k/v pair
previousKey = mergeIterator.key();
previousValue = mergeIterator.value();
mergeIterator.next();
}
}
// epilogue: write last key-group
if (previousKey != null) {
assert (!hasMetaDataFollowsFlag(previousKey));
setMetaDataFollowsFlagInKey(previousKey);
writeKeyValuePair(previousKey, previousValue);
// TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
outputView.writeShort(END_OF_KEY_GROUP_MARK);
}
}
@Override
public void write(DataOutputView out) throws IOException {
out.writeInt(splitNumber);
}
@Override
public void write(DataOutputView out) throws IOException {
out.writeChar(this.value);
}
@Override
public void write(DataOutputView out) throws IOException {
out.writeLong(this.lowerPart);
out.writeLong(this.upperPart);
}