@Override
public void copy(DataInputView in, DataOutputView target) throws IOException {
int len = in.readUnsignedByte();
target.writeByte(len);
if (len >= HIGH_BIT) {
int shift = 7;
int curr;
len = len & 0x7f;
while ((curr = in.readUnsignedByte()) >= HIGH_BIT) {
len |= (curr & 0x7f) << shift;
shift += 7;
target.writeByte(curr);
}
len |= curr << shift;
target.writeByte(curr);
}
for (int i = 0; i < len; i++) {
int c = in.readUnsignedByte();
target.writeByte(c);
while (c >= HIGH_BIT) {
c = in.readUnsignedByte();
target.writeByte(c);
}
}
}
@Override
public void restoreState(StreamTaskState taskState, long recoveryTimestamp) throws Exception {
super.restoreState(taskState, recoveryTimestamp);
final ClassLoader userClassloader = getUserCodeClassloader();
@SuppressWarnings("unchecked")
StateHandle<DataInputView> inputState =
(StateHandle<DataInputView>) taskState.getOperatorState();
DataInputView in = inputState.getState(userClassloader);
int numKeys = in.readInt();
this.windows = new HashMap<>(numKeys);
this.processingTimeTimers = new HashMap<>();
this.watermarkTimers = new HashMap<>();
for (int i = 0; i < numKeys; i++) {
int numWindows = in.readInt();
for (int j = 0; j < numWindows; j++) {
Context context = new Context(in, userClassloader);
Map<W, Context> keyWindows = windows.get(context.key);
if (keyWindows == null) {
keyWindows = new HashMap<>(numWindows);
windows.put(context.key, keyWindows);
}
keyWindows.put(context.window, context);
}
}
}
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
int len = source.readInt();
target.writeInt(len);
for (int i = 0; i < len; i++) {
boolean isNonNull = source.readBoolean();
target.writeBoolean(isNonNull);
if (isNonNull) {
componentSerializer.copy(source, target);
}
}
}
/**
* Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current
* state handle
*
* @throws IOException
* @throws ClassNotFoundException
* @throws RocksDBException
*/
private void restoreKVStateMetaData()
throws IOException, ClassNotFoundException, RocksDBException {
// read number of k/v states
int numColumns = currentStateHandleInView.readInt();
// those two lists are aligned and should later have the same size!
currentStateHandleKVStateColumnFamilies = new ArrayList<>(numColumns);
// restore the empty columns for the k/v states through the metadata
for (int i = 0; i < numColumns; i++) {
StateDescriptor<?, ?> stateDescriptor =
InstantiationUtil.deserializeObject(
currentStateHandleInStream, rocksDBKeyedStateBackend.userCodeClassLoader);
Tuple2<ColumnFamilyHandle, StateDescriptor<?, ?>> columnFamily =
rocksDBKeyedStateBackend.kvStateInformation.get(stateDescriptor.getName());
if (null == columnFamily) {
ColumnFamilyDescriptor columnFamilyDescriptor =
new ColumnFamilyDescriptor(
stateDescriptor.getName().getBytes(), rocksDBKeyedStateBackend.columnOptions);
columnFamily =
new Tuple2<ColumnFamilyHandle, StateDescriptor<?, ?>>(
rocksDBKeyedStateBackend.db.createColumnFamily(columnFamilyDescriptor),
stateDescriptor);
rocksDBKeyedStateBackend.kvStateInformation.put(stateDescriptor.getName(), columnFamily);
}
currentStateHandleKVStateColumnFamilies.add(columnFamily.f0);
}
}
@Override
public C[] deserialize(DataInputView source) throws IOException {
int len = source.readInt();
C[] array = create(len);
for (int i = 0; i < len; i++) {
boolean isNonNull = source.readBoolean();
if (isNonNull) {
array[i] = componentSerializer.deserialize(source);
} else {
array[i] = null;
}
}
return array;
}
private void restoreTimers(DataInputView in) throws IOException {
int numWatermarkTimers = in.readInt();
watermarkTimers = new HashSet<>(numWatermarkTimers);
watermarkTimersQueue = new PriorityQueue<>(Math.max(numWatermarkTimers, 1));
for (int i = 0; i < numWatermarkTimers; i++) {
K key = keySerializer.deserialize(in);
W window = windowSerializer.deserialize(in);
long timestamp = in.readLong();
Timer<K, W> timer = new Timer<>(timestamp, key, window);
watermarkTimers.add(timer);
watermarkTimersQueue.add(timer);
}
int numProcessingTimeTimers = in.readInt();
processingTimeTimersQueue = new PriorityQueue<>(Math.max(numProcessingTimeTimers, 1));
processingTimeTimers = new HashSet<>();
for (int i = 0; i < numProcessingTimeTimers; i++) {
K key = keySerializer.deserialize(in);
W window = windowSerializer.deserialize(in);
long timestamp = in.readLong();
Timer<K, W> timer = new Timer<>(timestamp, key, window);
processingTimeTimersQueue.add(timer);
processingTimeTimers.add(timer);
}
int numProcessingTimeTimerTimestamp = in.readInt();
processingTimeTimerTimestamps = HashMultiset.create();
for (int i = 0; i < numProcessingTimeTimerTimestamp; i++) {
long timestamp = in.readLong();
int count = in.readInt();
processingTimeTimerTimestamps.add(timestamp, count);
}
}
@Override
public void read(DataInputView in) throws IOException {
final int addr_length = in.readInt();
byte[] address = new byte[addr_length];
in.readFully(address);
this.dataPort = in.readInt();
this.fqdnHostName = StringUtils.readNullableString(in);
this.hostName = StringUtils.readNullableString(in);
try {
this.inetAddress = InetAddress.getByAddress(address);
} catch (UnknownHostException e) {
throw new IOException("This lookup should never fail.", e);
}
}
@Override
public TaggedUnion<T1, T2> deserialize(DataInputView source) throws IOException {
byte tag = source.readByte();
if (tag == 1) {
return TaggedUnion.one(oneSerializer.deserialize(source));
} else {
return TaggedUnion.two(twoSerializer.deserialize(source));
}
}
public static BigInteger readBigInteger(DataInputView source) throws IOException {
final int len = source.readInt();
if (len < 4) {
switch (len) {
case 0:
return null;
case 1:
return BigInteger.ZERO;
case 2:
return BigInteger.ONE;
case 3:
return BigInteger.TEN;
}
}
final byte[] bytes = new byte[len - 4];
source.read(bytes);
return new BigInteger(bytes);
}
public static boolean copyBigInteger(DataInputView source, DataOutputView target)
throws IOException {
final int len = source.readInt();
target.writeInt(len);
if (len > 4) {
target.write(source, len - 4);
}
return len == 0; // returns true if the copied record was null
}
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
byte tag = source.readByte();
target.writeByte(tag);
if (tag == 1) {
oneSerializer.copy(source, target);
} else {
twoSerializer.copy(source, target);
}
}
/**
* Constructs a new {@code Context} by reading from a {@link DataInputView} that contains a
* serialized context that we wrote in {@link
* #writeToState(StateBackend.CheckpointStateOutputView)}
*/
@SuppressWarnings("unchecked")
protected Context(DataInputView in, ClassLoader userClassloader) throws Exception {
this.key = keySerializer.deserialize(in);
this.window = windowSerializer.deserialize(in);
this.watermarkTimer = in.readLong();
this.processingTimeTimer = in.readLong();
int stateSize = in.readInt();
byte[] stateData = new byte[stateSize];
in.read(stateData);
state = InstantiationUtil.deserializeObject(stateData, userClassloader);
this.windowBuffer = windowBufferFactory.create();
int numElements = in.readInt();
MultiplexingStreamRecordSerializer<IN> recordSerializer =
new MultiplexingStreamRecordSerializer<>(inputSerializer);
for (int i = 0; i < numElements; i++) {
windowBuffer.storeElement(recordSerializer.deserialize(in).<IN>asRecord());
}
}
@Override
public void read(DataInputView in) throws IOException {
final boolean isNotNull = in.readBoolean();
if (isNotNull) {
final String scheme = StringUtils.readNullableString(in);
final String userInfo = StringUtils.readNullableString(in);
final String host = StringUtils.readNullableString(in);
final int port = in.readInt();
final String path = StringUtils.readNullableString(in);
final String query = StringUtils.readNullableString(in);
final String fragment = StringUtils.readNullableString(in);
try {
uri = new URI(scheme, userInfo, host, port, path, query, fragment);
} catch (URISyntaxException e) {
throw new IOException("Error reconstructing URI", e);
}
}
}
private static void verifyRead(OperatorStateHandle fullHandle, int numPartitions)
throws IOException {
int count = 0;
try (FSDataInputStream in = fullHandle.openInputStream()) {
long[] offsets =
fullHandle
.getStateNameToPartitionOffsets()
.get(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);
Assert.assertNotNull(offsets);
DataInputView div = new DataInputViewStreamWrapper(in);
for (int i = 0; i < numPartitions; ++i) {
in.seek(offsets[i]);
Assert.assertEquals(i, div.readInt());
++count;
}
}
Assert.assertEquals(numPartitions, count);
}
/**
* Restore the KV-state / ColumnFamily data for all key-groups referenced by the current state
* handle
*
* @throws IOException
* @throws RocksDBException
*/
private void restoreKVStateData() throws IOException, RocksDBException {
// for all key-groups in the current state handle...
for (Tuple2<Integer, Long> keyGroupOffset :
currentKeyGroupsStateHandle.getGroupRangeOffsets()) {
long offset = keyGroupOffset.f1;
// not empty key-group?
if (0L != offset) {
currentStateHandleInStream.seek(offset);
boolean keyGroupHasMoreKeys = true;
// TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
int kvStateId = currentStateHandleInView.readShort();
ColumnFamilyHandle handle = currentStateHandleKVStateColumnFamilies.get(kvStateId);
// insert all k/v pairs into DB
while (keyGroupHasMoreKeys) {
byte[] key =
BytePrimitiveArraySerializer.INSTANCE.deserialize(currentStateHandleInView);
byte[] value =
BytePrimitiveArraySerializer.INSTANCE.deserialize(currentStateHandleInView);
if (RocksDBSnapshotOperation.hasMetaDataFollowsFlag(key)) {
// clear the signal bit in the key to make it ready for insertion again
RocksDBSnapshotOperation.clearMetaDataFollowsFlag(key);
rocksDBKeyedStateBackend.db.put(handle, key, value);
// TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
kvStateId =
RocksDBSnapshotOperation.END_OF_KEY_GROUP_MARK
& currentStateHandleInView.readShort();
if (RocksDBSnapshotOperation.END_OF_KEY_GROUP_MARK == kvStateId) {
keyGroupHasMoreKeys = false;
} else {
handle = currentStateHandleKVStateColumnFamilies.get(kvStateId);
}
} else {
rocksDBKeyedStateBackend.db.put(handle, key, value);
}
}
}
}
}
@Override
public void read(final DataInputView in) throws IOException {
int len = in.readUnsignedByte();
if (len >= HIGH_BIT) {
int shift = 7;
int curr;
len = len & 0x7f;
while ((curr = in.readUnsignedByte()) >= HIGH_BIT) {
len |= (curr & 0x7f) << shift;
shift += 7;
}
len |= curr << shift;
}
this.len = len;
this.hashCode = 0;
ensureSize(len);
final char[] data = this.value;
for (int i = 0; i < len; i++) {
int c = in.readUnsignedByte();
if (c < HIGH_BIT) {
data[i] = (char) c;
} else {
int shift = 7;
int curr;
c = c & 0x7f;
while ((curr = in.readUnsignedByte()) >= HIGH_BIT) {
c |= (curr & 0x7f) << shift;
shift += 7;
}
c |= curr << shift;
data[i] = (char) c;
}
}
}
@Override
public void read(final DataInputView in) throws IOException {
int size = in.readInt();
this.list.clear();
try {
for (; size > 0; size--) {
final V val = this.valueClass.newInstance();
val.read(in);
this.list.add(val);
}
} catch (final InstantiationException e) {
throw new RuntimeException(e);
} catch (final IllegalAccessException e) {
throw new RuntimeException(e);
}
}
@Override
public void copy(DataInputView source, DataOutputView target) throws IOException {
target.writeChar(source.readChar());
}
@Override
public Character deserialize(Character reuse, DataInputView source) throws IOException {
return Character.valueOf(source.readChar());
}
/** For backwards compatibility, remove again later! */
@Deprecated
private void restoreOldSavepointKeyedState(Collection<KeyGroupsStateHandle> restoreState)
throws Exception {
if (restoreState.isEmpty()) {
return;
}
Preconditions.checkState(1 == restoreState.size(), "Only one element expected here.");
HashMap<String, RocksDBStateBackend.FinalFullyAsyncSnapshot> namedStates =
InstantiationUtil.deserializeObject(
restoreState.iterator().next().openInputStream(), userCodeClassLoader);
Preconditions.checkState(1 == namedStates.size(), "Only one element expected here.");
DataInputView inputView =
namedStates.values().iterator().next().stateHandle.getState(userCodeClassLoader);
// clear k/v state information before filling it
kvStateInformation.clear();
// first get the column family mapping
int numColumns = inputView.readInt();
Map<Byte, StateDescriptor> columnFamilyMapping = new HashMap<>(numColumns);
for (int i = 0; i < numColumns; i++) {
byte mappingByte = inputView.readByte();
ObjectInputStream ooIn =
new InstantiationUtil.ClassLoaderObjectInputStream(
new DataInputViewStream(inputView), userCodeClassLoader);
StateDescriptor stateDescriptor = (StateDescriptor) ooIn.readObject();
columnFamilyMapping.put(mappingByte, stateDescriptor);
// this will fill in the k/v state information
getColumnFamily(stateDescriptor);
}
// try and read until EOF
try {
// the EOFException will get us out of this...
while (true) {
byte mappingByte = inputView.readByte();
ColumnFamilyHandle handle = getColumnFamily(columnFamilyMapping.get(mappingByte));
byte[] keyAndNamespace = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView);
ByteArrayInputStreamWithPos bis = new ByteArrayInputStreamWithPos(keyAndNamespace);
K reconstructedKey = keySerializer.deserialize(new DataInputViewStreamWrapper(bis));
int len = bis.getPosition();
int keyGroup =
(byte) KeyGroupRangeAssignment.assignToKeyGroup(reconstructedKey, numberOfKeyGroups);
if (keyGroupPrefixBytes == 1) {
// copy and override one byte (42) between key and namespace
System.arraycopy(keyAndNamespace, 0, keyAndNamespace, 1, len);
keyAndNamespace[0] = (byte) keyGroup;
} else {
byte[] largerKey = new byte[1 + keyAndNamespace.length];
// write key-group
largerKey[0] = (byte) ((keyGroup >> 8) & 0xFF);
largerKey[1] = (byte) (keyGroup & 0xFF);
// write key
System.arraycopy(keyAndNamespace, 0, largerKey, 2, len);
// skip one byte (42), write namespace
System.arraycopy(
keyAndNamespace, 1 + len, largerKey, 2 + len, keyAndNamespace.length - len - 1);
keyAndNamespace = largerKey;
}
byte[] value = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView);
db.put(handle, keyAndNamespace, value);
}
} catch (EOFException e) {
// expected
}
}
@Override
public void read(DataInputView in) throws IOException {
splitNumber = in.readInt();
}
@Override
public void read(DataInputView in) throws IOException {
this.lowerPart = in.readLong();
this.upperPart = in.readLong();
}
@Override
public void read(DataInputView in) throws IOException {
this.value = in.readChar();
}