@Override public void copy(DataInputView in, DataOutputView target) throws IOException { int len = in.readUnsignedByte(); target.writeByte(len); if (len >= HIGH_BIT) { int shift = 7; int curr; len = len & 0x7f; while ((curr = in.readUnsignedByte()) >= HIGH_BIT) { len |= (curr & 0x7f) << shift; shift += 7; target.writeByte(curr); } len |= curr << shift; target.writeByte(curr); } for (int i = 0; i < len; i++) { int c = in.readUnsignedByte(); target.writeByte(c); while (c >= HIGH_BIT) { c = in.readUnsignedByte(); target.writeByte(c); } } }
@Override public void restoreState(StreamTaskState taskState, long recoveryTimestamp) throws Exception { super.restoreState(taskState, recoveryTimestamp); final ClassLoader userClassloader = getUserCodeClassloader(); @SuppressWarnings("unchecked") StateHandle<DataInputView> inputState = (StateHandle<DataInputView>) taskState.getOperatorState(); DataInputView in = inputState.getState(userClassloader); int numKeys = in.readInt(); this.windows = new HashMap<>(numKeys); this.processingTimeTimers = new HashMap<>(); this.watermarkTimers = new HashMap<>(); for (int i = 0; i < numKeys; i++) { int numWindows = in.readInt(); for (int j = 0; j < numWindows; j++) { Context context = new Context(in, userClassloader); Map<W, Context> keyWindows = windows.get(context.key); if (keyWindows == null) { keyWindows = new HashMap<>(numWindows); windows.put(context.key, keyWindows); } keyWindows.put(context.window, context); } } }
@Override public void copy(DataInputView source, DataOutputView target) throws IOException { int len = source.readInt(); target.writeInt(len); for (int i = 0; i < len; i++) { boolean isNonNull = source.readBoolean(); target.writeBoolean(isNonNull); if (isNonNull) { componentSerializer.copy(source, target); } } }
/** * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current * state handle * * @throws IOException * @throws ClassNotFoundException * @throws RocksDBException */ private void restoreKVStateMetaData() throws IOException, ClassNotFoundException, RocksDBException { // read number of k/v states int numColumns = currentStateHandleInView.readInt(); // those two lists are aligned and should later have the same size! currentStateHandleKVStateColumnFamilies = new ArrayList<>(numColumns); // restore the empty columns for the k/v states through the metadata for (int i = 0; i < numColumns; i++) { StateDescriptor<?, ?> stateDescriptor = InstantiationUtil.deserializeObject( currentStateHandleInStream, rocksDBKeyedStateBackend.userCodeClassLoader); Tuple2<ColumnFamilyHandle, StateDescriptor<?, ?>> columnFamily = rocksDBKeyedStateBackend.kvStateInformation.get(stateDescriptor.getName()); if (null == columnFamily) { ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor( stateDescriptor.getName().getBytes(), rocksDBKeyedStateBackend.columnOptions); columnFamily = new Tuple2<ColumnFamilyHandle, StateDescriptor<?, ?>>( rocksDBKeyedStateBackend.db.createColumnFamily(columnFamilyDescriptor), stateDescriptor); rocksDBKeyedStateBackend.kvStateInformation.put(stateDescriptor.getName(), columnFamily); } currentStateHandleKVStateColumnFamilies.add(columnFamily.f0); } }
@Override public C[] deserialize(DataInputView source) throws IOException { int len = source.readInt(); C[] array = create(len); for (int i = 0; i < len; i++) { boolean isNonNull = source.readBoolean(); if (isNonNull) { array[i] = componentSerializer.deserialize(source); } else { array[i] = null; } } return array; }
private void restoreTimers(DataInputView in) throws IOException { int numWatermarkTimers = in.readInt(); watermarkTimers = new HashSet<>(numWatermarkTimers); watermarkTimersQueue = new PriorityQueue<>(Math.max(numWatermarkTimers, 1)); for (int i = 0; i < numWatermarkTimers; i++) { K key = keySerializer.deserialize(in); W window = windowSerializer.deserialize(in); long timestamp = in.readLong(); Timer<K, W> timer = new Timer<>(timestamp, key, window); watermarkTimers.add(timer); watermarkTimersQueue.add(timer); } int numProcessingTimeTimers = in.readInt(); processingTimeTimersQueue = new PriorityQueue<>(Math.max(numProcessingTimeTimers, 1)); processingTimeTimers = new HashSet<>(); for (int i = 0; i < numProcessingTimeTimers; i++) { K key = keySerializer.deserialize(in); W window = windowSerializer.deserialize(in); long timestamp = in.readLong(); Timer<K, W> timer = new Timer<>(timestamp, key, window); processingTimeTimersQueue.add(timer); processingTimeTimers.add(timer); } int numProcessingTimeTimerTimestamp = in.readInt(); processingTimeTimerTimestamps = HashMultiset.create(); for (int i = 0; i < numProcessingTimeTimerTimestamp; i++) { long timestamp = in.readLong(); int count = in.readInt(); processingTimeTimerTimestamps.add(timestamp, count); } }
@Override public void read(DataInputView in) throws IOException { final int addr_length = in.readInt(); byte[] address = new byte[addr_length]; in.readFully(address); this.dataPort = in.readInt(); this.fqdnHostName = StringUtils.readNullableString(in); this.hostName = StringUtils.readNullableString(in); try { this.inetAddress = InetAddress.getByAddress(address); } catch (UnknownHostException e) { throw new IOException("This lookup should never fail.", e); } }
@Override public TaggedUnion<T1, T2> deserialize(DataInputView source) throws IOException { byte tag = source.readByte(); if (tag == 1) { return TaggedUnion.one(oneSerializer.deserialize(source)); } else { return TaggedUnion.two(twoSerializer.deserialize(source)); } }
public static BigInteger readBigInteger(DataInputView source) throws IOException { final int len = source.readInt(); if (len < 4) { switch (len) { case 0: return null; case 1: return BigInteger.ZERO; case 2: return BigInteger.ONE; case 3: return BigInteger.TEN; } } final byte[] bytes = new byte[len - 4]; source.read(bytes); return new BigInteger(bytes); }
public static boolean copyBigInteger(DataInputView source, DataOutputView target) throws IOException { final int len = source.readInt(); target.writeInt(len); if (len > 4) { target.write(source, len - 4); } return len == 0; // returns true if the copied record was null }
@Override public void copy(DataInputView source, DataOutputView target) throws IOException { byte tag = source.readByte(); target.writeByte(tag); if (tag == 1) { oneSerializer.copy(source, target); } else { twoSerializer.copy(source, target); } }
/** * Constructs a new {@code Context} by reading from a {@link DataInputView} that contains a * serialized context that we wrote in {@link * #writeToState(StateBackend.CheckpointStateOutputView)} */ @SuppressWarnings("unchecked") protected Context(DataInputView in, ClassLoader userClassloader) throws Exception { this.key = keySerializer.deserialize(in); this.window = windowSerializer.deserialize(in); this.watermarkTimer = in.readLong(); this.processingTimeTimer = in.readLong(); int stateSize = in.readInt(); byte[] stateData = new byte[stateSize]; in.read(stateData); state = InstantiationUtil.deserializeObject(stateData, userClassloader); this.windowBuffer = windowBufferFactory.create(); int numElements = in.readInt(); MultiplexingStreamRecordSerializer<IN> recordSerializer = new MultiplexingStreamRecordSerializer<>(inputSerializer); for (int i = 0; i < numElements; i++) { windowBuffer.storeElement(recordSerializer.deserialize(in).<IN>asRecord()); } }
@Override public void read(DataInputView in) throws IOException { final boolean isNotNull = in.readBoolean(); if (isNotNull) { final String scheme = StringUtils.readNullableString(in); final String userInfo = StringUtils.readNullableString(in); final String host = StringUtils.readNullableString(in); final int port = in.readInt(); final String path = StringUtils.readNullableString(in); final String query = StringUtils.readNullableString(in); final String fragment = StringUtils.readNullableString(in); try { uri = new URI(scheme, userInfo, host, port, path, query, fragment); } catch (URISyntaxException e) { throw new IOException("Error reconstructing URI", e); } } }
private static void verifyRead(OperatorStateHandle fullHandle, int numPartitions) throws IOException { int count = 0; try (FSDataInputStream in = fullHandle.openInputStream()) { long[] offsets = fullHandle .getStateNameToPartitionOffsets() .get(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME); Assert.assertNotNull(offsets); DataInputView div = new DataInputViewStreamWrapper(in); for (int i = 0; i < numPartitions; ++i) { in.seek(offsets[i]); Assert.assertEquals(i, div.readInt()); ++count; } } Assert.assertEquals(numPartitions, count); }
/** * Restore the KV-state / ColumnFamily data for all key-groups referenced by the current state * handle * * @throws IOException * @throws RocksDBException */ private void restoreKVStateData() throws IOException, RocksDBException { // for all key-groups in the current state handle... for (Tuple2<Integer, Long> keyGroupOffset : currentKeyGroupsStateHandle.getGroupRangeOffsets()) { long offset = keyGroupOffset.f1; // not empty key-group? if (0L != offset) { currentStateHandleInStream.seek(offset); boolean keyGroupHasMoreKeys = true; // TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible int kvStateId = currentStateHandleInView.readShort(); ColumnFamilyHandle handle = currentStateHandleKVStateColumnFamilies.get(kvStateId); // insert all k/v pairs into DB while (keyGroupHasMoreKeys) { byte[] key = BytePrimitiveArraySerializer.INSTANCE.deserialize(currentStateHandleInView); byte[] value = BytePrimitiveArraySerializer.INSTANCE.deserialize(currentStateHandleInView); if (RocksDBSnapshotOperation.hasMetaDataFollowsFlag(key)) { // clear the signal bit in the key to make it ready for insertion again RocksDBSnapshotOperation.clearMetaDataFollowsFlag(key); rocksDBKeyedStateBackend.db.put(handle, key, value); // TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible kvStateId = RocksDBSnapshotOperation.END_OF_KEY_GROUP_MARK & currentStateHandleInView.readShort(); if (RocksDBSnapshotOperation.END_OF_KEY_GROUP_MARK == kvStateId) { keyGroupHasMoreKeys = false; } else { handle = currentStateHandleKVStateColumnFamilies.get(kvStateId); } } else { rocksDBKeyedStateBackend.db.put(handle, key, value); } } } } }
@Override public void read(final DataInputView in) throws IOException { int len = in.readUnsignedByte(); if (len >= HIGH_BIT) { int shift = 7; int curr; len = len & 0x7f; while ((curr = in.readUnsignedByte()) >= HIGH_BIT) { len |= (curr & 0x7f) << shift; shift += 7; } len |= curr << shift; } this.len = len; this.hashCode = 0; ensureSize(len); final char[] data = this.value; for (int i = 0; i < len; i++) { int c = in.readUnsignedByte(); if (c < HIGH_BIT) { data[i] = (char) c; } else { int shift = 7; int curr; c = c & 0x7f; while ((curr = in.readUnsignedByte()) >= HIGH_BIT) { c |= (curr & 0x7f) << shift; shift += 7; } c |= curr << shift; data[i] = (char) c; } } }
@Override public void read(final DataInputView in) throws IOException { int size = in.readInt(); this.list.clear(); try { for (; size > 0; size--) { final V val = this.valueClass.newInstance(); val.read(in); this.list.add(val); } } catch (final InstantiationException e) { throw new RuntimeException(e); } catch (final IllegalAccessException e) { throw new RuntimeException(e); } }
@Override public void copy(DataInputView source, DataOutputView target) throws IOException { target.writeChar(source.readChar()); }
@Override public Character deserialize(Character reuse, DataInputView source) throws IOException { return Character.valueOf(source.readChar()); }
/** For backwards compatibility, remove again later! */ @Deprecated private void restoreOldSavepointKeyedState(Collection<KeyGroupsStateHandle> restoreState) throws Exception { if (restoreState.isEmpty()) { return; } Preconditions.checkState(1 == restoreState.size(), "Only one element expected here."); HashMap<String, RocksDBStateBackend.FinalFullyAsyncSnapshot> namedStates = InstantiationUtil.deserializeObject( restoreState.iterator().next().openInputStream(), userCodeClassLoader); Preconditions.checkState(1 == namedStates.size(), "Only one element expected here."); DataInputView inputView = namedStates.values().iterator().next().stateHandle.getState(userCodeClassLoader); // clear k/v state information before filling it kvStateInformation.clear(); // first get the column family mapping int numColumns = inputView.readInt(); Map<Byte, StateDescriptor> columnFamilyMapping = new HashMap<>(numColumns); for (int i = 0; i < numColumns; i++) { byte mappingByte = inputView.readByte(); ObjectInputStream ooIn = new InstantiationUtil.ClassLoaderObjectInputStream( new DataInputViewStream(inputView), userCodeClassLoader); StateDescriptor stateDescriptor = (StateDescriptor) ooIn.readObject(); columnFamilyMapping.put(mappingByte, stateDescriptor); // this will fill in the k/v state information getColumnFamily(stateDescriptor); } // try and read until EOF try { // the EOFException will get us out of this... while (true) { byte mappingByte = inputView.readByte(); ColumnFamilyHandle handle = getColumnFamily(columnFamilyMapping.get(mappingByte)); byte[] keyAndNamespace = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView); ByteArrayInputStreamWithPos bis = new ByteArrayInputStreamWithPos(keyAndNamespace); K reconstructedKey = keySerializer.deserialize(new DataInputViewStreamWrapper(bis)); int len = bis.getPosition(); int keyGroup = (byte) KeyGroupRangeAssignment.assignToKeyGroup(reconstructedKey, numberOfKeyGroups); if (keyGroupPrefixBytes == 1) { // copy and override one byte (42) between key and namespace System.arraycopy(keyAndNamespace, 0, keyAndNamespace, 1, len); keyAndNamespace[0] = (byte) keyGroup; } else { byte[] largerKey = new byte[1 + keyAndNamespace.length]; // write key-group largerKey[0] = (byte) ((keyGroup >> 8) & 0xFF); largerKey[1] = (byte) (keyGroup & 0xFF); // write key System.arraycopy(keyAndNamespace, 0, largerKey, 2, len); // skip one byte (42), write namespace System.arraycopy( keyAndNamespace, 1 + len, largerKey, 2 + len, keyAndNamespace.length - len - 1); keyAndNamespace = largerKey; } byte[] value = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView); db.put(handle, keyAndNamespace, value); } } catch (EOFException e) { // expected } }
@Override public void read(DataInputView in) throws IOException { splitNumber = in.readInt(); }
@Override public void read(DataInputView in) throws IOException { this.lowerPart = in.readLong(); this.upperPart = in.readLong(); }
@Override public void read(DataInputView in) throws IOException { this.value = in.readChar(); }