@Override public void copy(DataInputView in, DataOutputView target) throws IOException { int len = in.readUnsignedByte(); target.writeByte(len); if (len >= HIGH_BIT) { int shift = 7; int curr; len = len & 0x7f; while ((curr = in.readUnsignedByte()) >= HIGH_BIT) { len |= (curr & 0x7f) << shift; shift += 7; target.writeByte(curr); } len |= curr << shift; target.writeByte(curr); } for (int i = 0; i < len; i++) { int c = in.readUnsignedByte(); target.writeByte(c); while (c >= HIGH_BIT) { c = in.readUnsignedByte(); target.writeByte(c); } } }
public static boolean copyBigInteger(DataInputView source, DataOutputView target) throws IOException { final int len = source.readInt(); target.writeInt(len); if (len > 4) { target.write(source, len - 4); } return len == 0; // returns true if the copied record was null }
@Override public void write(final DataOutputView out) throws IOException { out.writeInt(this.inetAddress.getAddress().length); out.write(this.inetAddress.getAddress()); out.writeInt(this.dataPort); StringUtils.writeNullableString(fqdnHostName, out); StringUtils.writeNullableString(hostName, out); }
@Override public void serialize(TaggedUnion<T1, T2> record, DataOutputView target) throws IOException { if (record.isOne()) { target.writeByte(1); oneSerializer.serialize(record.getOne(), target); } else { target.writeByte(2); twoSerializer.serialize(record.getTwo(), target); } }
private OperatorStateHandle writeAllTestKeyGroups( OperatorStateCheckpointOutputStream stream, int numPartitions) throws Exception { DataOutputView dov = new DataOutputViewStreamWrapper(stream); for (int i = 0; i < numPartitions; ++i) { Assert.assertEquals(i, stream.getNumberOfPartitions()); stream.startNewPartition(); dov.writeInt(i); } return stream.closeAndGetHandle(); }
@Override public void serialize(C[] value, DataOutputView target) throws IOException { target.writeInt(value.length); for (int i = 0; i < value.length; i++) { C val = value[i]; if (val == null) { target.writeBoolean(false); } else { target.writeBoolean(true); componentSerializer.serialize(val, target); } } }
@Override public void copy(DataInputView source, DataOutputView target) throws IOException { int len = source.readInt(); target.writeInt(len); for (int i = 0; i < len; i++) { boolean isNonNull = source.readBoolean(); target.writeBoolean(isNonNull); if (isNonNull) { componentSerializer.copy(source, target); } } }
@Override public void write(final DataOutputView out) throws IOException { out.writeInt(this.list.size()); for (final V value : this.list) { value.write(out); } }
@Override public void write(DataOutputView out) throws IOException { if (uri == null) { out.writeBoolean(false); } else { out.writeBoolean(true); StringUtils.writeNullableString(uri.getScheme(), out); StringUtils.writeNullableString(uri.getUserInfo(), out); StringUtils.writeNullableString(uri.getHost(), out); out.writeInt(uri.getPort()); StringUtils.writeNullableString(uri.getPath(), out); StringUtils.writeNullableString(uri.getQuery(), out); StringUtils.writeNullableString(uri.getFragment(), out); } }
@Override public void copy(DataInputView source, DataOutputView target) throws IOException { byte tag = source.readByte(); target.writeByte(tag); if (tag == 1) { oneSerializer.copy(source, target); } else { twoSerializer.copy(source, target); } }
@Override public void write(final DataOutputView out) throws IOException { int len = this.len; // write the length, variable-length encoded while (len >= HIGH_BIT) { out.write(len | HIGH_BIT); len >>>= 7; } out.write(len); // write the char data, variable length encoded for (int i = 0; i < this.len; i++) { int c = this.value[i]; while (c >= HIGH_BIT) { out.write(c | HIGH_BIT); c >>>= 7; } out.write(c); } }
private void snapshotTimers(DataOutputView out) throws IOException { out.writeInt(watermarkTimersQueue.size()); for (Timer<K, W> timer : watermarkTimersQueue) { keySerializer.serialize(timer.key, out); windowSerializer.serialize(timer.window, out); out.writeLong(timer.timestamp); } out.writeInt(processingTimeTimers.size()); for (Timer<K, W> timer : processingTimeTimers) { keySerializer.serialize(timer.key, out); windowSerializer.serialize(timer.window, out); out.writeLong(timer.timestamp); } out.writeInt(processingTimeTimerTimestamps.entrySet().size()); for (Multiset.Entry<Long> timerTimestampCounts : processingTimeTimerTimestamps.entrySet()) { out.writeLong(timerTimestampCounts.getElement()); out.writeInt(timerTimestampCounts.getCount()); } }
private void writeKVStateMetaData() throws IOException, InterruptedException { // write number of k/v states outputView.writeInt(stateBackend.kvStateInformation.size()); int kvStateId = 0; // iterate all column families, where each column family holds one k/v state, to write the // metadata for (Map.Entry<String, Tuple2<ColumnFamilyHandle, StateDescriptor<?, ?>>> column : stateBackend.kvStateInformation.entrySet()) { // be cooperative and check for interruption from time to time in the hot loop checkInterrupted(); // write StateDescriptor for this k/v state InstantiationUtil.serializeObject(outStream, column.getValue().f1); // retrieve iterator for this k/v states readOptions = new ReadOptions(); readOptions.setSnapshot(snapshot); RocksIterator iterator = stateBackend.db.newIterator(column.getValue().f0, readOptions); kvStateIterators.add(new Tuple2<>(iterator, kvStateId)); ++kvStateId; } }
public static void writeBigInteger(BigInteger record, DataOutputView target) throws IOException { // null value support if (record == null) { target.writeInt(0); return; } // fast paths for 0, 1, 10 // only reference equality is checked because equals would be too expensive else if (record == BigInteger.ZERO) { target.writeInt(1); return; } else if (record == BigInteger.ONE) { target.writeInt(2); return; } else if (record == BigInteger.TEN) { target.writeInt(3); return; } // default final byte[] bytes = record.toByteArray(); // the length we write is offset by four, because null and short-paths for ZERO, ONE, and TEN target.writeInt(bytes.length + 4); target.write(bytes); }
@Override public void copy(DataInputView source, DataOutputView target) throws IOException { target.writeChar(source.readChar()); }
@Override public void serialize(Character record, DataOutputView target) throws IOException { target.writeChar(record.charValue()); }
protected void writeKeyAndNamespace(DataOutputView out) throws IOException { keySerializer.serialize(currentKey, out); out.writeByte(42); namespaceSerializer.serialize(currentNamespace, out); }
private void writeKVStateData() throws IOException, InterruptedException { byte[] previousKey = null; byte[] previousValue = null; List<Tuple2<RocksIterator, Integer>> kvStateIteratorsHandover = this.kvStateIterators; this.kvStateIterators = null; // Here we transfer ownership of RocksIterators to the RocksDBMergeIterator try (RocksDBMergeIterator mergeIterator = new RocksDBMergeIterator(kvStateIteratorsHandover, stateBackend.keyGroupPrefixBytes)) { // preamble: setup with first key-group as our lookahead if (mergeIterator.isValid()) { // begin first key-group by recording the offset keyGroupRangeOffsets.setKeyGroupOffset(mergeIterator.keyGroup(), outStream.getPos()); // write the k/v-state id as metadata // TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible outputView.writeShort(mergeIterator.kvStateId()); previousKey = mergeIterator.key(); previousValue = mergeIterator.value(); mergeIterator.next(); } // main loop: write k/v pairs ordered by (key-group, kv-state), thereby tracking key-group // offsets. while (mergeIterator.isValid()) { assert (!hasMetaDataFollowsFlag(previousKey)); // set signal in first key byte that meta data will follow in the stream after this k/v // pair if (mergeIterator.isNewKeyGroup() || mergeIterator.isNewKeyValueState()) { // be cooperative and check for interruption from time to time in the hot loop checkInterrupted(); setMetaDataFollowsFlagInKey(previousKey); } writeKeyValuePair(previousKey, previousValue); // write meta data if we have to if (mergeIterator.isNewKeyGroup()) { // TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible outputView.writeShort(END_OF_KEY_GROUP_MARK); // begin new key-group keyGroupRangeOffsets.setKeyGroupOffset(mergeIterator.keyGroup(), outStream.getPos()); // write the kev-state // TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible outputView.writeShort(mergeIterator.kvStateId()); } else if (mergeIterator.isNewKeyValueState()) { // write the k/v-state // TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible outputView.writeShort(mergeIterator.kvStateId()); } // request next k/v pair previousKey = mergeIterator.key(); previousValue = mergeIterator.value(); mergeIterator.next(); } } // epilogue: write last key-group if (previousKey != null) { assert (!hasMetaDataFollowsFlag(previousKey)); setMetaDataFollowsFlagInKey(previousKey); writeKeyValuePair(previousKey, previousValue); // TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible outputView.writeShort(END_OF_KEY_GROUP_MARK); } }
@Override public void write(DataOutputView out) throws IOException { out.writeInt(splitNumber); }
@Override public void write(DataOutputView out) throws IOException { out.writeChar(this.value); }
@Override public void write(DataOutputView out) throws IOException { out.writeLong(this.lowerPart); out.writeLong(this.upperPart); }