Exemplo n.º 1
0
  @Override
  public void copy(DataInputView in, DataOutputView target) throws IOException {
    int len = in.readUnsignedByte();
    target.writeByte(len);

    if (len >= HIGH_BIT) {
      int shift = 7;
      int curr;
      len = len & 0x7f;
      while ((curr = in.readUnsignedByte()) >= HIGH_BIT) {
        len |= (curr & 0x7f) << shift;
        shift += 7;
        target.writeByte(curr);
      }
      len |= curr << shift;
      target.writeByte(curr);
    }

    for (int i = 0; i < len; i++) {
      int c = in.readUnsignedByte();
      target.writeByte(c);
      while (c >= HIGH_BIT) {
        c = in.readUnsignedByte();
        target.writeByte(c);
      }
    }
  }
Exemplo n.º 2
0
 public static boolean copyBigInteger(DataInputView source, DataOutputView target)
     throws IOException {
   final int len = source.readInt();
   target.writeInt(len);
   if (len > 4) {
     target.write(source, len - 4);
   }
   return len == 0; // returns true if the copied record was null
 }
  @Override
  public void write(final DataOutputView out) throws IOException {
    out.writeInt(this.inetAddress.getAddress().length);
    out.write(this.inetAddress.getAddress());

    out.writeInt(this.dataPort);

    StringUtils.writeNullableString(fqdnHostName, out);
    StringUtils.writeNullableString(hostName, out);
  }
Exemplo n.º 4
0
 @Override
 public void serialize(TaggedUnion<T1, T2> record, DataOutputView target) throws IOException {
   if (record.isOne()) {
     target.writeByte(1);
     oneSerializer.serialize(record.getOne(), target);
   } else {
     target.writeByte(2);
     twoSerializer.serialize(record.getTwo(), target);
   }
 }
  private OperatorStateHandle writeAllTestKeyGroups(
      OperatorStateCheckpointOutputStream stream, int numPartitions) throws Exception {

    DataOutputView dov = new DataOutputViewStreamWrapper(stream);
    for (int i = 0; i < numPartitions; ++i) {
      Assert.assertEquals(i, stream.getNumberOfPartitions());
      stream.startNewPartition();
      dov.writeInt(i);
    }

    return stream.closeAndGetHandle();
  }
Exemplo n.º 6
0
 @Override
 public void serialize(C[] value, DataOutputView target) throws IOException {
   target.writeInt(value.length);
   for (int i = 0; i < value.length; i++) {
     C val = value[i];
     if (val == null) {
       target.writeBoolean(false);
     } else {
       target.writeBoolean(true);
       componentSerializer.serialize(val, target);
     }
   }
 }
Exemplo n.º 7
0
  @Override
  public void copy(DataInputView source, DataOutputView target) throws IOException {
    int len = source.readInt();
    target.writeInt(len);

    for (int i = 0; i < len; i++) {
      boolean isNonNull = source.readBoolean();
      target.writeBoolean(isNonNull);

      if (isNonNull) {
        componentSerializer.copy(source, target);
      }
    }
  }
Exemplo n.º 8
0
 @Override
 public void write(final DataOutputView out) throws IOException {
   out.writeInt(this.list.size());
   for (final V value : this.list) {
     value.write(out);
   }
 }
Exemplo n.º 9
0
  @Override
  public void write(DataOutputView out) throws IOException {

    if (uri == null) {
      out.writeBoolean(false);
    } else {
      out.writeBoolean(true);
      StringUtils.writeNullableString(uri.getScheme(), out);
      StringUtils.writeNullableString(uri.getUserInfo(), out);
      StringUtils.writeNullableString(uri.getHost(), out);
      out.writeInt(uri.getPort());
      StringUtils.writeNullableString(uri.getPath(), out);
      StringUtils.writeNullableString(uri.getQuery(), out);
      StringUtils.writeNullableString(uri.getFragment(), out);
    }
  }
Exemplo n.º 10
0
 @Override
 public void copy(DataInputView source, DataOutputView target) throws IOException {
   byte tag = source.readByte();
   target.writeByte(tag);
   if (tag == 1) {
     oneSerializer.copy(source, target);
   } else {
     twoSerializer.copy(source, target);
   }
 }
Exemplo n.º 11
0
  @Override
  public void write(final DataOutputView out) throws IOException {
    int len = this.len;

    // write the length, variable-length encoded
    while (len >= HIGH_BIT) {
      out.write(len | HIGH_BIT);
      len >>>= 7;
    }
    out.write(len);

    // write the char data, variable length encoded
    for (int i = 0; i < this.len; i++) {
      int c = this.value[i];

      while (c >= HIGH_BIT) {
        out.write(c | HIGH_BIT);
        c >>>= 7;
      }
      out.write(c);
    }
  }
Exemplo n.º 12
0
  private void snapshotTimers(DataOutputView out) throws IOException {
    out.writeInt(watermarkTimersQueue.size());
    for (Timer<K, W> timer : watermarkTimersQueue) {
      keySerializer.serialize(timer.key, out);
      windowSerializer.serialize(timer.window, out);
      out.writeLong(timer.timestamp);
    }

    out.writeInt(processingTimeTimers.size());
    for (Timer<K, W> timer : processingTimeTimers) {
      keySerializer.serialize(timer.key, out);
      windowSerializer.serialize(timer.window, out);
      out.writeLong(timer.timestamp);
    }

    out.writeInt(processingTimeTimerTimestamps.entrySet().size());
    for (Multiset.Entry<Long> timerTimestampCounts : processingTimeTimerTimestamps.entrySet()) {
      out.writeLong(timerTimestampCounts.getElement());
      out.writeInt(timerTimestampCounts.getCount());
    }
  }
Exemplo n.º 13
0
    private void writeKVStateMetaData() throws IOException, InterruptedException {
      // write number of k/v states
      outputView.writeInt(stateBackend.kvStateInformation.size());

      int kvStateId = 0;
      // iterate all column families, where each column family holds one k/v state, to write the
      // metadata
      for (Map.Entry<String, Tuple2<ColumnFamilyHandle, StateDescriptor<?, ?>>> column :
          stateBackend.kvStateInformation.entrySet()) {

        // be cooperative and check for interruption from time to time in the hot loop
        checkInterrupted();

        // write StateDescriptor for this k/v state
        InstantiationUtil.serializeObject(outStream, column.getValue().f1);

        // retrieve iterator for this k/v states
        readOptions = new ReadOptions();
        readOptions.setSnapshot(snapshot);
        RocksIterator iterator = stateBackend.db.newIterator(column.getValue().f0, readOptions);
        kvStateIterators.add(new Tuple2<>(iterator, kvStateId));
        ++kvStateId;
      }
    }
Exemplo n.º 14
0
 public static void writeBigInteger(BigInteger record, DataOutputView target) throws IOException {
   // null value support
   if (record == null) {
     target.writeInt(0);
     return;
   }
   // fast paths for 0, 1, 10
   // only reference equality is checked because equals would be too expensive
   else if (record == BigInteger.ZERO) {
     target.writeInt(1);
     return;
   } else if (record == BigInteger.ONE) {
     target.writeInt(2);
     return;
   } else if (record == BigInteger.TEN) {
     target.writeInt(3);
     return;
   }
   // default
   final byte[] bytes = record.toByteArray();
   // the length we write is offset by four, because null and short-paths for ZERO, ONE, and TEN
   target.writeInt(bytes.length + 4);
   target.write(bytes);
 }
Exemplo n.º 15
0
 @Override
 public void copy(DataInputView source, DataOutputView target) throws IOException {
   target.writeChar(source.readChar());
 }
Exemplo n.º 16
0
 @Override
 public void serialize(Character record, DataOutputView target) throws IOException {
   target.writeChar(record.charValue());
 }
Exemplo n.º 17
0
 protected void writeKeyAndNamespace(DataOutputView out) throws IOException {
   keySerializer.serialize(currentKey, out);
   out.writeByte(42);
   namespaceSerializer.serialize(currentNamespace, out);
 }
Exemplo n.º 18
0
    private void writeKVStateData() throws IOException, InterruptedException {

      byte[] previousKey = null;
      byte[] previousValue = null;

      List<Tuple2<RocksIterator, Integer>> kvStateIteratorsHandover = this.kvStateIterators;
      this.kvStateIterators = null;

      // Here we transfer ownership of RocksIterators to the RocksDBMergeIterator
      try (RocksDBMergeIterator mergeIterator =
          new RocksDBMergeIterator(kvStateIteratorsHandover, stateBackend.keyGroupPrefixBytes)) {

        // preamble: setup with first key-group as our lookahead
        if (mergeIterator.isValid()) {
          // begin first key-group by recording the offset
          keyGroupRangeOffsets.setKeyGroupOffset(mergeIterator.keyGroup(), outStream.getPos());
          // write the k/v-state id as metadata
          // TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
          outputView.writeShort(mergeIterator.kvStateId());
          previousKey = mergeIterator.key();
          previousValue = mergeIterator.value();
          mergeIterator.next();
        }

        // main loop: write k/v pairs ordered by (key-group, kv-state), thereby tracking key-group
        // offsets.
        while (mergeIterator.isValid()) {

          assert (!hasMetaDataFollowsFlag(previousKey));

          // set signal in first key byte that meta data will follow in the stream after this k/v
          // pair
          if (mergeIterator.isNewKeyGroup() || mergeIterator.isNewKeyValueState()) {

            // be cooperative and check for interruption from time to time in the hot loop
            checkInterrupted();

            setMetaDataFollowsFlagInKey(previousKey);
          }

          writeKeyValuePair(previousKey, previousValue);

          // write meta data if we have to
          if (mergeIterator.isNewKeyGroup()) {
            // TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
            outputView.writeShort(END_OF_KEY_GROUP_MARK);
            // begin new key-group
            keyGroupRangeOffsets.setKeyGroupOffset(mergeIterator.keyGroup(), outStream.getPos());
            // write the kev-state
            // TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
            outputView.writeShort(mergeIterator.kvStateId());
          } else if (mergeIterator.isNewKeyValueState()) {
            // write the k/v-state
            // TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
            outputView.writeShort(mergeIterator.kvStateId());
          }

          // request next k/v pair
          previousKey = mergeIterator.key();
          previousValue = mergeIterator.value();
          mergeIterator.next();
        }
      }

      // epilogue: write last key-group
      if (previousKey != null) {
        assert (!hasMetaDataFollowsFlag(previousKey));
        setMetaDataFollowsFlagInKey(previousKey);
        writeKeyValuePair(previousKey, previousValue);
        // TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
        outputView.writeShort(END_OF_KEY_GROUP_MARK);
      }
    }
 @Override
 public void write(DataOutputView out) throws IOException {
   out.writeInt(splitNumber);
 }
Exemplo n.º 20
0
 @Override
 public void write(DataOutputView out) throws IOException {
   out.writeChar(this.value);
 }
Exemplo n.º 21
0
 @Override
 public void write(DataOutputView out) throws IOException {
   out.writeLong(this.lowerPart);
   out.writeLong(this.upperPart);
 }