コード例 #1
0
ファイル: StringValue.java プロジェクト: rmetzger/flink
  @Override
  public void copy(DataInputView in, DataOutputView target) throws IOException {
    int len = in.readUnsignedByte();
    target.writeByte(len);

    if (len >= HIGH_BIT) {
      int shift = 7;
      int curr;
      len = len & 0x7f;
      while ((curr = in.readUnsignedByte()) >= HIGH_BIT) {
        len |= (curr & 0x7f) << shift;
        shift += 7;
        target.writeByte(curr);
      }
      len |= curr << shift;
      target.writeByte(curr);
    }

    for (int i = 0; i < len; i++) {
      int c = in.readUnsignedByte();
      target.writeByte(c);
      while (c >= HIGH_BIT) {
        c = in.readUnsignedByte();
        target.writeByte(c);
      }
    }
  }
コード例 #2
0
ファイル: WindowOperator.java プロジェクト: lizhiyong/flink
  @Override
  public void restoreState(StreamTaskState taskState, long recoveryTimestamp) throws Exception {
    super.restoreState(taskState, recoveryTimestamp);

    final ClassLoader userClassloader = getUserCodeClassloader();

    @SuppressWarnings("unchecked")
    StateHandle<DataInputView> inputState =
        (StateHandle<DataInputView>) taskState.getOperatorState();
    DataInputView in = inputState.getState(userClassloader);

    int numKeys = in.readInt();
    this.windows = new HashMap<>(numKeys);
    this.processingTimeTimers = new HashMap<>();
    this.watermarkTimers = new HashMap<>();

    for (int i = 0; i < numKeys; i++) {
      int numWindows = in.readInt();
      for (int j = 0; j < numWindows; j++) {
        Context context = new Context(in, userClassloader);
        Map<W, Context> keyWindows = windows.get(context.key);
        if (keyWindows == null) {
          keyWindows = new HashMap<>(numWindows);
          windows.put(context.key, keyWindows);
        }
        keyWindows.put(context.window, context);
      }
    }
  }
コード例 #3
0
  @Override
  public void copy(DataInputView source, DataOutputView target) throws IOException {
    int len = source.readInt();
    target.writeInt(len);

    for (int i = 0; i < len; i++) {
      boolean isNonNull = source.readBoolean();
      target.writeBoolean(isNonNull);

      if (isNonNull) {
        componentSerializer.copy(source, target);
      }
    }
  }
コード例 #4
0
    /**
     * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current
     * state handle
     *
     * @throws IOException
     * @throws ClassNotFoundException
     * @throws RocksDBException
     */
    private void restoreKVStateMetaData()
        throws IOException, ClassNotFoundException, RocksDBException {
      // read number of k/v states
      int numColumns = currentStateHandleInView.readInt();

      // those two lists are aligned and should later have the same size!
      currentStateHandleKVStateColumnFamilies = new ArrayList<>(numColumns);

      // restore the empty columns for the k/v states through the metadata
      for (int i = 0; i < numColumns; i++) {

        StateDescriptor<?, ?> stateDescriptor =
            InstantiationUtil.deserializeObject(
                currentStateHandleInStream, rocksDBKeyedStateBackend.userCodeClassLoader);

        Tuple2<ColumnFamilyHandle, StateDescriptor<?, ?>> columnFamily =
            rocksDBKeyedStateBackend.kvStateInformation.get(stateDescriptor.getName());

        if (null == columnFamily) {
          ColumnFamilyDescriptor columnFamilyDescriptor =
              new ColumnFamilyDescriptor(
                  stateDescriptor.getName().getBytes(), rocksDBKeyedStateBackend.columnOptions);

          columnFamily =
              new Tuple2<ColumnFamilyHandle, StateDescriptor<?, ?>>(
                  rocksDBKeyedStateBackend.db.createColumnFamily(columnFamilyDescriptor),
                  stateDescriptor);

          rocksDBKeyedStateBackend.kvStateInformation.put(stateDescriptor.getName(), columnFamily);
        }

        currentStateHandleKVStateColumnFamilies.add(columnFamily.f0);
      }
    }
コード例 #5
0
  @Override
  public C[] deserialize(DataInputView source) throws IOException {
    int len = source.readInt();

    C[] array = create(len);

    for (int i = 0; i < len; i++) {
      boolean isNonNull = source.readBoolean();
      if (isNonNull) {
        array[i] = componentSerializer.deserialize(source);
      } else {
        array[i] = null;
      }
    }

    return array;
  }
コード例 #6
0
  private void restoreTimers(DataInputView in) throws IOException {
    int numWatermarkTimers = in.readInt();
    watermarkTimers = new HashSet<>(numWatermarkTimers);
    watermarkTimersQueue = new PriorityQueue<>(Math.max(numWatermarkTimers, 1));
    for (int i = 0; i < numWatermarkTimers; i++) {
      K key = keySerializer.deserialize(in);
      W window = windowSerializer.deserialize(in);
      long timestamp = in.readLong();
      Timer<K, W> timer = new Timer<>(timestamp, key, window);
      watermarkTimers.add(timer);
      watermarkTimersQueue.add(timer);
    }

    int numProcessingTimeTimers = in.readInt();
    processingTimeTimersQueue = new PriorityQueue<>(Math.max(numProcessingTimeTimers, 1));
    processingTimeTimers = new HashSet<>();
    for (int i = 0; i < numProcessingTimeTimers; i++) {
      K key = keySerializer.deserialize(in);
      W window = windowSerializer.deserialize(in);
      long timestamp = in.readLong();
      Timer<K, W> timer = new Timer<>(timestamp, key, window);
      processingTimeTimersQueue.add(timer);
      processingTimeTimers.add(timer);
    }

    int numProcessingTimeTimerTimestamp = in.readInt();
    processingTimeTimerTimestamps = HashMultiset.create();
    for (int i = 0; i < numProcessingTimeTimerTimestamp; i++) {
      long timestamp = in.readLong();
      int count = in.readInt();
      processingTimeTimerTimestamps.add(timestamp, count);
    }
  }
コード例 #7
0
  @Override
  public void read(DataInputView in) throws IOException {

    final int addr_length = in.readInt();
    byte[] address = new byte[addr_length];
    in.readFully(address);

    this.dataPort = in.readInt();

    this.fqdnHostName = StringUtils.readNullableString(in);
    this.hostName = StringUtils.readNullableString(in);

    try {
      this.inetAddress = InetAddress.getByAddress(address);
    } catch (UnknownHostException e) {
      throw new IOException("This lookup should never fail.", e);
    }
  }
コード例 #8
0
ファイル: CoGroupedStreams.java プロジェクト: nguyent/flink
 @Override
 public TaggedUnion<T1, T2> deserialize(DataInputView source) throws IOException {
   byte tag = source.readByte();
   if (tag == 1) {
     return TaggedUnion.one(oneSerializer.deserialize(source));
   } else {
     return TaggedUnion.two(twoSerializer.deserialize(source));
   }
 }
コード例 #9
0
ファイル: BigIntSerializer.java プロジェクト: rmetzger/flink
 public static BigInteger readBigInteger(DataInputView source) throws IOException {
   final int len = source.readInt();
   if (len < 4) {
     switch (len) {
       case 0:
         return null;
       case 1:
         return BigInteger.ZERO;
       case 2:
         return BigInteger.ONE;
       case 3:
         return BigInteger.TEN;
     }
   }
   final byte[] bytes = new byte[len - 4];
   source.read(bytes);
   return new BigInteger(bytes);
 }
コード例 #10
0
ファイル: BigIntSerializer.java プロジェクト: rmetzger/flink
 public static boolean copyBigInteger(DataInputView source, DataOutputView target)
     throws IOException {
   final int len = source.readInt();
   target.writeInt(len);
   if (len > 4) {
     target.write(source, len - 4);
   }
   return len == 0; // returns true if the copied record was null
 }
コード例 #11
0
ファイル: CoGroupedStreams.java プロジェクト: nguyent/flink
 @Override
 public void copy(DataInputView source, DataOutputView target) throws IOException {
   byte tag = source.readByte();
   target.writeByte(tag);
   if (tag == 1) {
     oneSerializer.copy(source, target);
   } else {
     twoSerializer.copy(source, target);
   }
 }
コード例 #12
0
ファイル: WindowOperator.java プロジェクト: lizhiyong/flink
    /**
     * Constructs a new {@code Context} by reading from a {@link DataInputView} that contains a
     * serialized context that we wrote in {@link
     * #writeToState(StateBackend.CheckpointStateOutputView)}
     */
    @SuppressWarnings("unchecked")
    protected Context(DataInputView in, ClassLoader userClassloader) throws Exception {
      this.key = keySerializer.deserialize(in);
      this.window = windowSerializer.deserialize(in);
      this.watermarkTimer = in.readLong();
      this.processingTimeTimer = in.readLong();

      int stateSize = in.readInt();
      byte[] stateData = new byte[stateSize];
      in.read(stateData);
      state = InstantiationUtil.deserializeObject(stateData, userClassloader);

      this.windowBuffer = windowBufferFactory.create();
      int numElements = in.readInt();
      MultiplexingStreamRecordSerializer<IN> recordSerializer =
          new MultiplexingStreamRecordSerializer<>(inputSerializer);
      for (int i = 0; i < numElements; i++) {
        windowBuffer.storeElement(recordSerializer.deserialize(in).<IN>asRecord());
      }
    }
コード例 #13
0
ファイル: Path.java プロジェクト: ktzoumas/incubator-flink
  @Override
  public void read(DataInputView in) throws IOException {

    final boolean isNotNull = in.readBoolean();
    if (isNotNull) {
      final String scheme = StringUtils.readNullableString(in);
      final String userInfo = StringUtils.readNullableString(in);
      final String host = StringUtils.readNullableString(in);
      final int port = in.readInt();
      final String path = StringUtils.readNullableString(in);
      final String query = StringUtils.readNullableString(in);
      final String fragment = StringUtils.readNullableString(in);

      try {
        uri = new URI(scheme, userInfo, host, port, path, query, fragment);
      } catch (URISyntaxException e) {
        throw new IOException("Error reconstructing URI", e);
      }
    }
  }
コード例 #14
0
  private static void verifyRead(OperatorStateHandle fullHandle, int numPartitions)
      throws IOException {
    int count = 0;
    try (FSDataInputStream in = fullHandle.openInputStream()) {
      long[] offsets =
          fullHandle
              .getStateNameToPartitionOffsets()
              .get(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);

      Assert.assertNotNull(offsets);

      DataInputView div = new DataInputViewStreamWrapper(in);
      for (int i = 0; i < numPartitions; ++i) {
        in.seek(offsets[i]);
        Assert.assertEquals(i, div.readInt());
        ++count;
      }
    }

    Assert.assertEquals(numPartitions, count);
  }
コード例 #15
0
 /**
  * Restore the KV-state / ColumnFamily data for all key-groups referenced by the current state
  * handle
  *
  * @throws IOException
  * @throws RocksDBException
  */
 private void restoreKVStateData() throws IOException, RocksDBException {
   // for all key-groups in the current state handle...
   for (Tuple2<Integer, Long> keyGroupOffset :
       currentKeyGroupsStateHandle.getGroupRangeOffsets()) {
     long offset = keyGroupOffset.f1;
     // not empty key-group?
     if (0L != offset) {
       currentStateHandleInStream.seek(offset);
       boolean keyGroupHasMoreKeys = true;
       // TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
       int kvStateId = currentStateHandleInView.readShort();
       ColumnFamilyHandle handle = currentStateHandleKVStateColumnFamilies.get(kvStateId);
       // insert all k/v pairs into DB
       while (keyGroupHasMoreKeys) {
         byte[] key =
             BytePrimitiveArraySerializer.INSTANCE.deserialize(currentStateHandleInView);
         byte[] value =
             BytePrimitiveArraySerializer.INSTANCE.deserialize(currentStateHandleInView);
         if (RocksDBSnapshotOperation.hasMetaDataFollowsFlag(key)) {
           // clear the signal bit in the key to make it ready for insertion again
           RocksDBSnapshotOperation.clearMetaDataFollowsFlag(key);
           rocksDBKeyedStateBackend.db.put(handle, key, value);
           // TODO this could be aware of keyGroupPrefixBytes and write only one byte if possible
           kvStateId =
               RocksDBSnapshotOperation.END_OF_KEY_GROUP_MARK
                   & currentStateHandleInView.readShort();
           if (RocksDBSnapshotOperation.END_OF_KEY_GROUP_MARK == kvStateId) {
             keyGroupHasMoreKeys = false;
           } else {
             handle = currentStateHandleKVStateColumnFamilies.get(kvStateId);
           }
         } else {
           rocksDBKeyedStateBackend.db.put(handle, key, value);
         }
       }
     }
   }
 }
コード例 #16
0
ファイル: StringValue.java プロジェクト: rmetzger/flink
  @Override
  public void read(final DataInputView in) throws IOException {
    int len = in.readUnsignedByte();

    if (len >= HIGH_BIT) {
      int shift = 7;
      int curr;
      len = len & 0x7f;
      while ((curr = in.readUnsignedByte()) >= HIGH_BIT) {
        len |= (curr & 0x7f) << shift;
        shift += 7;
      }
      len |= curr << shift;
    }

    this.len = len;
    this.hashCode = 0;
    ensureSize(len);
    final char[] data = this.value;

    for (int i = 0; i < len; i++) {
      int c = in.readUnsignedByte();
      if (c < HIGH_BIT) {
        data[i] = (char) c;
      } else {
        int shift = 7;
        int curr;
        c = c & 0x7f;
        while ((curr = in.readUnsignedByte()) >= HIGH_BIT) {
          c |= (curr & 0x7f) << shift;
          shift += 7;
        }
        c |= curr << shift;
        data[i] = (char) c;
      }
    }
  }
コード例 #17
0
  @Override
  public void read(final DataInputView in) throws IOException {
    int size = in.readInt();
    this.list.clear();

    try {
      for (; size > 0; size--) {
        final V val = this.valueClass.newInstance();
        val.read(in);

        this.list.add(val);
      }
    } catch (final InstantiationException e) {
      throw new RuntimeException(e);
    } catch (final IllegalAccessException e) {
      throw new RuntimeException(e);
    }
  }
コード例 #18
0
 @Override
 public void copy(DataInputView source, DataOutputView target) throws IOException {
   target.writeChar(source.readChar());
 }
コード例 #19
0
 @Override
 public Character deserialize(Character reuse, DataInputView source) throws IOException {
   return Character.valueOf(source.readChar());
 }
コード例 #20
0
  /** For backwards compatibility, remove again later! */
  @Deprecated
  private void restoreOldSavepointKeyedState(Collection<KeyGroupsStateHandle> restoreState)
      throws Exception {

    if (restoreState.isEmpty()) {
      return;
    }

    Preconditions.checkState(1 == restoreState.size(), "Only one element expected here.");
    HashMap<String, RocksDBStateBackend.FinalFullyAsyncSnapshot> namedStates =
        InstantiationUtil.deserializeObject(
            restoreState.iterator().next().openInputStream(), userCodeClassLoader);

    Preconditions.checkState(1 == namedStates.size(), "Only one element expected here.");
    DataInputView inputView =
        namedStates.values().iterator().next().stateHandle.getState(userCodeClassLoader);

    // clear k/v state information before filling it
    kvStateInformation.clear();

    // first get the column family mapping
    int numColumns = inputView.readInt();
    Map<Byte, StateDescriptor> columnFamilyMapping = new HashMap<>(numColumns);
    for (int i = 0; i < numColumns; i++) {
      byte mappingByte = inputView.readByte();

      ObjectInputStream ooIn =
          new InstantiationUtil.ClassLoaderObjectInputStream(
              new DataInputViewStream(inputView), userCodeClassLoader);

      StateDescriptor stateDescriptor = (StateDescriptor) ooIn.readObject();

      columnFamilyMapping.put(mappingByte, stateDescriptor);

      // this will fill in the k/v state information
      getColumnFamily(stateDescriptor);
    }

    // try and read until EOF
    try {
      // the EOFException will get us out of this...
      while (true) {
        byte mappingByte = inputView.readByte();
        ColumnFamilyHandle handle = getColumnFamily(columnFamilyMapping.get(mappingByte));
        byte[] keyAndNamespace = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView);

        ByteArrayInputStreamWithPos bis = new ByteArrayInputStreamWithPos(keyAndNamespace);

        K reconstructedKey = keySerializer.deserialize(new DataInputViewStreamWrapper(bis));
        int len = bis.getPosition();

        int keyGroup =
            (byte) KeyGroupRangeAssignment.assignToKeyGroup(reconstructedKey, numberOfKeyGroups);

        if (keyGroupPrefixBytes == 1) {
          // copy and override one byte (42) between key and namespace
          System.arraycopy(keyAndNamespace, 0, keyAndNamespace, 1, len);
          keyAndNamespace[0] = (byte) keyGroup;
        } else {
          byte[] largerKey = new byte[1 + keyAndNamespace.length];

          // write key-group
          largerKey[0] = (byte) ((keyGroup >> 8) & 0xFF);
          largerKey[1] = (byte) (keyGroup & 0xFF);

          // write key
          System.arraycopy(keyAndNamespace, 0, largerKey, 2, len);

          // skip one byte (42), write namespace
          System.arraycopy(
              keyAndNamespace, 1 + len, largerKey, 2 + len, keyAndNamespace.length - len - 1);
          keyAndNamespace = largerKey;
        }

        byte[] value = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView);
        db.put(handle, keyAndNamespace, value);
      }
    } catch (EOFException e) {
      // expected
    }
  }
コード例 #21
0
 @Override
 public void read(DataInputView in) throws IOException {
   splitNumber = in.readInt();
 }
コード例 #22
0
 @Override
 public void read(DataInputView in) throws IOException {
   this.lowerPart = in.readLong();
   this.upperPart = in.readLong();
 }
コード例 #23
0
 @Override
 public void read(DataInputView in) throws IOException {
   this.value = in.readChar();
 }