Пример #1
0
 @Override
 public boolean putMdAttr(String fileCode, MdAttr mdAttr) {
   try {
     db.put(fileCode.getBytes(RDB_DECODE), JSON.toJSONString(mdAttr).getBytes(RDB_DECODE));
     return true;
   } catch (Exception e) {
     logger.error(String.format("[ERROR] caught the unexpected exception -- %s\n", e));
   }
   return false;
 }
Пример #2
0
 @Override
 public int insert(String table, String key, HashMap<String, ByteIterator> values) {
   try {
     byte[] serialized = serialize(values);
     db.put(key.getBytes(), serialized);
   } catch (RocksDBException e) {
     System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e);
     assert (false);
   }
   return 0;
 }
Пример #3
0
 @Override
 public boolean setOrCreateHashBucket(String distrCode, String fileCode) {
   List<String> fileCodeList;
   try {
     byte[] bytes = db.get(distrCode.getBytes(RDB_DECODE));
     if (bytes != null) {
       fileCodeList = JSON.parseObject(new String(bytes, RDB_DECODE), List.class);
     } else {
       fileCodeList = new ArrayList<String>();
     }
     fileCodeList.add(fileCode);
     db.put(String.valueOf(distrCode).getBytes(), JSON.toJSONString(fileCodeList).getBytes());
     return true;
   } catch (Exception e) {
     logger.error(String.format("[ERROR] caught the unexpected exception -- %s\n", e));
   }
   return false;
 }
Пример #4
0
  @Test
  public void snapshots() throws RocksDBException {
    RocksDB db = null;
    Options options = null;
    ReadOptions readOptions = null;
    try {

      options = new Options();
      options.setCreateIfMissing(true);

      db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath());
      db.put("key".getBytes(), "value".getBytes());
      // Get new Snapshot of database
      Snapshot snapshot = db.getSnapshot();
      readOptions = new ReadOptions();
      // set snapshot in ReadOptions
      readOptions.setSnapshot(snapshot);
      // retrieve key value pair
      assertThat(new String(db.get("key".getBytes()))).isEqualTo("value");
      // retrieve key value pair created before
      // the snapshot was made
      assertThat(new String(db.get(readOptions, "key".getBytes()))).isEqualTo("value");
      // add new key/value pair
      db.put("newkey".getBytes(), "newvalue".getBytes());
      // using no snapshot the latest db entries
      // will be taken into account
      assertThat(new String(db.get("newkey".getBytes()))).isEqualTo("newvalue");
      // snapshopot was created before newkey
      assertThat(db.get(readOptions, "newkey".getBytes())).isNull();
      // Retrieve snapshot from read options
      Snapshot sameSnapshot = readOptions.snapshot();
      readOptions.setSnapshot(sameSnapshot);
      // results must be the same with new Snapshot
      // instance using the same native pointer
      assertThat(new String(db.get(readOptions, "key".getBytes()))).isEqualTo("value");
      // update key value pair to newvalue
      db.put("key".getBytes(), "newvalue".getBytes());
      // read with previously created snapshot will
      // read previous version of key value pair
      assertThat(new String(db.get(readOptions, "key".getBytes()))).isEqualTo("value");
      // read for newkey using the snapshot must be
      // null
      assertThat(db.get(readOptions, "newkey".getBytes())).isNull();
      // setting null to snapshot in ReadOptions leads
      // to no Snapshot being used.
      readOptions.setSnapshot(null);
      assertThat(new String(db.get(readOptions, "newkey".getBytes()))).isEqualTo("newvalue");
      // release Snapshot
      db.releaseSnapshot(snapshot);
    } finally {
      if (db != null) {
        db.close();
      }
      if (options != null) {
        options.dispose();
      }
      if (readOptions != null) {
        readOptions.dispose();
      }
    }
  }
Пример #5
0
  /** For backwards compatibility, remove again later! */
  @Deprecated
  private void restoreOldSavepointKeyedState(Collection<KeyGroupsStateHandle> restoreState)
      throws Exception {

    if (restoreState.isEmpty()) {
      return;
    }

    Preconditions.checkState(1 == restoreState.size(), "Only one element expected here.");
    HashMap<String, RocksDBStateBackend.FinalFullyAsyncSnapshot> namedStates =
        InstantiationUtil.deserializeObject(
            restoreState.iterator().next().openInputStream(), userCodeClassLoader);

    Preconditions.checkState(1 == namedStates.size(), "Only one element expected here.");
    DataInputView inputView =
        namedStates.values().iterator().next().stateHandle.getState(userCodeClassLoader);

    // clear k/v state information before filling it
    kvStateInformation.clear();

    // first get the column family mapping
    int numColumns = inputView.readInt();
    Map<Byte, StateDescriptor> columnFamilyMapping = new HashMap<>(numColumns);
    for (int i = 0; i < numColumns; i++) {
      byte mappingByte = inputView.readByte();

      ObjectInputStream ooIn =
          new InstantiationUtil.ClassLoaderObjectInputStream(
              new DataInputViewStream(inputView), userCodeClassLoader);

      StateDescriptor stateDescriptor = (StateDescriptor) ooIn.readObject();

      columnFamilyMapping.put(mappingByte, stateDescriptor);

      // this will fill in the k/v state information
      getColumnFamily(stateDescriptor);
    }

    // try and read until EOF
    try {
      // the EOFException will get us out of this...
      while (true) {
        byte mappingByte = inputView.readByte();
        ColumnFamilyHandle handle = getColumnFamily(columnFamilyMapping.get(mappingByte));
        byte[] keyAndNamespace = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView);

        ByteArrayInputStreamWithPos bis = new ByteArrayInputStreamWithPos(keyAndNamespace);

        K reconstructedKey = keySerializer.deserialize(new DataInputViewStreamWrapper(bis));
        int len = bis.getPosition();

        int keyGroup =
            (byte) KeyGroupRangeAssignment.assignToKeyGroup(reconstructedKey, numberOfKeyGroups);

        if (keyGroupPrefixBytes == 1) {
          // copy and override one byte (42) between key and namespace
          System.arraycopy(keyAndNamespace, 0, keyAndNamespace, 1, len);
          keyAndNamespace[0] = (byte) keyGroup;
        } else {
          byte[] largerKey = new byte[1 + keyAndNamespace.length];

          // write key-group
          largerKey[0] = (byte) ((keyGroup >> 8) & 0xFF);
          largerKey[1] = (byte) (keyGroup & 0xFF);

          // write key
          System.arraycopy(keyAndNamespace, 0, largerKey, 2, len);

          // skip one byte (42), write namespace
          System.arraycopy(
              keyAndNamespace, 1 + len, largerKey, 2 + len, keyAndNamespace.length - len - 1);
          keyAndNamespace = largerKey;
        }

        byte[] value = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView);
        db.put(handle, keyAndNamespace, value);
      }
    } catch (EOFException e) {
      // expected
    }
  }
Пример #6
0
  @Override
  public void init() throws DBException {
    System.out.println("Initializing RocksDB...");
    String db_path = DB_PATH;
    options = new Options();
    options
        .setCreateIfMissing(true)
        .createStatistics()
        .setWriteBufferSize(8 * SizeUnit.KB)
        .setMaxWriteBufferNumber(3)
        .setMaxBackgroundCompactions(10)
        .setCompressionType(CompressionType.SNAPPY_COMPRESSION)
        .setCompactionStyle(CompactionStyle.UNIVERSAL);
    Statistics stats = options.statisticsPtr();

    assert (options.createIfMissing() == true);
    assert (options.writeBufferSize() == 8 * SizeUnit.KB);
    assert (options.maxWriteBufferNumber() == 3);
    assert (options.maxBackgroundCompactions() == 10);
    assert (options.compressionType() == CompressionType.SNAPPY_COMPRESSION);
    assert (options.compactionStyle() == CompactionStyle.UNIVERSAL);

    assert (options.memTableFactoryName().equals("SkipListFactory"));
    options.setMemTableConfig(
        new HashSkipListMemTableConfig()
            .setHeight(4)
            .setBranchingFactor(4)
            .setBucketCount(2000000));
    assert (options.memTableFactoryName().equals("HashSkipListRepFactory"));

    options.setMemTableConfig(new HashLinkedListMemTableConfig().setBucketCount(100000));
    assert (options.memTableFactoryName().equals("HashLinkedListRepFactory"));

    options.setMemTableConfig(new VectorMemTableConfig().setReservedSize(10000));
    assert (options.memTableFactoryName().equals("VectorRepFactory"));

    options.setMemTableConfig(new SkipListMemTableConfig());
    assert (options.memTableFactoryName().equals("SkipListFactory"));

    //        options.setTableFormatConfig(new PlainTableConfig());
    //        // Plain-Table requires mmap read
    //        options.setAllowMmapReads(true);
    //        assert(options.tableFactoryName().equals("PlainTable"));
    //
    //        options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000,
    //                10000, 10));
    //        options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000));
    //
    //        Filter bloomFilter = new BloomFilter(10);
    //        BlockBasedTableConfig table_options = new BlockBasedTableConfig();
    //        table_options.setBlockCacheSize(64 * SizeUnit.KB)
    //                .setFilter(bloomFilter)
    //                .setCacheNumShardBits(6)
    //                .setBlockSizeDeviation(5)
    //                .setBlockRestartInterval(10)
    //                .setCacheIndexAndFilterBlocks(true)
    //                .setHashIndexAllowCollision(false)
    //                .setBlockCacheCompressedSize(64 * SizeUnit.KB)
    //                .setBlockCacheCompressedNumShardBits(10);
    //
    //        assert(table_options.blockCacheSize() == 64 * SizeUnit.KB);
    //        assert(table_options.cacheNumShardBits() == 6);
    //        assert(table_options.blockSizeDeviation() == 5);
    //        assert(table_options.blockRestartInterval() == 10);
    //        assert(table_options.cacheIndexAndFilterBlocks() == true);
    //        assert(table_options.hashIndexAllowCollision() == false);
    //        assert(table_options.blockCacheCompressedSize() == 64 * SizeUnit.KB);
    //        assert(table_options.blockCacheCompressedNumShardBits() == 10);
    //
    //        options.setTableFormatConfig(table_options);
    //        assert(options.tableFactoryName().equals("BlockBasedTable"));

    try {
      db = RocksDB.open(options, db_path);
      db.put("hello".getBytes(), "world".getBytes());
      byte[] value = db.get("hello".getBytes());
      assert ("world".equals(new String(value)));
      String str = db.getProperty("rocksdb.stats");
      assert (str != null && str != "");
    } catch (RocksDBException e) {
      System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e);
      assert (db == null);
      assert (false);
    }

    System.out.println("Initializing RocksDB is over");
  }