@Override public boolean putMdAttr(String fileCode, MdAttr mdAttr) { try { db.put(fileCode.getBytes(RDB_DECODE), JSON.toJSONString(mdAttr).getBytes(RDB_DECODE)); return true; } catch (Exception e) { logger.error(String.format("[ERROR] caught the unexpected exception -- %s\n", e)); } return false; }
@Override public int insert(String table, String key, HashMap<String, ByteIterator> values) { try { byte[] serialized = serialize(values); db.put(key.getBytes(), serialized); } catch (RocksDBException e) { System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e); assert (false); } return 0; }
@Override public boolean setOrCreateHashBucket(String distrCode, String fileCode) { List<String> fileCodeList; try { byte[] bytes = db.get(distrCode.getBytes(RDB_DECODE)); if (bytes != null) { fileCodeList = JSON.parseObject(new String(bytes, RDB_DECODE), List.class); } else { fileCodeList = new ArrayList<String>(); } fileCodeList.add(fileCode); db.put(String.valueOf(distrCode).getBytes(), JSON.toJSONString(fileCodeList).getBytes()); return true; } catch (Exception e) { logger.error(String.format("[ERROR] caught the unexpected exception -- %s\n", e)); } return false; }
@Test public void snapshots() throws RocksDBException { RocksDB db = null; Options options = null; ReadOptions readOptions = null; try { options = new Options(); options.setCreateIfMissing(true); db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath()); db.put("key".getBytes(), "value".getBytes()); // Get new Snapshot of database Snapshot snapshot = db.getSnapshot(); readOptions = new ReadOptions(); // set snapshot in ReadOptions readOptions.setSnapshot(snapshot); // retrieve key value pair assertThat(new String(db.get("key".getBytes()))).isEqualTo("value"); // retrieve key value pair created before // the snapshot was made assertThat(new String(db.get(readOptions, "key".getBytes()))).isEqualTo("value"); // add new key/value pair db.put("newkey".getBytes(), "newvalue".getBytes()); // using no snapshot the latest db entries // will be taken into account assertThat(new String(db.get("newkey".getBytes()))).isEqualTo("newvalue"); // snapshopot was created before newkey assertThat(db.get(readOptions, "newkey".getBytes())).isNull(); // Retrieve snapshot from read options Snapshot sameSnapshot = readOptions.snapshot(); readOptions.setSnapshot(sameSnapshot); // results must be the same with new Snapshot // instance using the same native pointer assertThat(new String(db.get(readOptions, "key".getBytes()))).isEqualTo("value"); // update key value pair to newvalue db.put("key".getBytes(), "newvalue".getBytes()); // read with previously created snapshot will // read previous version of key value pair assertThat(new String(db.get(readOptions, "key".getBytes()))).isEqualTo("value"); // read for newkey using the snapshot must be // null assertThat(db.get(readOptions, "newkey".getBytes())).isNull(); // setting null to snapshot in ReadOptions leads // to no Snapshot being used. readOptions.setSnapshot(null); assertThat(new String(db.get(readOptions, "newkey".getBytes()))).isEqualTo("newvalue"); // release Snapshot db.releaseSnapshot(snapshot); } finally { if (db != null) { db.close(); } if (options != null) { options.dispose(); } if (readOptions != null) { readOptions.dispose(); } } }
/** For backwards compatibility, remove again later! */ @Deprecated private void restoreOldSavepointKeyedState(Collection<KeyGroupsStateHandle> restoreState) throws Exception { if (restoreState.isEmpty()) { return; } Preconditions.checkState(1 == restoreState.size(), "Only one element expected here."); HashMap<String, RocksDBStateBackend.FinalFullyAsyncSnapshot> namedStates = InstantiationUtil.deserializeObject( restoreState.iterator().next().openInputStream(), userCodeClassLoader); Preconditions.checkState(1 == namedStates.size(), "Only one element expected here."); DataInputView inputView = namedStates.values().iterator().next().stateHandle.getState(userCodeClassLoader); // clear k/v state information before filling it kvStateInformation.clear(); // first get the column family mapping int numColumns = inputView.readInt(); Map<Byte, StateDescriptor> columnFamilyMapping = new HashMap<>(numColumns); for (int i = 0; i < numColumns; i++) { byte mappingByte = inputView.readByte(); ObjectInputStream ooIn = new InstantiationUtil.ClassLoaderObjectInputStream( new DataInputViewStream(inputView), userCodeClassLoader); StateDescriptor stateDescriptor = (StateDescriptor) ooIn.readObject(); columnFamilyMapping.put(mappingByte, stateDescriptor); // this will fill in the k/v state information getColumnFamily(stateDescriptor); } // try and read until EOF try { // the EOFException will get us out of this... while (true) { byte mappingByte = inputView.readByte(); ColumnFamilyHandle handle = getColumnFamily(columnFamilyMapping.get(mappingByte)); byte[] keyAndNamespace = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView); ByteArrayInputStreamWithPos bis = new ByteArrayInputStreamWithPos(keyAndNamespace); K reconstructedKey = keySerializer.deserialize(new DataInputViewStreamWrapper(bis)); int len = bis.getPosition(); int keyGroup = (byte) KeyGroupRangeAssignment.assignToKeyGroup(reconstructedKey, numberOfKeyGroups); if (keyGroupPrefixBytes == 1) { // copy and override one byte (42) between key and namespace System.arraycopy(keyAndNamespace, 0, keyAndNamespace, 1, len); keyAndNamespace[0] = (byte) keyGroup; } else { byte[] largerKey = new byte[1 + keyAndNamespace.length]; // write key-group largerKey[0] = (byte) ((keyGroup >> 8) & 0xFF); largerKey[1] = (byte) (keyGroup & 0xFF); // write key System.arraycopy(keyAndNamespace, 0, largerKey, 2, len); // skip one byte (42), write namespace System.arraycopy( keyAndNamespace, 1 + len, largerKey, 2 + len, keyAndNamespace.length - len - 1); keyAndNamespace = largerKey; } byte[] value = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView); db.put(handle, keyAndNamespace, value); } } catch (EOFException e) { // expected } }
@Override public void init() throws DBException { System.out.println("Initializing RocksDB..."); String db_path = DB_PATH; options = new Options(); options .setCreateIfMissing(true) .createStatistics() .setWriteBufferSize(8 * SizeUnit.KB) .setMaxWriteBufferNumber(3) .setMaxBackgroundCompactions(10) .setCompressionType(CompressionType.SNAPPY_COMPRESSION) .setCompactionStyle(CompactionStyle.UNIVERSAL); Statistics stats = options.statisticsPtr(); assert (options.createIfMissing() == true); assert (options.writeBufferSize() == 8 * SizeUnit.KB); assert (options.maxWriteBufferNumber() == 3); assert (options.maxBackgroundCompactions() == 10); assert (options.compressionType() == CompressionType.SNAPPY_COMPRESSION); assert (options.compactionStyle() == CompactionStyle.UNIVERSAL); assert (options.memTableFactoryName().equals("SkipListFactory")); options.setMemTableConfig( new HashSkipListMemTableConfig() .setHeight(4) .setBranchingFactor(4) .setBucketCount(2000000)); assert (options.memTableFactoryName().equals("HashSkipListRepFactory")); options.setMemTableConfig(new HashLinkedListMemTableConfig().setBucketCount(100000)); assert (options.memTableFactoryName().equals("HashLinkedListRepFactory")); options.setMemTableConfig(new VectorMemTableConfig().setReservedSize(10000)); assert (options.memTableFactoryName().equals("VectorRepFactory")); options.setMemTableConfig(new SkipListMemTableConfig()); assert (options.memTableFactoryName().equals("SkipListFactory")); // options.setTableFormatConfig(new PlainTableConfig()); // // Plain-Table requires mmap read // options.setAllowMmapReads(true); // assert(options.tableFactoryName().equals("PlainTable")); // // options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000, // 10000, 10)); // options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000)); // // Filter bloomFilter = new BloomFilter(10); // BlockBasedTableConfig table_options = new BlockBasedTableConfig(); // table_options.setBlockCacheSize(64 * SizeUnit.KB) // .setFilter(bloomFilter) // .setCacheNumShardBits(6) // .setBlockSizeDeviation(5) // .setBlockRestartInterval(10) // .setCacheIndexAndFilterBlocks(true) // .setHashIndexAllowCollision(false) // .setBlockCacheCompressedSize(64 * SizeUnit.KB) // .setBlockCacheCompressedNumShardBits(10); // // assert(table_options.blockCacheSize() == 64 * SizeUnit.KB); // assert(table_options.cacheNumShardBits() == 6); // assert(table_options.blockSizeDeviation() == 5); // assert(table_options.blockRestartInterval() == 10); // assert(table_options.cacheIndexAndFilterBlocks() == true); // assert(table_options.hashIndexAllowCollision() == false); // assert(table_options.blockCacheCompressedSize() == 64 * SizeUnit.KB); // assert(table_options.blockCacheCompressedNumShardBits() == 10); // // options.setTableFormatConfig(table_options); // assert(options.tableFactoryName().equals("BlockBasedTable")); try { db = RocksDB.open(options, db_path); db.put("hello".getBytes(), "world".getBytes()); byte[] value = db.get("hello".getBytes()); assert ("world".equals(new String(value))); String str = db.getProperty("rocksdb.stats"); assert (str != null && str != ""); } catch (RocksDBException e) { System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e); assert (db == null); assert (false); } System.out.println("Initializing RocksDB is over"); }