@Override
public boolean putMdAttr(String fileCode, MdAttr mdAttr) {
try {
db.put(fileCode.getBytes(RDB_DECODE), JSON.toJSONString(mdAttr).getBytes(RDB_DECODE));
return true;
} catch (Exception e) {
logger.error(String.format("[ERROR] caught the unexpected exception -- %s\n", e));
}
return false;
}
@Override
public int insert(String table, String key, HashMap<String, ByteIterator> values) {
try {
byte[] serialized = serialize(values);
db.put(key.getBytes(), serialized);
} catch (RocksDBException e) {
System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e);
assert (false);
}
return 0;
}
@Override
public boolean setOrCreateHashBucket(String distrCode, String fileCode) {
List<String> fileCodeList;
try {
byte[] bytes = db.get(distrCode.getBytes(RDB_DECODE));
if (bytes != null) {
fileCodeList = JSON.parseObject(new String(bytes, RDB_DECODE), List.class);
} else {
fileCodeList = new ArrayList<String>();
}
fileCodeList.add(fileCode);
db.put(String.valueOf(distrCode).getBytes(), JSON.toJSONString(fileCodeList).getBytes());
return true;
} catch (Exception e) {
logger.error(String.format("[ERROR] caught the unexpected exception -- %s\n", e));
}
return false;
}
@Test
public void snapshots() throws RocksDBException {
RocksDB db = null;
Options options = null;
ReadOptions readOptions = null;
try {
options = new Options();
options.setCreateIfMissing(true);
db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath());
db.put("key".getBytes(), "value".getBytes());
// Get new Snapshot of database
Snapshot snapshot = db.getSnapshot();
readOptions = new ReadOptions();
// set snapshot in ReadOptions
readOptions.setSnapshot(snapshot);
// retrieve key value pair
assertThat(new String(db.get("key".getBytes()))).isEqualTo("value");
// retrieve key value pair created before
// the snapshot was made
assertThat(new String(db.get(readOptions, "key".getBytes()))).isEqualTo("value");
// add new key/value pair
db.put("newkey".getBytes(), "newvalue".getBytes());
// using no snapshot the latest db entries
// will be taken into account
assertThat(new String(db.get("newkey".getBytes()))).isEqualTo("newvalue");
// snapshopot was created before newkey
assertThat(db.get(readOptions, "newkey".getBytes())).isNull();
// Retrieve snapshot from read options
Snapshot sameSnapshot = readOptions.snapshot();
readOptions.setSnapshot(sameSnapshot);
// results must be the same with new Snapshot
// instance using the same native pointer
assertThat(new String(db.get(readOptions, "key".getBytes()))).isEqualTo("value");
// update key value pair to newvalue
db.put("key".getBytes(), "newvalue".getBytes());
// read with previously created snapshot will
// read previous version of key value pair
assertThat(new String(db.get(readOptions, "key".getBytes()))).isEqualTo("value");
// read for newkey using the snapshot must be
// null
assertThat(db.get(readOptions, "newkey".getBytes())).isNull();
// setting null to snapshot in ReadOptions leads
// to no Snapshot being used.
readOptions.setSnapshot(null);
assertThat(new String(db.get(readOptions, "newkey".getBytes()))).isEqualTo("newvalue");
// release Snapshot
db.releaseSnapshot(snapshot);
} finally {
if (db != null) {
db.close();
}
if (options != null) {
options.dispose();
}
if (readOptions != null) {
readOptions.dispose();
}
}
}
/** For backwards compatibility, remove again later! */
@Deprecated
private void restoreOldSavepointKeyedState(Collection<KeyGroupsStateHandle> restoreState)
throws Exception {
if (restoreState.isEmpty()) {
return;
}
Preconditions.checkState(1 == restoreState.size(), "Only one element expected here.");
HashMap<String, RocksDBStateBackend.FinalFullyAsyncSnapshot> namedStates =
InstantiationUtil.deserializeObject(
restoreState.iterator().next().openInputStream(), userCodeClassLoader);
Preconditions.checkState(1 == namedStates.size(), "Only one element expected here.");
DataInputView inputView =
namedStates.values().iterator().next().stateHandle.getState(userCodeClassLoader);
// clear k/v state information before filling it
kvStateInformation.clear();
// first get the column family mapping
int numColumns = inputView.readInt();
Map<Byte, StateDescriptor> columnFamilyMapping = new HashMap<>(numColumns);
for (int i = 0; i < numColumns; i++) {
byte mappingByte = inputView.readByte();
ObjectInputStream ooIn =
new InstantiationUtil.ClassLoaderObjectInputStream(
new DataInputViewStream(inputView), userCodeClassLoader);
StateDescriptor stateDescriptor = (StateDescriptor) ooIn.readObject();
columnFamilyMapping.put(mappingByte, stateDescriptor);
// this will fill in the k/v state information
getColumnFamily(stateDescriptor);
}
// try and read until EOF
try {
// the EOFException will get us out of this...
while (true) {
byte mappingByte = inputView.readByte();
ColumnFamilyHandle handle = getColumnFamily(columnFamilyMapping.get(mappingByte));
byte[] keyAndNamespace = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView);
ByteArrayInputStreamWithPos bis = new ByteArrayInputStreamWithPos(keyAndNamespace);
K reconstructedKey = keySerializer.deserialize(new DataInputViewStreamWrapper(bis));
int len = bis.getPosition();
int keyGroup =
(byte) KeyGroupRangeAssignment.assignToKeyGroup(reconstructedKey, numberOfKeyGroups);
if (keyGroupPrefixBytes == 1) {
// copy and override one byte (42) between key and namespace
System.arraycopy(keyAndNamespace, 0, keyAndNamespace, 1, len);
keyAndNamespace[0] = (byte) keyGroup;
} else {
byte[] largerKey = new byte[1 + keyAndNamespace.length];
// write key-group
largerKey[0] = (byte) ((keyGroup >> 8) & 0xFF);
largerKey[1] = (byte) (keyGroup & 0xFF);
// write key
System.arraycopy(keyAndNamespace, 0, largerKey, 2, len);
// skip one byte (42), write namespace
System.arraycopy(
keyAndNamespace, 1 + len, largerKey, 2 + len, keyAndNamespace.length - len - 1);
keyAndNamespace = largerKey;
}
byte[] value = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView);
db.put(handle, keyAndNamespace, value);
}
} catch (EOFException e) {
// expected
}
}
@Override
public void init() throws DBException {
System.out.println("Initializing RocksDB...");
String db_path = DB_PATH;
options = new Options();
options
.setCreateIfMissing(true)
.createStatistics()
.setWriteBufferSize(8 * SizeUnit.KB)
.setMaxWriteBufferNumber(3)
.setMaxBackgroundCompactions(10)
.setCompressionType(CompressionType.SNAPPY_COMPRESSION)
.setCompactionStyle(CompactionStyle.UNIVERSAL);
Statistics stats = options.statisticsPtr();
assert (options.createIfMissing() == true);
assert (options.writeBufferSize() == 8 * SizeUnit.KB);
assert (options.maxWriteBufferNumber() == 3);
assert (options.maxBackgroundCompactions() == 10);
assert (options.compressionType() == CompressionType.SNAPPY_COMPRESSION);
assert (options.compactionStyle() == CompactionStyle.UNIVERSAL);
assert (options.memTableFactoryName().equals("SkipListFactory"));
options.setMemTableConfig(
new HashSkipListMemTableConfig()
.setHeight(4)
.setBranchingFactor(4)
.setBucketCount(2000000));
assert (options.memTableFactoryName().equals("HashSkipListRepFactory"));
options.setMemTableConfig(new HashLinkedListMemTableConfig().setBucketCount(100000));
assert (options.memTableFactoryName().equals("HashLinkedListRepFactory"));
options.setMemTableConfig(new VectorMemTableConfig().setReservedSize(10000));
assert (options.memTableFactoryName().equals("VectorRepFactory"));
options.setMemTableConfig(new SkipListMemTableConfig());
assert (options.memTableFactoryName().equals("SkipListFactory"));
// options.setTableFormatConfig(new PlainTableConfig());
// // Plain-Table requires mmap read
// options.setAllowMmapReads(true);
// assert(options.tableFactoryName().equals("PlainTable"));
//
// options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000,
// 10000, 10));
// options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000));
//
// Filter bloomFilter = new BloomFilter(10);
// BlockBasedTableConfig table_options = new BlockBasedTableConfig();
// table_options.setBlockCacheSize(64 * SizeUnit.KB)
// .setFilter(bloomFilter)
// .setCacheNumShardBits(6)
// .setBlockSizeDeviation(5)
// .setBlockRestartInterval(10)
// .setCacheIndexAndFilterBlocks(true)
// .setHashIndexAllowCollision(false)
// .setBlockCacheCompressedSize(64 * SizeUnit.KB)
// .setBlockCacheCompressedNumShardBits(10);
//
// assert(table_options.blockCacheSize() == 64 * SizeUnit.KB);
// assert(table_options.cacheNumShardBits() == 6);
// assert(table_options.blockSizeDeviation() == 5);
// assert(table_options.blockRestartInterval() == 10);
// assert(table_options.cacheIndexAndFilterBlocks() == true);
// assert(table_options.hashIndexAllowCollision() == false);
// assert(table_options.blockCacheCompressedSize() == 64 * SizeUnit.KB);
// assert(table_options.blockCacheCompressedNumShardBits() == 10);
//
// options.setTableFormatConfig(table_options);
// assert(options.tableFactoryName().equals("BlockBasedTable"));
try {
db = RocksDB.open(options, db_path);
db.put("hello".getBytes(), "world".getBytes());
byte[] value = db.get("hello".getBytes());
assert ("world".equals(new String(value)));
String str = db.getProperty("rocksdb.stats");
assert (str != null && str != "");
} catch (RocksDBException e) {
System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e);
assert (db == null);
assert (false);
}
System.out.println("Initializing RocksDB is over");
}