private void testFormat() throws IOException {
Map<Long, byte[]> map = New.hashMap();
StreamStore store = new StreamStore(map);
store.setMinBlockSize(10);
store.setMaxBlockSize(20);
store.setNextKey(123);
byte[] id;
id = store.put(new ByteArrayInputStream(new byte[200]));
assertEquals(200, store.length(id));
assertEquals("02c8018801", StringUtils.convertBytesToHex(id));
id = store.put(new ByteArrayInputStream(new byte[0]));
assertEquals("", StringUtils.convertBytesToHex(id));
id = store.put(new ByteArrayInputStream(new byte[1]));
assertEquals("000100", StringUtils.convertBytesToHex(id));
id = store.put(new ByteArrayInputStream(new byte[3]));
assertEquals("0003000000", StringUtils.convertBytesToHex(id));
id = store.put(new ByteArrayInputStream(new byte[10]));
assertEquals("010a8901", StringUtils.convertBytesToHex(id));
byte[] combined = StringUtils.convertHexToBytes("0001aa0002bbcc");
assertEquals(3, store.length(combined));
InputStream in = store.get(combined);
assertEquals(1, in.skip(1));
assertEquals(0xbb, in.read());
assertEquals(1, in.skip(1));
}
private void testWithFullMap() throws IOException {
final AtomicInteger tests = new AtomicInteger();
Map<Long, byte[]> map =
new HashMap<Long, byte[]>() {
private static final long serialVersionUID = 1L;
public boolean containsKey(Object k) {
tests.incrementAndGet();
if (((Long) k) < Long.MAX_VALUE / 2) {
// simulate a *very* full map
return true;
}
return super.containsKey(k);
}
};
StreamStore store = new StreamStore(map);
store.setMinBlockSize(20);
store.setMaxBlockSize(100);
store.setNextKey(0);
store.put(new ByteArrayInputStream(new byte[100]));
assertEquals(1, map.size());
assertEquals(64, tests.get());
assertEquals(Long.MAX_VALUE / 2 + 1, store.getNextKey());
}
private void testWithExistingData() throws IOException {
final AtomicInteger tests = new AtomicInteger();
Map<Long, byte[]> map =
new HashMap<Long, byte[]>() {
private static final long serialVersionUID = 1L;
public boolean containsKey(Object k) {
tests.incrementAndGet();
return super.containsKey(k);
}
};
StreamStore store = new StreamStore(map);
store.setMinBlockSize(10);
store.setMaxBlockSize(20);
store.setNextKey(0);
for (int i = 0; i < 10; i++) {
store.put(new ByteArrayInputStream(new byte[20]));
}
assertEquals(10, map.size());
assertEquals(10, tests.get());
for (int i = 0; i < 10; i++) {
map.containsKey(i);
}
assertEquals(20, tests.get());
store = new StreamStore(map);
store.setMinBlockSize(10);
store.setMaxBlockSize(20);
store.setNextKey(0);
assertEquals(0, store.getNextKey());
for (int i = 0; i < 5; i++) {
store.put(new ByteArrayInputStream(new byte[20]));
}
assertEquals(88, tests.get());
assertEquals(15, store.getNextKey());
assertEquals(15, map.size());
for (int i = 0; i < 15; i++) {
map.containsKey(i);
}
}
@Override
public void init() {
if (init) {
return;
}
init = true;
Store s = database.getMvStore();
MVStore mvStore;
if (s == null) {
// in-memory database
mvStore = MVStore.open(null);
} else {
mvStore = s.getStore();
}
lobMap = mvStore.openMap("lobMap");
refMap = mvStore.openMap("lobRef");
dataMap = mvStore.openMap("lobData");
streamStore = new StreamStore(dataMap);
// garbage collection of the last blocks
if (database.isReadOnly()) {
return;
}
if (dataMap.isEmpty()) {
return;
}
// search the last referenced block
// (a lob may not have any referenced blocks if data is kept inline,
// so we need to loop)
long lastUsedKey = -1;
Long lobId = lobMap.lastKey();
while (lobId != null) {
Object[] v = lobMap.get(lobId);
byte[] id = (byte[]) v[0];
lastUsedKey = streamStore.getMaxBlockKey(id);
if (lastUsedKey >= 0) {
break;
}
lobId = lobMap.lowerKey(lobId);
}
// delete all blocks that are newer
while (true) {
Long last = dataMap.lastKey();
if (last == null || last <= lastUsedKey) {
break;
}
dataMap.remove(last);
}
// don't re-use block ids, except at the very end
Long last = dataMap.lastKey();
if (last != null) {
streamStore.setNextKey(last + 1);
}
}
private void testLoop() throws IOException {
Map<Long, byte[]> map = New.hashMap();
StreamStore store = new StreamStore(map);
assertEquals(256 * 1024, store.getMaxBlockSize());
assertEquals(256, store.getMinBlockSize());
store.setNextKey(0);
assertEquals(0, store.getNextKey());
test(store, 10, 20, 1000);
for (int i = 0; i < 20; i++) {
test(store, 0, 128, i);
test(store, 10, 128, i);
}
for (int i = 20; i < 200; i += 10) {
test(store, 0, 128, i);
test(store, 10, 128, i);
}
}