private void testFormat() throws IOException {
Map<Long, byte[]> map = New.hashMap();
StreamStore store = new StreamStore(map);
store.setMinBlockSize(10);
store.setMaxBlockSize(20);
store.setNextKey(123);
byte[] id;
id = store.put(new ByteArrayInputStream(new byte[200]));
assertEquals(200, store.length(id));
assertEquals("02c8018801", StringUtils.convertBytesToHex(id));
id = store.put(new ByteArrayInputStream(new byte[0]));
assertEquals("", StringUtils.convertBytesToHex(id));
id = store.put(new ByteArrayInputStream(new byte[1]));
assertEquals("000100", StringUtils.convertBytesToHex(id));
id = store.put(new ByteArrayInputStream(new byte[3]));
assertEquals("0003000000", StringUtils.convertBytesToHex(id));
id = store.put(new ByteArrayInputStream(new byte[10]));
assertEquals("010a8901", StringUtils.convertBytesToHex(id));
byte[] combined = StringUtils.convertHexToBytes("0001aa0002bbcc");
assertEquals(3, store.length(combined));
InputStream in = store.get(combined);
assertEquals(1, in.skip(1));
assertEquals(0xbb, in.read());
assertEquals(1, in.skip(1));
}
private ByteArrayInputStream nextBuffer() {
while (idBuffer.hasRemaining()) {
switch (idBuffer.get()) {
case 0:
{
int len = DataUtils.readVarInt(idBuffer);
if (skip >= len) {
skip -= len;
idBuffer.position(idBuffer.position() + len);
continue;
}
int p = (int) (idBuffer.position() + skip);
int l = (int) (len - skip);
idBuffer.position(p + l);
return new ByteArrayInputStream(idBuffer.array(), p, l);
}
case 1:
{
int len = DataUtils.readVarInt(idBuffer);
long key = DataUtils.readVarLong(idBuffer);
if (skip >= len) {
skip -= len;
continue;
}
byte[] data = store.getBlock(key);
int s = (int) skip;
skip = 0;
return new ByteArrayInputStream(data, s, data.length - s);
}
case 2:
{
long len = DataUtils.readVarLong(idBuffer);
long key = DataUtils.readVarLong(idBuffer);
if (skip >= len) {
skip -= len;
continue;
}
byte[] k = store.getBlock(key);
ByteBuffer newBuffer =
ByteBuffer.allocate(k.length + idBuffer.limit() - idBuffer.position());
newBuffer.put(k);
newBuffer.put(idBuffer);
newBuffer.flip();
idBuffer = newBuffer;
return nextBuffer();
}
default:
throw DataUtils.newIllegalArgumentException(
"Unsupported id {0}", Arrays.toString(idBuffer.array()));
}
}
return null;
}
@Override
public void init() {
if (init) {
return;
}
init = true;
Store s = database.getMvStore();
MVStore mvStore;
if (s == null) {
// in-memory database
mvStore = MVStore.open(null);
} else {
mvStore = s.getStore();
}
lobMap = mvStore.openMap("lobMap");
refMap = mvStore.openMap("lobRef");
dataMap = mvStore.openMap("lobData");
streamStore = new StreamStore(dataMap);
// garbage collection of the last blocks
if (database.isReadOnly()) {
return;
}
if (dataMap.isEmpty()) {
return;
}
// search the last referenced block
// (a lob may not have any referenced blocks if data is kept inline,
// so we need to loop)
long lastUsedKey = -1;
Long lobId = lobMap.lastKey();
while (lobId != null) {
Object[] v = lobMap.get(lobId);
byte[] id = (byte[]) v[0];
lastUsedKey = streamStore.getMaxBlockKey(id);
if (lastUsedKey >= 0) {
break;
}
lobId = lobMap.lowerKey(lobId);
}
// delete all blocks that are newer
while (true) {
Long last = dataMap.lastKey();
if (last == null || last <= lastUsedKey) {
break;
}
dataMap.remove(last);
}
// don't re-use block ids, except at the very end
Long last = dataMap.lastKey();
if (last != null) {
streamStore.setNextKey(last + 1);
}
}
private void removeLob(int tableId, long lobId) {
if (TRACE) {
trace("remove " + tableId + "/" + lobId);
}
Object[] value = lobMap.remove(lobId);
if (value == null) {
// already removed
return;
}
byte[] streamStoreId = (byte[]) value[0];
Object[] key = new Object[] {streamStoreId, lobId};
refMap.remove(key);
// check if there are more entries for this streamStoreId
key = new Object[] {streamStoreId, 0L};
value = refMap.ceilingKey(key);
boolean hasMoreEntries = false;
if (value != null) {
byte[] s2 = (byte[]) value[0];
if (Arrays.equals(streamStoreId, s2)) {
hasMoreEntries = true;
}
}
if (!hasMoreEntries) {
streamStore.remove(streamStoreId);
}
}
private void testLoop() throws IOException {
Map<Long, byte[]> map = New.hashMap();
StreamStore store = new StreamStore(map);
assertEquals(256 * 1024, store.getMaxBlockSize());
assertEquals(256, store.getMinBlockSize());
store.setNextKey(0);
assertEquals(0, store.getNextKey());
test(store, 10, 20, 1000);
for (int i = 0; i < 20; i++) {
test(store, 0, 128, i);
test(store, 10, 128, i);
}
for (int i = 20; i < 200; i += 10) {
test(store, 0, 128, i);
test(store, 10, 128, i);
}
}
private ValueLobDb createLob(InputStream in, int type) throws IOException {
byte[] streamStoreId;
try {
streamStoreId = streamStore.put(in);
} catch (Exception e) {
throw DbException.convertToIOException(e);
}
long lobId = generateLobId();
long length = streamStore.length(streamStoreId);
int tableId = LobStorageFrontend.TABLE_TEMP;
Object[] value = new Object[] {streamStoreId, tableId, length, 0};
lobMap.put(lobId, value);
Object[] key = new Object[] {streamStoreId, lobId};
refMap.put(key, Boolean.TRUE);
ValueLobDb lob = ValueLobDb.create(type, database, tableId, lobId, null, length);
if (TRACE) {
trace("create " + tableId + "/" + lobId);
}
return lob;
}
private void testWithFullMap() throws IOException {
final AtomicInteger tests = new AtomicInteger();
Map<Long, byte[]> map =
new HashMap<Long, byte[]>() {
private static final long serialVersionUID = 1L;
public boolean containsKey(Object k) {
tests.incrementAndGet();
if (((Long) k) < Long.MAX_VALUE / 2) {
// simulate a *very* full map
return true;
}
return super.containsKey(k);
}
};
StreamStore store = new StreamStore(map);
store.setMinBlockSize(20);
store.setMaxBlockSize(100);
store.setNextKey(0);
store.put(new ByteArrayInputStream(new byte[100]));
assertEquals(1, map.size());
assertEquals(64, tests.get());
assertEquals(Long.MAX_VALUE / 2 + 1, store.getNextKey());
}
private void testTreeStructure() throws IOException {
final AtomicInteger reads = new AtomicInteger();
Map<Long, byte[]> map =
new HashMap<Long, byte[]>() {
private static final long serialVersionUID = 1L;
public byte[] get(Object k) {
reads.incrementAndGet();
return super.get(k);
}
};
StreamStore store = new StreamStore(map);
store.setMinBlockSize(10);
store.setMaxBlockSize(100);
byte[] id = store.put(new ByteArrayInputStream(new byte[10000]));
InputStream in = store.get(id);
assertEquals(0, in.read());
assertEquals(3, reads.get());
}
private void testDetectIllegalId() throws IOException {
Map<Long, byte[]> map = New.hashMap();
StreamStore store = new StreamStore(map);
try {
store.length(new byte[] {3, 0, 0});
fail();
} catch (IllegalArgumentException e) {
// expected
}
try {
store.remove(new byte[] {3, 0, 0});
fail();
} catch (IllegalArgumentException e) {
// expected
}
map.put(0L, new byte[] {3, 0, 0});
InputStream in = store.get(new byte[] {2, 1, 0});
try {
in.read();
fail();
} catch (IllegalArgumentException e) {
// expected
}
}
@Override
public InputStream getInputStream(ValueLobDb lob, byte[] hmac, long byteCount)
throws IOException {
init();
Object[] value = lobMap.get(lob.getLobId());
if (value == null) {
if (lob.getTableId() == LobStorageFrontend.TABLE_RESULT
|| lob.getTableId() == LobStorageFrontend.TABLE_ID_SESSION_VARIABLE) {
throw DbException.get(
ErrorCode.LOB_CLOSED_ON_TIMEOUT_1, "" + lob.getLobId() + "/" + lob.getTableId());
}
throw DbException.throwInternalError(
"Lob not found: " + lob.getLobId() + "/" + lob.getTableId());
}
byte[] streamStoreId = (byte[]) value[0];
return streamStore.get(streamStoreId);
}
Stream(StreamStore store, byte[] id) {
this.store = store;
this.length = store.length(id);
this.idBuffer = ByteBuffer.wrap(id);
}
private void test(StreamStore store, int minBlockSize, int maxBlockSize, int length)
throws IOException {
store.setMinBlockSize(minBlockSize);
assertEquals(minBlockSize, store.getMinBlockSize());
store.setMaxBlockSize(maxBlockSize);
assertEquals(maxBlockSize, store.getMaxBlockSize());
long next = store.getNextKey();
Random r = new Random(length);
byte[] data = new byte[length];
r.nextBytes(data);
byte[] id = store.put(new ByteArrayInputStream(data));
if (length > 0 && length >= minBlockSize) {
assertFalse(store.isInPlace(id));
} else {
assertTrue(store.isInPlace(id));
}
long next2 = store.getNextKey();
if (length > 0 && length >= minBlockSize) {
assertTrue(next2 > next);
} else {
assertEquals(next, next2);
}
if (length == 0) {
assertEquals(0, id.length);
}
assertEquals(length, store.length(id));
InputStream in = store.get(id);
ByteArrayOutputStream out = new ByteArrayOutputStream();
IOUtils.copy(in, out);
assertTrue(Arrays.equals(data, out.toByteArray()));
in = store.get(id);
in.close();
assertEquals(-1, in.read());
in = store.get(id);
assertEquals(0, in.skip(0));
if (length > 0) {
assertEquals(1, in.skip(1));
if (length > 1) {
assertEquals(data[1] & 255, in.read());
if (length > 2) {
assertEquals(1, in.skip(1));
if (length > 3) {
assertEquals(data[3] & 255, in.read());
}
} else {
assertEquals(0, in.skip(1));
}
} else {
assertEquals(-1, in.read());
}
} else {
assertEquals(0, in.skip(1));
}
if (length > 12) {
in = store.get(id);
assertEquals(12, in.skip(12));
assertEquals(data[12] & 255, in.read());
long skipped = 0;
while (true) {
long s = in.skip(Integer.MAX_VALUE);
if (s == 0) {
break;
}
skipped += s;
}
assertEquals(length - 13, skipped);
assertEquals(-1, in.read());
}
store.remove(id);
assertEquals(0, store.getMap().size());
}
private void testWithExistingData() throws IOException {
final AtomicInteger tests = new AtomicInteger();
Map<Long, byte[]> map =
new HashMap<Long, byte[]>() {
private static final long serialVersionUID = 1L;
public boolean containsKey(Object k) {
tests.incrementAndGet();
return super.containsKey(k);
}
};
StreamStore store = new StreamStore(map);
store.setMinBlockSize(10);
store.setMaxBlockSize(20);
store.setNextKey(0);
for (int i = 0; i < 10; i++) {
store.put(new ByteArrayInputStream(new byte[20]));
}
assertEquals(10, map.size());
assertEquals(10, tests.get());
for (int i = 0; i < 10; i++) {
map.containsKey(i);
}
assertEquals(20, tests.get());
store = new StreamStore(map);
store.setMinBlockSize(10);
store.setMaxBlockSize(20);
store.setNextKey(0);
assertEquals(0, store.getNextKey());
for (int i = 0; i < 5; i++) {
store.put(new ByteArrayInputStream(new byte[20]));
}
assertEquals(88, tests.get());
assertEquals(15, store.getNextKey());
assertEquals(15, map.size());
for (int i = 0; i < 15; i++) {
map.containsKey(i);
}
}