private void init() {
String s = settings.get(LAST_TRANSACTION_ID);
if (s != null) {
lastTransactionId = Long.parseLong(s);
lastTransactionIdStored = lastTransactionId;
}
Long lastKey = openTransactions.lastKey();
if (lastKey != null && lastKey.longValue() > lastTransactionId) {
throw DataUtils.newIllegalStateException("Last transaction not stored");
}
Cursor<Long> cursor = openTransactions.keyIterator(null);
while (cursor.hasNext()) {
long id = cursor.next();
Object[] data = openTransactions.get(id);
int status = (Integer) data[0];
String name = (String) data[1];
long[] next = {id + 1, -1};
long[] last = undoLog.floorKey(next);
if (last == null) {
// no entry
} else if (last[0] == id) {
Transaction t = new Transaction(this, id, status, name, last[1]);
t.setStored(true);
openTransactionMap.put(id, t);
}
}
}
private void testConcurrentIterate() {
MVStore s = new MVStore.Builder().pageSplitSize(3).open();
s.setVersionsToKeep(100);
final MVMap<Integer, Integer> map = s.openMap("test");
final int len = 10;
final Random r = new Random();
Task t =
new Task() {
@Override
public void call() throws Exception {
while (!stop) {
int x = r.nextInt(len);
if (r.nextBoolean()) {
map.remove(x);
} else {
map.put(x, r.nextInt(100));
}
}
}
};
t.execute();
for (int k = 0; k < 10000; k++) {
Iterator<Integer> it = map.keyIterator(r.nextInt(len));
long old = s.getCurrentVersion();
s.commit();
while (map.getVersion() == old) {
Thread.yield();
}
while (it.hasNext()) {
it.next();
}
}
t.get();
s.close();
}
private void testConcurrentStoreAndRemoveMap() throws InterruptedException {
String fileName = "memFS:testConcurrentStoreAndRemoveMap.h3";
final MVStore s = openStore(fileName);
int count = 200;
for (int i = 0; i < count; i++) {
MVMap<Integer, Integer> m = s.openMap("d" + i);
m.put(1, 1);
}
final AtomicInteger counter = new AtomicInteger();
Task task =
new Task() {
@Override
public void call() throws Exception {
while (!stop) {
counter.incrementAndGet();
s.commit();
}
}
};
task.execute();
Thread.sleep(1);
for (int i = 0; i < count || counter.get() < count; i++) {
MVMap<Integer, Integer> m = s.openMap("d" + i);
m.put(1, 10);
s.removeMap(m);
if (task.isFinished()) {
break;
}
}
task.get();
s.close();
FileUtils.deleteRecursive("memFS:", false);
}
void persistMovieSet(MovieSet movieSet) throws Exception {
String newValue = movieSetObjectWriter.writeValueAsString(movieSet);
String oldValue = movieMap.get(movieSet.getDbId());
if (!StringUtils.equals(newValue, oldValue)) {
movieSetMap.put(movieSet.getDbId(), newValue);
}
}
/**
* Prepare a transaction.
*
* @param transactionId the transaction id
*/
void prepare(Transaction t) {
storeTransaction(t);
Object[] old = openTransactions.get(t.getId());
Object[] v = {Transaction.STATUS_PREPARED, old[1]};
openTransactions.put(t.getId(), v);
store.commit();
}
private void removeLob(int tableId, long lobId) {
if (TRACE) {
trace("remove " + tableId + "/" + lobId);
}
Object[] value = lobMap.remove(lobId);
if (value == null) {
// already removed
return;
}
byte[] streamStoreId = (byte[]) value[0];
Object[] key = new Object[] {streamStoreId, lobId};
refMap.remove(key);
// check if there are more entries for this streamStoreId
key = new Object[] {streamStoreId, 0L};
value = refMap.ceilingKey(key);
boolean hasMoreEntries = false;
if (value != null) {
byte[] s2 = (byte[]) value[0];
if (Arrays.equals(streamStoreId, s2)) {
hasMoreEntries = true;
}
}
if (!hasMoreEntries) {
streamStore.remove(streamStoreId);
}
}
void persistMovie(Movie movie) throws Exception {
String newValue = movieObjectWriter.writeValueAsString(movie);
String oldValue = movieMap.get(movie.getDbId());
if (!StringUtils.equals(newValue, oldValue)) {
// write movie to DB
movieMap.put(movie.getDbId(), newValue);
}
}
/**
* Open a copy of the map in read-only mode.
*
* @return the opened map
*/
protected MVMap<K, V> openReadOnly() {
MVMap<K, V> m = new MVMap<K, V>(keyType, valueType);
m.readOnly = true;
HashMap<String, String> config = New.hashMap();
config.put("id", String.valueOf(id));
config.put("createVersion", String.valueOf(createVersion));
m.init(store, config);
m.root = root;
return m;
}
@Override
public void setTable(ValueLobDb lob, int tableId) {
init();
long lobId = lob.getLobId();
Object[] value = lobMap.remove(lobId);
if (TRACE) {
trace("move " + lob.getTableId() + "/" + lob.getLobId() + " > " + tableId + "/" + lobId);
}
value[1] = tableId;
lobMap.put(lobId, value);
}
/** Test the concurrent map implementation. */
private void testConcurrentMap() throws InterruptedException {
final MVStore s = openStore(null);
final MVMap<Integer, Integer> m =
s.openMap("data", new MVMapConcurrent.Builder<Integer, Integer>());
final int size = 20;
final Random rand = new Random(1);
Task task =
new Task() {
@Override
public void call() throws Exception {
try {
while (!stop) {
if (rand.nextBoolean()) {
m.put(rand.nextInt(size), 1);
} else {
m.remove(rand.nextInt(size));
}
m.get(rand.nextInt(size));
m.firstKey();
m.lastKey();
m.ceilingKey(5);
m.floorKey(5);
m.higherKey(5);
m.lowerKey(5);
for (Iterator<Integer> it = m.keyIterator(null); it.hasNext(); ) {
it.next();
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
};
task.execute();
Thread.sleep(1);
for (int j = 0; j < 100; j++) {
for (int i = 0; i < 100; i++) {
if (rand.nextBoolean()) {
m.put(rand.nextInt(size), 2);
} else {
m.remove(rand.nextInt(size));
}
m.get(rand.nextInt(size));
}
s.commit();
Thread.sleep(1);
}
task.get();
s.close();
}
private void testConcurrentOnlineBackup() throws Exception {
String fileName = getBaseDir() + "/onlineBackup.h3";
String fileNameRestore = getBaseDir() + "/onlineRestore.h3";
final MVStore s = openStore(fileName);
final MVMap<Integer, byte[]> map = s.openMap("test");
final Random r = new Random();
Task t =
new Task() {
@Override
public void call() throws Exception {
while (!stop) {
for (int i = 0; i < 10; i++) {
map.put(i, new byte[100 * r.nextInt(100)]);
}
s.commit();
map.clear();
s.commit();
long len = s.getFileStore().size();
if (len > 1024 * 1024) {
// slow down writing a lot
Thread.sleep(200);
} else if (len > 20 * 1024) {
// slow down writing
Thread.sleep(20);
}
}
}
};
t.execute();
for (int i = 0; i < 10; i++) {
// System.out.println("test " + i);
s.setReuseSpace(false);
byte[] buff = readFileSlowly(s.getFileStore().getFile(), s.getFileStore().size());
s.setReuseSpace(true);
FileOutputStream out = new FileOutputStream(fileNameRestore);
out.write(buff);
MVStore s2 = openStore(fileNameRestore);
MVMap<Integer, byte[]> test = s2.openMap("test");
for (Integer k : test.keySet()) {
test.get(k);
}
s2.close();
// let it compact
Thread.sleep(10);
}
t.get();
s.close();
}
private Object[] getArray(K key, long maxLog) {
Object[] data = map.get(key);
while (true) {
long tx;
if (data == null) {
// doesn't exist or deleted by a committed transaction
return null;
}
tx = (Long) data[0];
long logId = (Long) data[1];
if (tx == transaction.transactionId) {
// added by this transaction
if (logId < maxLog) {
return data;
}
}
// added or updated by another transaction
boolean open = transaction.store.openTransactions.containsKey(tx);
if (!open) {
// it is committed
return data;
}
// get the value before the uncommitted transaction
long[] x = new long[] {tx, logId};
data = transaction.store.undoLog.get(x);
data = (Object[]) data[3];
}
}
private void storeTransaction(Transaction t) {
if (store.getUnsavedPageCount() > MAX_UNSAVED_PAGES) {
store.commit();
}
if (t.isStored()) {
return;
}
t.setStored(true);
long transactionId = t.getId();
Object[] v = {t.getStatus(), null};
openTransactions.put(transactionId, v);
openTransactionMap.put(transactionId, t);
if (lastTransactionId > lastTransactionIdStored) {
lastTransactionIdStored += 32;
settings.put(LAST_TRANSACTION_ID, "" + lastTransactionIdStored);
}
}
TransactionMap(Transaction transaction, String name, DataType keyType, DataType valueType) {
this.transaction = transaction;
ArrayType arrayType =
new ArrayType(new DataType[] {new ObjectDataType(), new ObjectDataType(), valueType});
MVMap.Builder<K, Object[]> builder =
new MVMap.Builder<K, Object[]>().keyType(keyType).valueType(arrayType);
map = transaction.store.store.openMap(name, builder);
mapId = map.getId();
}
private long generateLobId() {
synchronized (nextLobIdSync) {
if (nextLobId == 0) {
Long id = lobMap.lastKey();
nextLobId = id == null ? 1 : id + 1;
}
return nextLobId++;
}
}
private ValueLobDb createLob(InputStream in, int type) throws IOException {
byte[] streamStoreId;
try {
streamStoreId = streamStore.put(in);
} catch (Exception e) {
throw DbException.convertToIOException(e);
}
long lobId = generateLobId();
long length = streamStore.length(streamStoreId);
int tableId = LobStorageFrontend.TABLE_TEMP;
Object[] value = new Object[] {streamStoreId, tableId, length, 0};
lobMap.put(lobId, value);
Object[] key = new Object[] {streamStoreId, lobId};
refMap.put(key, Boolean.TRUE);
ValueLobDb lob = ValueLobDb.create(type, database, tableId, lobId, null, length);
if (TRACE) {
trace("create " + tableId + "/" + lobId);
}
return lob;
}
/**
* Rollback to an old savepoint.
*
* @param t the transaction
* @param maxLogId the last log id
* @param toLogId the log id to roll back to
*/
void rollbackTo(Transaction t, long maxLogId, long toLogId) {
for (long logId = maxLogId - 1; logId >= toLogId; logId--) {
Object[] op = undoLog.get(new long[] {t.getId(), logId});
int mapId = ((Integer) op[1]).intValue();
Map<String, String> meta = store.getMetaMap();
String m = meta.get("map." + mapId);
String mapName = DataUtils.parseMap(m).get("name");
MVMap<Object, Object[]> map = store.openMap(mapName);
Object key = op[2];
Object[] oldValue = (Object[]) op[3];
if (oldValue == null) {
// this transaction added the value
map.remove(key);
} else {
// this transaction updated the value
map.put(key, oldValue);
}
undoLog.remove(op);
}
}
/**
* Get the size of the map as seen by this transaction.
*
* @return the size
*/
public long getSize() {
// TODO this method is very slow
long size = 0;
Cursor<K> cursor = map.keyIterator(null);
while (cursor.hasNext()) {
K key = cursor.next();
if (get(key) != null) {
size++;
}
}
return size;
}
private void testConcurrentRead() throws InterruptedException {
final MVStore s = openStore(null);
final MVMap<Integer, Integer> m = s.openMap("data");
final int size = 3;
int x = (int) s.getCurrentVersion();
for (int i = 0; i < size; i++) {
m.put(i, x);
}
s.commit();
Task task =
new Task() {
@Override
public void call() throws Exception {
while (!stop) {
long v = s.getCurrentVersion() - 1;
Map<Integer, Integer> old = m.openVersion(v);
for (int i = 0; i < size; i++) {
Integer x = old.get(i);
if (x == null || (int) v != x) {
Map<Integer, Integer> old2 = m.openVersion(v);
throw new AssertionError(x + "<>" + v + " at " + i + " " + old2);
}
}
}
}
};
task.execute();
Thread.sleep(1);
for (int j = 0; j < 100; j++) {
x = (int) s.getCurrentVersion();
for (int i = 0; i < size; i++) {
m.put(i, x);
}
s.commit();
Thread.sleep(1);
}
task.get();
s.close();
}
@Override
public void init() {
if (init) {
return;
}
init = true;
Store s = database.getMvStore();
MVStore mvStore;
if (s == null) {
// in-memory database
mvStore = MVStore.open(null);
} else {
mvStore = s.getStore();
}
lobMap = mvStore.openMap("lobMap");
refMap = mvStore.openMap("lobRef");
dataMap = mvStore.openMap("lobData");
streamStore = new StreamStore(dataMap);
// garbage collection of the last blocks
if (database.isReadOnly()) {
return;
}
if (dataMap.isEmpty()) {
return;
}
// search the last referenced block
// (a lob may not have any referenced blocks if data is kept inline,
// so we need to loop)
long lastUsedKey = -1;
Long lobId = lobMap.lastKey();
while (lobId != null) {
Object[] v = lobMap.get(lobId);
byte[] id = (byte[]) v[0];
lastUsedKey = streamStore.getMaxBlockKey(id);
if (lastUsedKey >= 0) {
break;
}
lobId = lobMap.lowerKey(lobId);
}
// delete all blocks that are newer
while (true) {
Long last = dataMap.lastKey();
if (last == null || last <= lastUsedKey) {
break;
}
dataMap.remove(last);
}
// don't re-use block ids, except at the very end
Long last = dataMap.lastKey();
if (last != null) {
streamStore.setNextKey(last + 1);
}
}
@Override
public ValueLobDb copyLob(ValueLobDb old, int tableId, long length) {
init();
int type = old.getType();
long oldLobId = old.getLobId();
long oldLength = old.getPrecision();
if (oldLength != length) {
throw DbException.throwInternalError("Length is different");
}
Object[] value = lobMap.get(oldLobId);
value = value.clone();
byte[] streamStoreId = (byte[]) value[0];
long lobId = generateLobId();
value[1] = tableId;
lobMap.put(lobId, value);
Object[] key = new Object[] {streamStoreId, lobId};
refMap.put(key, Boolean.TRUE);
ValueLobDb lob = ValueLobDb.create(type, database, tableId, lobId, null, length);
if (TRACE) {
trace("copy " + old.getTableId() + "/" + old.getLobId() + " > " + tableId + "/" + lobId);
}
return lob;
}
/**
* Commit a transaction.
*
* @param t the transaction
* @param maxLogId the last log id
*/
void commit(Transaction t, long maxLogId) {
for (long logId = 0; logId < maxLogId; logId++) {
long[] undoKey = new long[] {t.getId(), logId};
Object[] op = undoLog.get(undoKey);
int opType = (Integer) op[0];
if (opType == Transaction.OP_REMOVE) {
int mapId = (Integer) op[1];
Map<String, String> meta = store.getMetaMap();
String m = meta.get("map." + mapId);
String mapName = DataUtils.parseMap(m).get("name");
MVMap<Object, Object[]> map = store.openMap(mapName);
Object key = op[2];
Object[] value = map.get(key);
// possibly the entry was added later on
// so we have to check
if (value[2] == null) {
// remove the value
map.remove(key);
}
}
undoLog.remove(undoKey);
}
endTransaction(t);
}
/**
* Open an old version for the given map.
*
* @param version the version
* @return the map
*/
public MVMap<K, V> openVersion(long version) {
if (readOnly) {
throw DataUtils.newUnsupportedOperationException(
"This map is read-only - need to call the method on the writable map");
}
DataUtils.checkArgument(
version >= createVersion,
"Unknown version {0}; this map was created in version is {1}",
version,
createVersion);
Page newest = null;
// need to copy because it can change
Page r = root;
if (version >= r.getVersion()
&& (version == store.getCurrentVersion()
|| r.getVersion() >= 0
|| version <= createVersion
|| store.getFile() == null)) {
newest = r;
} else {
// find the newest page that has a getVersion() <= version
int i = searchRoot(version);
if (i < 0) {
// not found
if (i == -1) {
// smaller than all in-memory versions
return store.openMapVersion(version, id, this);
}
i = -i - 2;
}
newest = oldRoots.get(i);
}
MVMap<K, V> m = openReadOnly();
m.root = newest;
return m;
}
@Override
public InputStream getInputStream(ValueLobDb lob, byte[] hmac, long byteCount)
throws IOException {
init();
Object[] value = lobMap.get(lob.getLobId());
if (value == null) {
if (lob.getTableId() == LobStorageFrontend.TABLE_RESULT
|| lob.getTableId() == LobStorageFrontend.TABLE_ID_SESSION_VARIABLE) {
throw DbException.get(
ErrorCode.LOB_CLOSED_ON_TIMEOUT_1, "" + lob.getLobId() + "/" + lob.getTableId());
}
throw DbException.throwInternalError(
"Lob not found: " + lob.getLobId() + "/" + lob.getTableId());
}
byte[] streamStoreId = (byte[]) value[0];
return streamStore.get(streamStoreId);
}
@Override
public void removeAllForTable(int tableId) {
init();
if (database.getMvStore().getStore().isClosed()) {
return;
}
// this might not be very efficient -
// to speed it up, we would need yet another map
ArrayList<Long> list = New.arrayList();
for (Entry<Long, Object[]> e : lobMap.entrySet()) {
Object[] value = e.getValue();
int t = (Integer) value[1];
if (t == tableId) {
list.add(e.getKey());
}
}
for (long lobId : list) {
removeLob(tableId, lobId);
}
if (tableId == LobStorageFrontend.TABLE_ID_SESSION_VARIABLE) {
removeAllForTable(LobStorageFrontend.TABLE_TEMP);
removeAllForTable(LobStorageFrontend.TABLE_RESULT);
}
}
/**
* Get the largest key that is smaller than the given key, or null if no such key exists.
*
* @param key the key (may not be null)
* @return the result
*/
public K lowerKey(K key) {
// TODO Auto-generated method stub
return map.lowerKey(key);
}
/**
* Get the smallest key that is larger than the given key, or null if no such key exists.
*
* @param key the key (may not be null)
* @return the result
*/
public K higherKey(K key) {
// TODO transactional higherKey
return map.higherKey(key);
}
/**
* Get the smallest key that is larger or equal to this key.
*
* @param key the key (may not be null)
* @return the result
*/
public K ceilingKey(K key) {
// TODO transactional ceilingKey
return map.ceilingKey(key);
}
/**
* Get the last key.
*
* @return the last key, or null if empty
*/
public K lastKey() {
// TODO transactional lastKey
return map.lastKey();
}
/**
* Get the first key.
*
* @return the first key, or null if empty
*/
public K firstKey() {
// TODO transactional firstKey
return map.firstKey();
}