private void init() {
String s = settings.get(LAST_TRANSACTION_ID);
if (s != null) {
lastTransactionId = Long.parseLong(s);
lastTransactionIdStored = lastTransactionId;
}
Long lastKey = openTransactions.lastKey();
if (lastKey != null && lastKey.longValue() > lastTransactionId) {
throw DataUtils.newIllegalStateException("Last transaction not stored");
}
Cursor<Long> cursor = openTransactions.keyIterator(null);
while (cursor.hasNext()) {
long id = cursor.next();
Object[] data = openTransactions.get(id);
int status = (Integer) data[0];
String name = (String) data[1];
long[] next = {id + 1, -1};
long[] last = undoLog.floorKey(next);
if (last == null) {
// no entry
} else if (last[0] == id) {
Transaction t = new Transaction(this, id, status, name, last[1]);
t.setStored(true);
openTransactionMap.put(id, t);
}
}
}
private Object[] getArray(K key, long maxLog) {
Object[] data = map.get(key);
while (true) {
long tx;
if (data == null) {
// doesn't exist or deleted by a committed transaction
return null;
}
tx = (Long) data[0];
long logId = (Long) data[1];
if (tx == transaction.transactionId) {
// added by this transaction
if (logId < maxLog) {
return data;
}
}
// added or updated by another transaction
boolean open = transaction.store.openTransactions.containsKey(tx);
if (!open) {
// it is committed
return data;
}
// get the value before the uncommitted transaction
long[] x = new long[] {tx, logId};
data = transaction.store.undoLog.get(x);
data = (Object[]) data[3];
}
}
void persistMovieSet(MovieSet movieSet) throws Exception {
String newValue = movieSetObjectWriter.writeValueAsString(movieSet);
String oldValue = movieMap.get(movieSet.getDbId());
if (!StringUtils.equals(newValue, oldValue)) {
movieSetMap.put(movieSet.getDbId(), newValue);
}
}
/**
* Prepare a transaction.
*
* @param transactionId the transaction id
*/
void prepare(Transaction t) {
storeTransaction(t);
Object[] old = openTransactions.get(t.getId());
Object[] v = {Transaction.STATUS_PREPARED, old[1]};
openTransactions.put(t.getId(), v);
store.commit();
}
void persistMovie(Movie movie) throws Exception {
String newValue = movieObjectWriter.writeValueAsString(movie);
String oldValue = movieMap.get(movie.getDbId());
if (!StringUtils.equals(newValue, oldValue)) {
// write movie to DB
movieMap.put(movie.getDbId(), newValue);
}
}
@Override
public void init() {
if (init) {
return;
}
init = true;
Store s = database.getMvStore();
MVStore mvStore;
if (s == null) {
// in-memory database
mvStore = MVStore.open(null);
} else {
mvStore = s.getStore();
}
lobMap = mvStore.openMap("lobMap");
refMap = mvStore.openMap("lobRef");
dataMap = mvStore.openMap("lobData");
streamStore = new StreamStore(dataMap);
// garbage collection of the last blocks
if (database.isReadOnly()) {
return;
}
if (dataMap.isEmpty()) {
return;
}
// search the last referenced block
// (a lob may not have any referenced blocks if data is kept inline,
// so we need to loop)
long lastUsedKey = -1;
Long lobId = lobMap.lastKey();
while (lobId != null) {
Object[] v = lobMap.get(lobId);
byte[] id = (byte[]) v[0];
lastUsedKey = streamStore.getMaxBlockKey(id);
if (lastUsedKey >= 0) {
break;
}
lobId = lobMap.lowerKey(lobId);
}
// delete all blocks that are newer
while (true) {
Long last = dataMap.lastKey();
if (last == null || last <= lastUsedKey) {
break;
}
dataMap.remove(last);
}
// don't re-use block ids, except at the very end
Long last = dataMap.lastKey();
if (last != null) {
streamStore.setNextKey(last + 1);
}
}
/** Test the concurrent map implementation. */
private void testConcurrentMap() throws InterruptedException {
final MVStore s = openStore(null);
final MVMap<Integer, Integer> m =
s.openMap("data", new MVMapConcurrent.Builder<Integer, Integer>());
final int size = 20;
final Random rand = new Random(1);
Task task =
new Task() {
@Override
public void call() throws Exception {
try {
while (!stop) {
if (rand.nextBoolean()) {
m.put(rand.nextInt(size), 1);
} else {
m.remove(rand.nextInt(size));
}
m.get(rand.nextInt(size));
m.firstKey();
m.lastKey();
m.ceilingKey(5);
m.floorKey(5);
m.higherKey(5);
m.lowerKey(5);
for (Iterator<Integer> it = m.keyIterator(null); it.hasNext(); ) {
it.next();
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
};
task.execute();
Thread.sleep(1);
for (int j = 0; j < 100; j++) {
for (int i = 0; i < 100; i++) {
if (rand.nextBoolean()) {
m.put(rand.nextInt(size), 2);
} else {
m.remove(rand.nextInt(size));
}
m.get(rand.nextInt(size));
}
s.commit();
Thread.sleep(1);
}
task.get();
s.close();
}
/**
* Commit a transaction.
*
* @param t the transaction
* @param maxLogId the last log id
*/
void commit(Transaction t, long maxLogId) {
for (long logId = 0; logId < maxLogId; logId++) {
long[] undoKey = new long[] {t.getId(), logId};
Object[] op = undoLog.get(undoKey);
int opType = (Integer) op[0];
if (opType == Transaction.OP_REMOVE) {
int mapId = (Integer) op[1];
Map<String, String> meta = store.getMetaMap();
String m = meta.get("map." + mapId);
String mapName = DataUtils.parseMap(m).get("name");
MVMap<Object, Object[]> map = store.openMap(mapName);
Object key = op[2];
Object[] value = map.get(key);
// possibly the entry was added later on
// so we have to check
if (value[2] == null) {
// remove the value
map.remove(key);
}
}
undoLog.remove(undoKey);
}
endTransaction(t);
}
private void testConcurrentOnlineBackup() throws Exception {
String fileName = getBaseDir() + "/onlineBackup.h3";
String fileNameRestore = getBaseDir() + "/onlineRestore.h3";
final MVStore s = openStore(fileName);
final MVMap<Integer, byte[]> map = s.openMap("test");
final Random r = new Random();
Task t =
new Task() {
@Override
public void call() throws Exception {
while (!stop) {
for (int i = 0; i < 10; i++) {
map.put(i, new byte[100 * r.nextInt(100)]);
}
s.commit();
map.clear();
s.commit();
long len = s.getFileStore().size();
if (len > 1024 * 1024) {
// slow down writing a lot
Thread.sleep(200);
} else if (len > 20 * 1024) {
// slow down writing
Thread.sleep(20);
}
}
}
};
t.execute();
for (int i = 0; i < 10; i++) {
// System.out.println("test " + i);
s.setReuseSpace(false);
byte[] buff = readFileSlowly(s.getFileStore().getFile(), s.getFileStore().size());
s.setReuseSpace(true);
FileOutputStream out = new FileOutputStream(fileNameRestore);
out.write(buff);
MVStore s2 = openStore(fileNameRestore);
MVMap<Integer, byte[]> test = s2.openMap("test");
for (Integer k : test.keySet()) {
test.get(k);
}
s2.close();
// let it compact
Thread.sleep(10);
}
t.get();
s.close();
}
@Override
public InputStream getInputStream(ValueLobDb lob, byte[] hmac, long byteCount)
throws IOException {
init();
Object[] value = lobMap.get(lob.getLobId());
if (value == null) {
if (lob.getTableId() == LobStorageFrontend.TABLE_RESULT
|| lob.getTableId() == LobStorageFrontend.TABLE_ID_SESSION_VARIABLE) {
throw DbException.get(
ErrorCode.LOB_CLOSED_ON_TIMEOUT_1, "" + lob.getLobId() + "/" + lob.getTableId());
}
throw DbException.throwInternalError(
"Lob not found: " + lob.getLobId() + "/" + lob.getTableId());
}
byte[] streamStoreId = (byte[]) value[0];
return streamStore.get(streamStoreId);
}
/**
* Rollback to an old savepoint.
*
* @param t the transaction
* @param maxLogId the last log id
* @param toLogId the log id to roll back to
*/
void rollbackTo(Transaction t, long maxLogId, long toLogId) {
for (long logId = maxLogId - 1; logId >= toLogId; logId--) {
Object[] op = undoLog.get(new long[] {t.getId(), logId});
int mapId = ((Integer) op[1]).intValue();
Map<String, String> meta = store.getMetaMap();
String m = meta.get("map." + mapId);
String mapName = DataUtils.parseMap(m).get("name");
MVMap<Object, Object[]> map = store.openMap(mapName);
Object key = op[2];
Object[] oldValue = (Object[]) op[3];
if (oldValue == null) {
// this transaction added the value
map.remove(key);
} else {
// this transaction updated the value
map.put(key, oldValue);
}
undoLog.remove(op);
}
}
@Override
public ValueLobDb copyLob(ValueLobDb old, int tableId, long length) {
init();
int type = old.getType();
long oldLobId = old.getLobId();
long oldLength = old.getPrecision();
if (oldLength != length) {
throw DbException.throwInternalError("Length is different");
}
Object[] value = lobMap.get(oldLobId);
value = value.clone();
byte[] streamStoreId = (byte[]) value[0];
long lobId = generateLobId();
value[1] = tableId;
lobMap.put(lobId, value);
Object[] key = new Object[] {streamStoreId, lobId};
refMap.put(key, Boolean.TRUE);
ValueLobDb lob = ValueLobDb.create(type, database, tableId, lobId, null, length);
if (TRACE) {
trace("copy " + old.getTableId() + "/" + old.getLobId() + " > " + tableId + "/" + lobId);
}
return lob;
}
/**
* Try to set or remove the value. When updating only unchanged entries, then the value is only
* changed if it was not changed after opening the map.
*
* @param key the key
* @param value the new value (null to remove the value)
* @param onlyIfUnchanged only set the value if it was not changed (by this or another
* transaction) since the map was opened
* @return true if the value was set
*/
public boolean trySet(K key, V value, boolean onlyIfUnchanged) {
Object[] current = map.get(key);
if (onlyIfUnchanged) {
Object[] old = getArray(key, readLogId);
if (!map.areValuesEqual(old, current)) {
long tx = (Long) current[0];
if (tx == transaction.transactionId) {
if (value == null) {
// ignore removing an entry
// if it was added or changed
// in the same statement
return true;
} else if (current[2] == null) {
// add an entry that was removed
// in the same statement
} else {
return false;
}
} else {
return false;
}
}
}
int opType;
if (current == null || current[2] == null) {
if (value == null) {
// remove a removed value
opType = Transaction.OP_SET;
} else {
opType = Transaction.OP_ADD;
}
} else {
if (value == null) {
opType = Transaction.OP_REMOVE;
} else {
opType = Transaction.OP_SET;
}
}
Object[] newValue = {transaction.transactionId, transaction.logId, value};
if (current == null) {
// a new value
Object[] old = map.putIfAbsent(key, newValue);
if (old == null) {
transaction.log(opType, mapId, key, current);
return true;
}
return false;
}
long tx = (Long) current[0];
if (tx == transaction.transactionId) {
// added or updated by this transaction
if (map.replace(key, current, newValue)) {
transaction.log(opType, mapId, key, current);
return true;
}
// strange, somebody overwrite the value
// even thought the change was not committed
return false;
}
// added or updated by another transaction
boolean open = transaction.store.openTransactions.containsKey(tx);
if (!open) {
// the transaction is committed:
// overwrite the value
if (map.replace(key, current, newValue)) {
transaction.log(opType, mapId, key, current);
return true;
}
// somebody else was faster
return false;
}
// the transaction is not yet committed
return false;
}
/**
* Set the name of a transaction.
*
* @param t the transaction
* @param name the new name
*/
void setTransactionName(Transaction t, String name) {
storeTransaction(t);
Object[] old = openTransactions.get(t.getId());
Object[] v = {old[0], name};
openTransactions.put(t.getId(), v);
}
private void testConcurrentWrite(final AtomicInteger detected, final AtomicInteger notDetected)
throws InterruptedException {
final MVStore s = openStore(null);
final MVMap<Integer, Integer> m = s.openMap("data");
final int size = 20;
final Random rand = new Random(1);
Task task =
new Task() {
@Override
public void call() throws Exception {
while (!stop) {
try {
if (rand.nextBoolean()) {
m.put(rand.nextInt(size), 1);
} else {
m.remove(rand.nextInt(size));
}
m.get(rand.nextInt(size));
} catch (ConcurrentModificationException e) {
detected.incrementAndGet();
} catch (NegativeArraySizeException e) {
notDetected.incrementAndGet();
} catch (ArrayIndexOutOfBoundsException e) {
notDetected.incrementAndGet();
} catch (IllegalArgumentException e) {
notDetected.incrementAndGet();
} catch (NullPointerException e) {
notDetected.incrementAndGet();
}
}
}
};
task.execute();
Thread.sleep(1);
for (int j = 0; j < 10; j++) {
for (int i = 0; i < 10; i++) {
try {
if (rand.nextBoolean()) {
m.put(rand.nextInt(size), 2);
} else {
m.remove(rand.nextInt(size));
}
m.get(rand.nextInt(size));
} catch (ConcurrentModificationException e) {
detected.incrementAndGet();
} catch (NegativeArraySizeException e) {
notDetected.incrementAndGet();
} catch (ArrayIndexOutOfBoundsException e) {
notDetected.incrementAndGet();
} catch (IllegalArgumentException e) {
notDetected.incrementAndGet();
} catch (NullPointerException e) {
notDetected.incrementAndGet();
}
}
s.commit();
Thread.sleep(1);
}
task.get();
s.close();
}
