private void init() { String s = settings.get(LAST_TRANSACTION_ID); if (s != null) { lastTransactionId = Long.parseLong(s); lastTransactionIdStored = lastTransactionId; } Long lastKey = openTransactions.lastKey(); if (lastKey != null && lastKey.longValue() > lastTransactionId) { throw DataUtils.newIllegalStateException("Last transaction not stored"); } Cursor<Long> cursor = openTransactions.keyIterator(null); while (cursor.hasNext()) { long id = cursor.next(); Object[] data = openTransactions.get(id); int status = (Integer) data[0]; String name = (String) data[1]; long[] next = {id + 1, -1}; long[] last = undoLog.floorKey(next); if (last == null) { // no entry } else if (last[0] == id) { Transaction t = new Transaction(this, id, status, name, last[1]); t.setStored(true); openTransactionMap.put(id, t); } } }
private void testConcurrentIterate() { MVStore s = new MVStore.Builder().pageSplitSize(3).open(); s.setVersionsToKeep(100); final MVMap<Integer, Integer> map = s.openMap("test"); final int len = 10; final Random r = new Random(); Task t = new Task() { @Override public void call() throws Exception { while (!stop) { int x = r.nextInt(len); if (r.nextBoolean()) { map.remove(x); } else { map.put(x, r.nextInt(100)); } } } }; t.execute(); for (int k = 0; k < 10000; k++) { Iterator<Integer> it = map.keyIterator(r.nextInt(len)); long old = s.getCurrentVersion(); s.commit(); while (map.getVersion() == old) { Thread.yield(); } while (it.hasNext()) { it.next(); } } t.get(); s.close(); }
private void testConcurrentStoreAndRemoveMap() throws InterruptedException { String fileName = "memFS:testConcurrentStoreAndRemoveMap.h3"; final MVStore s = openStore(fileName); int count = 200; for (int i = 0; i < count; i++) { MVMap<Integer, Integer> m = s.openMap("d" + i); m.put(1, 1); } final AtomicInteger counter = new AtomicInteger(); Task task = new Task() { @Override public void call() throws Exception { while (!stop) { counter.incrementAndGet(); s.commit(); } } }; task.execute(); Thread.sleep(1); for (int i = 0; i < count || counter.get() < count; i++) { MVMap<Integer, Integer> m = s.openMap("d" + i); m.put(1, 10); s.removeMap(m); if (task.isFinished()) { break; } } task.get(); s.close(); FileUtils.deleteRecursive("memFS:", false); }
void persistMovieSet(MovieSet movieSet) throws Exception { String newValue = movieSetObjectWriter.writeValueAsString(movieSet); String oldValue = movieMap.get(movieSet.getDbId()); if (!StringUtils.equals(newValue, oldValue)) { movieSetMap.put(movieSet.getDbId(), newValue); } }
/** * Prepare a transaction. * * @param transactionId the transaction id */ void prepare(Transaction t) { storeTransaction(t); Object[] old = openTransactions.get(t.getId()); Object[] v = {Transaction.STATUS_PREPARED, old[1]}; openTransactions.put(t.getId(), v); store.commit(); }
private void removeLob(int tableId, long lobId) { if (TRACE) { trace("remove " + tableId + "/" + lobId); } Object[] value = lobMap.remove(lobId); if (value == null) { // already removed return; } byte[] streamStoreId = (byte[]) value[0]; Object[] key = new Object[] {streamStoreId, lobId}; refMap.remove(key); // check if there are more entries for this streamStoreId key = new Object[] {streamStoreId, 0L}; value = refMap.ceilingKey(key); boolean hasMoreEntries = false; if (value != null) { byte[] s2 = (byte[]) value[0]; if (Arrays.equals(streamStoreId, s2)) { hasMoreEntries = true; } } if (!hasMoreEntries) { streamStore.remove(streamStoreId); } }
void persistMovie(Movie movie) throws Exception { String newValue = movieObjectWriter.writeValueAsString(movie); String oldValue = movieMap.get(movie.getDbId()); if (!StringUtils.equals(newValue, oldValue)) { // write movie to DB movieMap.put(movie.getDbId(), newValue); } }
/** * Open a copy of the map in read-only mode. * * @return the opened map */ protected MVMap<K, V> openReadOnly() { MVMap<K, V> m = new MVMap<K, V>(keyType, valueType); m.readOnly = true; HashMap<String, String> config = New.hashMap(); config.put("id", String.valueOf(id)); config.put("createVersion", String.valueOf(createVersion)); m.init(store, config); m.root = root; return m; }
@Override public void setTable(ValueLobDb lob, int tableId) { init(); long lobId = lob.getLobId(); Object[] value = lobMap.remove(lobId); if (TRACE) { trace("move " + lob.getTableId() + "/" + lob.getLobId() + " > " + tableId + "/" + lobId); } value[1] = tableId; lobMap.put(lobId, value); }
/** Test the concurrent map implementation. */ private void testConcurrentMap() throws InterruptedException { final MVStore s = openStore(null); final MVMap<Integer, Integer> m = s.openMap("data", new MVMapConcurrent.Builder<Integer, Integer>()); final int size = 20; final Random rand = new Random(1); Task task = new Task() { @Override public void call() throws Exception { try { while (!stop) { if (rand.nextBoolean()) { m.put(rand.nextInt(size), 1); } else { m.remove(rand.nextInt(size)); } m.get(rand.nextInt(size)); m.firstKey(); m.lastKey(); m.ceilingKey(5); m.floorKey(5); m.higherKey(5); m.lowerKey(5); for (Iterator<Integer> it = m.keyIterator(null); it.hasNext(); ) { it.next(); } } } catch (Exception e) { e.printStackTrace(); } } }; task.execute(); Thread.sleep(1); for (int j = 0; j < 100; j++) { for (int i = 0; i < 100; i++) { if (rand.nextBoolean()) { m.put(rand.nextInt(size), 2); } else { m.remove(rand.nextInt(size)); } m.get(rand.nextInt(size)); } s.commit(); Thread.sleep(1); } task.get(); s.close(); }
private void testConcurrentOnlineBackup() throws Exception { String fileName = getBaseDir() + "/onlineBackup.h3"; String fileNameRestore = getBaseDir() + "/onlineRestore.h3"; final MVStore s = openStore(fileName); final MVMap<Integer, byte[]> map = s.openMap("test"); final Random r = new Random(); Task t = new Task() { @Override public void call() throws Exception { while (!stop) { for (int i = 0; i < 10; i++) { map.put(i, new byte[100 * r.nextInt(100)]); } s.commit(); map.clear(); s.commit(); long len = s.getFileStore().size(); if (len > 1024 * 1024) { // slow down writing a lot Thread.sleep(200); } else if (len > 20 * 1024) { // slow down writing Thread.sleep(20); } } } }; t.execute(); for (int i = 0; i < 10; i++) { // System.out.println("test " + i); s.setReuseSpace(false); byte[] buff = readFileSlowly(s.getFileStore().getFile(), s.getFileStore().size()); s.setReuseSpace(true); FileOutputStream out = new FileOutputStream(fileNameRestore); out.write(buff); MVStore s2 = openStore(fileNameRestore); MVMap<Integer, byte[]> test = s2.openMap("test"); for (Integer k : test.keySet()) { test.get(k); } s2.close(); // let it compact Thread.sleep(10); } t.get(); s.close(); }
private Object[] getArray(K key, long maxLog) { Object[] data = map.get(key); while (true) { long tx; if (data == null) { // doesn't exist or deleted by a committed transaction return null; } tx = (Long) data[0]; long logId = (Long) data[1]; if (tx == transaction.transactionId) { // added by this transaction if (logId < maxLog) { return data; } } // added or updated by another transaction boolean open = transaction.store.openTransactions.containsKey(tx); if (!open) { // it is committed return data; } // get the value before the uncommitted transaction long[] x = new long[] {tx, logId}; data = transaction.store.undoLog.get(x); data = (Object[]) data[3]; } }
private void storeTransaction(Transaction t) { if (store.getUnsavedPageCount() > MAX_UNSAVED_PAGES) { store.commit(); } if (t.isStored()) { return; } t.setStored(true); long transactionId = t.getId(); Object[] v = {t.getStatus(), null}; openTransactions.put(transactionId, v); openTransactionMap.put(transactionId, t); if (lastTransactionId > lastTransactionIdStored) { lastTransactionIdStored += 32; settings.put(LAST_TRANSACTION_ID, "" + lastTransactionIdStored); } }
TransactionMap(Transaction transaction, String name, DataType keyType, DataType valueType) { this.transaction = transaction; ArrayType arrayType = new ArrayType(new DataType[] {new ObjectDataType(), new ObjectDataType(), valueType}); MVMap.Builder<K, Object[]> builder = new MVMap.Builder<K, Object[]>().keyType(keyType).valueType(arrayType); map = transaction.store.store.openMap(name, builder); mapId = map.getId(); }
private long generateLobId() { synchronized (nextLobIdSync) { if (nextLobId == 0) { Long id = lobMap.lastKey(); nextLobId = id == null ? 1 : id + 1; } return nextLobId++; } }
private ValueLobDb createLob(InputStream in, int type) throws IOException { byte[] streamStoreId; try { streamStoreId = streamStore.put(in); } catch (Exception e) { throw DbException.convertToIOException(e); } long lobId = generateLobId(); long length = streamStore.length(streamStoreId); int tableId = LobStorageFrontend.TABLE_TEMP; Object[] value = new Object[] {streamStoreId, tableId, length, 0}; lobMap.put(lobId, value); Object[] key = new Object[] {streamStoreId, lobId}; refMap.put(key, Boolean.TRUE); ValueLobDb lob = ValueLobDb.create(type, database, tableId, lobId, null, length); if (TRACE) { trace("create " + tableId + "/" + lobId); } return lob; }
/** * Rollback to an old savepoint. * * @param t the transaction * @param maxLogId the last log id * @param toLogId the log id to roll back to */ void rollbackTo(Transaction t, long maxLogId, long toLogId) { for (long logId = maxLogId - 1; logId >= toLogId; logId--) { Object[] op = undoLog.get(new long[] {t.getId(), logId}); int mapId = ((Integer) op[1]).intValue(); Map<String, String> meta = store.getMetaMap(); String m = meta.get("map." + mapId); String mapName = DataUtils.parseMap(m).get("name"); MVMap<Object, Object[]> map = store.openMap(mapName); Object key = op[2]; Object[] oldValue = (Object[]) op[3]; if (oldValue == null) { // this transaction added the value map.remove(key); } else { // this transaction updated the value map.put(key, oldValue); } undoLog.remove(op); } }
/** * Get the size of the map as seen by this transaction. * * @return the size */ public long getSize() { // TODO this method is very slow long size = 0; Cursor<K> cursor = map.keyIterator(null); while (cursor.hasNext()) { K key = cursor.next(); if (get(key) != null) { size++; } } return size; }
private void testConcurrentRead() throws InterruptedException { final MVStore s = openStore(null); final MVMap<Integer, Integer> m = s.openMap("data"); final int size = 3; int x = (int) s.getCurrentVersion(); for (int i = 0; i < size; i++) { m.put(i, x); } s.commit(); Task task = new Task() { @Override public void call() throws Exception { while (!stop) { long v = s.getCurrentVersion() - 1; Map<Integer, Integer> old = m.openVersion(v); for (int i = 0; i < size; i++) { Integer x = old.get(i); if (x == null || (int) v != x) { Map<Integer, Integer> old2 = m.openVersion(v); throw new AssertionError(x + "<>" + v + " at " + i + " " + old2); } } } } }; task.execute(); Thread.sleep(1); for (int j = 0; j < 100; j++) { x = (int) s.getCurrentVersion(); for (int i = 0; i < size; i++) { m.put(i, x); } s.commit(); Thread.sleep(1); } task.get(); s.close(); }
@Override public void init() { if (init) { return; } init = true; Store s = database.getMvStore(); MVStore mvStore; if (s == null) { // in-memory database mvStore = MVStore.open(null); } else { mvStore = s.getStore(); } lobMap = mvStore.openMap("lobMap"); refMap = mvStore.openMap("lobRef"); dataMap = mvStore.openMap("lobData"); streamStore = new StreamStore(dataMap); // garbage collection of the last blocks if (database.isReadOnly()) { return; } if (dataMap.isEmpty()) { return; } // search the last referenced block // (a lob may not have any referenced blocks if data is kept inline, // so we need to loop) long lastUsedKey = -1; Long lobId = lobMap.lastKey(); while (lobId != null) { Object[] v = lobMap.get(lobId); byte[] id = (byte[]) v[0]; lastUsedKey = streamStore.getMaxBlockKey(id); if (lastUsedKey >= 0) { break; } lobId = lobMap.lowerKey(lobId); } // delete all blocks that are newer while (true) { Long last = dataMap.lastKey(); if (last == null || last <= lastUsedKey) { break; } dataMap.remove(last); } // don't re-use block ids, except at the very end Long last = dataMap.lastKey(); if (last != null) { streamStore.setNextKey(last + 1); } }
@Override public ValueLobDb copyLob(ValueLobDb old, int tableId, long length) { init(); int type = old.getType(); long oldLobId = old.getLobId(); long oldLength = old.getPrecision(); if (oldLength != length) { throw DbException.throwInternalError("Length is different"); } Object[] value = lobMap.get(oldLobId); value = value.clone(); byte[] streamStoreId = (byte[]) value[0]; long lobId = generateLobId(); value[1] = tableId; lobMap.put(lobId, value); Object[] key = new Object[] {streamStoreId, lobId}; refMap.put(key, Boolean.TRUE); ValueLobDb lob = ValueLobDb.create(type, database, tableId, lobId, null, length); if (TRACE) { trace("copy " + old.getTableId() + "/" + old.getLobId() + " > " + tableId + "/" + lobId); } return lob; }
/** * Commit a transaction. * * @param t the transaction * @param maxLogId the last log id */ void commit(Transaction t, long maxLogId) { for (long logId = 0; logId < maxLogId; logId++) { long[] undoKey = new long[] {t.getId(), logId}; Object[] op = undoLog.get(undoKey); int opType = (Integer) op[0]; if (opType == Transaction.OP_REMOVE) { int mapId = (Integer) op[1]; Map<String, String> meta = store.getMetaMap(); String m = meta.get("map." + mapId); String mapName = DataUtils.parseMap(m).get("name"); MVMap<Object, Object[]> map = store.openMap(mapName); Object key = op[2]; Object[] value = map.get(key); // possibly the entry was added later on // so we have to check if (value[2] == null) { // remove the value map.remove(key); } } undoLog.remove(undoKey); } endTransaction(t); }
/** * Open an old version for the given map. * * @param version the version * @return the map */ public MVMap<K, V> openVersion(long version) { if (readOnly) { throw DataUtils.newUnsupportedOperationException( "This map is read-only - need to call the method on the writable map"); } DataUtils.checkArgument( version >= createVersion, "Unknown version {0}; this map was created in version is {1}", version, createVersion); Page newest = null; // need to copy because it can change Page r = root; if (version >= r.getVersion() && (version == store.getCurrentVersion() || r.getVersion() >= 0 || version <= createVersion || store.getFile() == null)) { newest = r; } else { // find the newest page that has a getVersion() <= version int i = searchRoot(version); if (i < 0) { // not found if (i == -1) { // smaller than all in-memory versions return store.openMapVersion(version, id, this); } i = -i - 2; } newest = oldRoots.get(i); } MVMap<K, V> m = openReadOnly(); m.root = newest; return m; }
@Override public InputStream getInputStream(ValueLobDb lob, byte[] hmac, long byteCount) throws IOException { init(); Object[] value = lobMap.get(lob.getLobId()); if (value == null) { if (lob.getTableId() == LobStorageFrontend.TABLE_RESULT || lob.getTableId() == LobStorageFrontend.TABLE_ID_SESSION_VARIABLE) { throw DbException.get( ErrorCode.LOB_CLOSED_ON_TIMEOUT_1, "" + lob.getLobId() + "/" + lob.getTableId()); } throw DbException.throwInternalError( "Lob not found: " + lob.getLobId() + "/" + lob.getTableId()); } byte[] streamStoreId = (byte[]) value[0]; return streamStore.get(streamStoreId); }
@Override public void removeAllForTable(int tableId) { init(); if (database.getMvStore().getStore().isClosed()) { return; } // this might not be very efficient - // to speed it up, we would need yet another map ArrayList<Long> list = New.arrayList(); for (Entry<Long, Object[]> e : lobMap.entrySet()) { Object[] value = e.getValue(); int t = (Integer) value[1]; if (t == tableId) { list.add(e.getKey()); } } for (long lobId : list) { removeLob(tableId, lobId); } if (tableId == LobStorageFrontend.TABLE_ID_SESSION_VARIABLE) { removeAllForTable(LobStorageFrontend.TABLE_TEMP); removeAllForTable(LobStorageFrontend.TABLE_RESULT); } }
/** * Get the largest key that is smaller than the given key, or null if no such key exists. * * @param key the key (may not be null) * @return the result */ public K lowerKey(K key) { // TODO Auto-generated method stub return map.lowerKey(key); }
/** * Get the smallest key that is larger than the given key, or null if no such key exists. * * @param key the key (may not be null) * @return the result */ public K higherKey(K key) { // TODO transactional higherKey return map.higherKey(key); }
/** * Get the smallest key that is larger or equal to this key. * * @param key the key (may not be null) * @return the result */ public K ceilingKey(K key) { // TODO transactional ceilingKey return map.ceilingKey(key); }
/** * Get the last key. * * @return the last key, or null if empty */ public K lastKey() { // TODO transactional lastKey return map.lastKey(); }
/** * Get the first key. * * @return the first key, or null if empty */ public K firstKey() { // TODO transactional firstKey return map.firstKey(); }