private void init() {
   String s = settings.get(LAST_TRANSACTION_ID);
   if (s != null) {
     lastTransactionId = Long.parseLong(s);
     lastTransactionIdStored = lastTransactionId;
   }
   Long lastKey = openTransactions.lastKey();
   if (lastKey != null && lastKey.longValue() > lastTransactionId) {
     throw DataUtils.newIllegalStateException("Last transaction not stored");
   }
   Cursor<Long> cursor = openTransactions.keyIterator(null);
   while (cursor.hasNext()) {
     long id = cursor.next();
     Object[] data = openTransactions.get(id);
     int status = (Integer) data[0];
     String name = (String) data[1];
     long[] next = {id + 1, -1};
     long[] last = undoLog.floorKey(next);
     if (last == null) {
       // no entry
     } else if (last[0] == id) {
       Transaction t = new Transaction(this, id, status, name, last[1]);
       t.setStored(true);
       openTransactionMap.put(id, t);
     }
   }
 }
 private void testConcurrentIterate() {
   MVStore s = new MVStore.Builder().pageSplitSize(3).open();
   s.setVersionsToKeep(100);
   final MVMap<Integer, Integer> map = s.openMap("test");
   final int len = 10;
   final Random r = new Random();
   Task t =
       new Task() {
         @Override
         public void call() throws Exception {
           while (!stop) {
             int x = r.nextInt(len);
             if (r.nextBoolean()) {
               map.remove(x);
             } else {
               map.put(x, r.nextInt(100));
             }
           }
         }
       };
   t.execute();
   for (int k = 0; k < 10000; k++) {
     Iterator<Integer> it = map.keyIterator(r.nextInt(len));
     long old = s.getCurrentVersion();
     s.commit();
     while (map.getVersion() == old) {
       Thread.yield();
     }
     while (it.hasNext()) {
       it.next();
     }
   }
   t.get();
   s.close();
 }
 private void testConcurrentStoreAndRemoveMap() throws InterruptedException {
   String fileName = "memFS:testConcurrentStoreAndRemoveMap.h3";
   final MVStore s = openStore(fileName);
   int count = 200;
   for (int i = 0; i < count; i++) {
     MVMap<Integer, Integer> m = s.openMap("d" + i);
     m.put(1, 1);
   }
   final AtomicInteger counter = new AtomicInteger();
   Task task =
       new Task() {
         @Override
         public void call() throws Exception {
           while (!stop) {
             counter.incrementAndGet();
             s.commit();
           }
         }
       };
   task.execute();
   Thread.sleep(1);
   for (int i = 0; i < count || counter.get() < count; i++) {
     MVMap<Integer, Integer> m = s.openMap("d" + i);
     m.put(1, 10);
     s.removeMap(m);
     if (task.isFinished()) {
       break;
     }
   }
   task.get();
   s.close();
   FileUtils.deleteRecursive("memFS:", false);
 }
 void persistMovieSet(MovieSet movieSet) throws Exception {
   String newValue = movieSetObjectWriter.writeValueAsString(movieSet);
   String oldValue = movieMap.get(movieSet.getDbId());
   if (!StringUtils.equals(newValue, oldValue)) {
     movieSetMap.put(movieSet.getDbId(), newValue);
   }
 }
 /**
  * Prepare a transaction.
  *
  * @param transactionId the transaction id
  */
 void prepare(Transaction t) {
   storeTransaction(t);
   Object[] old = openTransactions.get(t.getId());
   Object[] v = {Transaction.STATUS_PREPARED, old[1]};
   openTransactions.put(t.getId(), v);
   store.commit();
 }
 private void removeLob(int tableId, long lobId) {
   if (TRACE) {
     trace("remove " + tableId + "/" + lobId);
   }
   Object[] value = lobMap.remove(lobId);
   if (value == null) {
     // already removed
     return;
   }
   byte[] streamStoreId = (byte[]) value[0];
   Object[] key = new Object[] {streamStoreId, lobId};
   refMap.remove(key);
   // check if there are more entries for this streamStoreId
   key = new Object[] {streamStoreId, 0L};
   value = refMap.ceilingKey(key);
   boolean hasMoreEntries = false;
   if (value != null) {
     byte[] s2 = (byte[]) value[0];
     if (Arrays.equals(streamStoreId, s2)) {
       hasMoreEntries = true;
     }
   }
   if (!hasMoreEntries) {
     streamStore.remove(streamStoreId);
   }
 }
  void persistMovie(Movie movie) throws Exception {
    String newValue = movieObjectWriter.writeValueAsString(movie);
    String oldValue = movieMap.get(movie.getDbId());

    if (!StringUtils.equals(newValue, oldValue)) {
      // write movie to DB
      movieMap.put(movie.getDbId(), newValue);
    }
  }
Exemple #8
0
 /**
  * Open a copy of the map in read-only mode.
  *
  * @return the opened map
  */
 protected MVMap<K, V> openReadOnly() {
   MVMap<K, V> m = new MVMap<K, V>(keyType, valueType);
   m.readOnly = true;
   HashMap<String, String> config = New.hashMap();
   config.put("id", String.valueOf(id));
   config.put("createVersion", String.valueOf(createVersion));
   m.init(store, config);
   m.root = root;
   return m;
 }
 @Override
 public void setTable(ValueLobDb lob, int tableId) {
   init();
   long lobId = lob.getLobId();
   Object[] value = lobMap.remove(lobId);
   if (TRACE) {
     trace("move " + lob.getTableId() + "/" + lob.getLobId() + " > " + tableId + "/" + lobId);
   }
   value[1] = tableId;
   lobMap.put(lobId, value);
 }
 /** Test the concurrent map implementation. */
 private void testConcurrentMap() throws InterruptedException {
   final MVStore s = openStore(null);
   final MVMap<Integer, Integer> m =
       s.openMap("data", new MVMapConcurrent.Builder<Integer, Integer>());
   final int size = 20;
   final Random rand = new Random(1);
   Task task =
       new Task() {
         @Override
         public void call() throws Exception {
           try {
             while (!stop) {
               if (rand.nextBoolean()) {
                 m.put(rand.nextInt(size), 1);
               } else {
                 m.remove(rand.nextInt(size));
               }
               m.get(rand.nextInt(size));
               m.firstKey();
               m.lastKey();
               m.ceilingKey(5);
               m.floorKey(5);
               m.higherKey(5);
               m.lowerKey(5);
               for (Iterator<Integer> it = m.keyIterator(null); it.hasNext(); ) {
                 it.next();
               }
             }
           } catch (Exception e) {
             e.printStackTrace();
           }
         }
       };
   task.execute();
   Thread.sleep(1);
   for (int j = 0; j < 100; j++) {
     for (int i = 0; i < 100; i++) {
       if (rand.nextBoolean()) {
         m.put(rand.nextInt(size), 2);
       } else {
         m.remove(rand.nextInt(size));
       }
       m.get(rand.nextInt(size));
     }
     s.commit();
     Thread.sleep(1);
   }
   task.get();
   s.close();
 }
 private void testConcurrentOnlineBackup() throws Exception {
   String fileName = getBaseDir() + "/onlineBackup.h3";
   String fileNameRestore = getBaseDir() + "/onlineRestore.h3";
   final MVStore s = openStore(fileName);
   final MVMap<Integer, byte[]> map = s.openMap("test");
   final Random r = new Random();
   Task t =
       new Task() {
         @Override
         public void call() throws Exception {
           while (!stop) {
             for (int i = 0; i < 10; i++) {
               map.put(i, new byte[100 * r.nextInt(100)]);
             }
             s.commit();
             map.clear();
             s.commit();
             long len = s.getFileStore().size();
             if (len > 1024 * 1024) {
               // slow down writing a lot
               Thread.sleep(200);
             } else if (len > 20 * 1024) {
               // slow down writing
               Thread.sleep(20);
             }
           }
         }
       };
   t.execute();
   for (int i = 0; i < 10; i++) {
     // System.out.println("test " + i);
     s.setReuseSpace(false);
     byte[] buff = readFileSlowly(s.getFileStore().getFile(), s.getFileStore().size());
     s.setReuseSpace(true);
     FileOutputStream out = new FileOutputStream(fileNameRestore);
     out.write(buff);
     MVStore s2 = openStore(fileNameRestore);
     MVMap<Integer, byte[]> test = s2.openMap("test");
     for (Integer k : test.keySet()) {
       test.get(k);
     }
     s2.close();
     // let it compact
     Thread.sleep(10);
   }
   t.get();
   s.close();
 }
 private Object[] getArray(K key, long maxLog) {
   Object[] data = map.get(key);
   while (true) {
     long tx;
     if (data == null) {
       // doesn't exist or deleted by a committed transaction
       return null;
     }
     tx = (Long) data[0];
     long logId = (Long) data[1];
     if (tx == transaction.transactionId) {
       // added by this transaction
       if (logId < maxLog) {
         return data;
       }
     }
     // added or updated by another transaction
     boolean open = transaction.store.openTransactions.containsKey(tx);
     if (!open) {
       // it is committed
       return data;
     }
     // get the value before the uncommitted transaction
     long[] x = new long[] {tx, logId};
     data = transaction.store.undoLog.get(x);
     data = (Object[]) data[3];
   }
 }
 private void storeTransaction(Transaction t) {
   if (store.getUnsavedPageCount() > MAX_UNSAVED_PAGES) {
     store.commit();
   }
   if (t.isStored()) {
     return;
   }
   t.setStored(true);
   long transactionId = t.getId();
   Object[] v = {t.getStatus(), null};
   openTransactions.put(transactionId, v);
   openTransactionMap.put(transactionId, t);
   if (lastTransactionId > lastTransactionIdStored) {
     lastTransactionIdStored += 32;
     settings.put(LAST_TRANSACTION_ID, "" + lastTransactionIdStored);
   }
 }
 TransactionMap(Transaction transaction, String name, DataType keyType, DataType valueType) {
   this.transaction = transaction;
   ArrayType arrayType =
       new ArrayType(new DataType[] {new ObjectDataType(), new ObjectDataType(), valueType});
   MVMap.Builder<K, Object[]> builder =
       new MVMap.Builder<K, Object[]>().keyType(keyType).valueType(arrayType);
   map = transaction.store.store.openMap(name, builder);
   mapId = map.getId();
 }
 private long generateLobId() {
   synchronized (nextLobIdSync) {
     if (nextLobId == 0) {
       Long id = lobMap.lastKey();
       nextLobId = id == null ? 1 : id + 1;
     }
     return nextLobId++;
   }
 }
 private ValueLobDb createLob(InputStream in, int type) throws IOException {
   byte[] streamStoreId;
   try {
     streamStoreId = streamStore.put(in);
   } catch (Exception e) {
     throw DbException.convertToIOException(e);
   }
   long lobId = generateLobId();
   long length = streamStore.length(streamStoreId);
   int tableId = LobStorageFrontend.TABLE_TEMP;
   Object[] value = new Object[] {streamStoreId, tableId, length, 0};
   lobMap.put(lobId, value);
   Object[] key = new Object[] {streamStoreId, lobId};
   refMap.put(key, Boolean.TRUE);
   ValueLobDb lob = ValueLobDb.create(type, database, tableId, lobId, null, length);
   if (TRACE) {
     trace("create " + tableId + "/" + lobId);
   }
   return lob;
 }
 /**
  * Rollback to an old savepoint.
  *
  * @param t the transaction
  * @param maxLogId the last log id
  * @param toLogId the log id to roll back to
  */
 void rollbackTo(Transaction t, long maxLogId, long toLogId) {
   for (long logId = maxLogId - 1; logId >= toLogId; logId--) {
     Object[] op = undoLog.get(new long[] {t.getId(), logId});
     int mapId = ((Integer) op[1]).intValue();
     Map<String, String> meta = store.getMetaMap();
     String m = meta.get("map." + mapId);
     String mapName = DataUtils.parseMap(m).get("name");
     MVMap<Object, Object[]> map = store.openMap(mapName);
     Object key = op[2];
     Object[] oldValue = (Object[]) op[3];
     if (oldValue == null) {
       // this transaction added the value
       map.remove(key);
     } else {
       // this transaction updated the value
       map.put(key, oldValue);
     }
     undoLog.remove(op);
   }
 }
 /**
  * Get the size of the map as seen by this transaction.
  *
  * @return the size
  */
 public long getSize() {
   // TODO this method is very slow
   long size = 0;
   Cursor<K> cursor = map.keyIterator(null);
   while (cursor.hasNext()) {
     K key = cursor.next();
     if (get(key) != null) {
       size++;
     }
   }
   return size;
 }
 private void testConcurrentRead() throws InterruptedException {
   final MVStore s = openStore(null);
   final MVMap<Integer, Integer> m = s.openMap("data");
   final int size = 3;
   int x = (int) s.getCurrentVersion();
   for (int i = 0; i < size; i++) {
     m.put(i, x);
   }
   s.commit();
   Task task =
       new Task() {
         @Override
         public void call() throws Exception {
           while (!stop) {
             long v = s.getCurrentVersion() - 1;
             Map<Integer, Integer> old = m.openVersion(v);
             for (int i = 0; i < size; i++) {
               Integer x = old.get(i);
               if (x == null || (int) v != x) {
                 Map<Integer, Integer> old2 = m.openVersion(v);
                 throw new AssertionError(x + "<>" + v + " at " + i + " " + old2);
               }
             }
           }
         }
       };
   task.execute();
   Thread.sleep(1);
   for (int j = 0; j < 100; j++) {
     x = (int) s.getCurrentVersion();
     for (int i = 0; i < size; i++) {
       m.put(i, x);
     }
     s.commit();
     Thread.sleep(1);
   }
   task.get();
   s.close();
 }
 @Override
 public void init() {
   if (init) {
     return;
   }
   init = true;
   Store s = database.getMvStore();
   MVStore mvStore;
   if (s == null) {
     // in-memory database
     mvStore = MVStore.open(null);
   } else {
     mvStore = s.getStore();
   }
   lobMap = mvStore.openMap("lobMap");
   refMap = mvStore.openMap("lobRef");
   dataMap = mvStore.openMap("lobData");
   streamStore = new StreamStore(dataMap);
   // garbage collection of the last blocks
   if (database.isReadOnly()) {
     return;
   }
   if (dataMap.isEmpty()) {
     return;
   }
   // search the last referenced block
   // (a lob may not have any referenced blocks if data is kept inline,
   // so we need to loop)
   long lastUsedKey = -1;
   Long lobId = lobMap.lastKey();
   while (lobId != null) {
     Object[] v = lobMap.get(lobId);
     byte[] id = (byte[]) v[0];
     lastUsedKey = streamStore.getMaxBlockKey(id);
     if (lastUsedKey >= 0) {
       break;
     }
     lobId = lobMap.lowerKey(lobId);
   }
   // delete all blocks that are newer
   while (true) {
     Long last = dataMap.lastKey();
     if (last == null || last <= lastUsedKey) {
       break;
     }
     dataMap.remove(last);
   }
   // don't re-use block ids, except at the very end
   Long last = dataMap.lastKey();
   if (last != null) {
     streamStore.setNextKey(last + 1);
   }
 }
 @Override
 public ValueLobDb copyLob(ValueLobDb old, int tableId, long length) {
   init();
   int type = old.getType();
   long oldLobId = old.getLobId();
   long oldLength = old.getPrecision();
   if (oldLength != length) {
     throw DbException.throwInternalError("Length is different");
   }
   Object[] value = lobMap.get(oldLobId);
   value = value.clone();
   byte[] streamStoreId = (byte[]) value[0];
   long lobId = generateLobId();
   value[1] = tableId;
   lobMap.put(lobId, value);
   Object[] key = new Object[] {streamStoreId, lobId};
   refMap.put(key, Boolean.TRUE);
   ValueLobDb lob = ValueLobDb.create(type, database, tableId, lobId, null, length);
   if (TRACE) {
     trace("copy " + old.getTableId() + "/" + old.getLobId() + " > " + tableId + "/" + lobId);
   }
   return lob;
 }
 /**
  * Commit a transaction.
  *
  * @param t the transaction
  * @param maxLogId the last log id
  */
 void commit(Transaction t, long maxLogId) {
   for (long logId = 0; logId < maxLogId; logId++) {
     long[] undoKey = new long[] {t.getId(), logId};
     Object[] op = undoLog.get(undoKey);
     int opType = (Integer) op[0];
     if (opType == Transaction.OP_REMOVE) {
       int mapId = (Integer) op[1];
       Map<String, String> meta = store.getMetaMap();
       String m = meta.get("map." + mapId);
       String mapName = DataUtils.parseMap(m).get("name");
       MVMap<Object, Object[]> map = store.openMap(mapName);
       Object key = op[2];
       Object[] value = map.get(key);
       // possibly the entry was added later on
       // so we have to check
       if (value[2] == null) {
         // remove the value
         map.remove(key);
       }
     }
     undoLog.remove(undoKey);
   }
   endTransaction(t);
 }
Exemple #23
0
 /**
  * Open an old version for the given map.
  *
  * @param version the version
  * @return the map
  */
 public MVMap<K, V> openVersion(long version) {
   if (readOnly) {
     throw DataUtils.newUnsupportedOperationException(
         "This map is read-only - need to call the method on the writable map");
   }
   DataUtils.checkArgument(
       version >= createVersion,
       "Unknown version {0}; this map was created in version is {1}",
       version,
       createVersion);
   Page newest = null;
   // need to copy because it can change
   Page r = root;
   if (version >= r.getVersion()
       && (version == store.getCurrentVersion()
           || r.getVersion() >= 0
           || version <= createVersion
           || store.getFile() == null)) {
     newest = r;
   } else {
     // find the newest page that has a getVersion() <= version
     int i = searchRoot(version);
     if (i < 0) {
       // not found
       if (i == -1) {
         // smaller than all in-memory versions
         return store.openMapVersion(version, id, this);
       }
       i = -i - 2;
     }
     newest = oldRoots.get(i);
   }
   MVMap<K, V> m = openReadOnly();
   m.root = newest;
   return m;
 }
 @Override
 public InputStream getInputStream(ValueLobDb lob, byte[] hmac, long byteCount)
     throws IOException {
   init();
   Object[] value = lobMap.get(lob.getLobId());
   if (value == null) {
     if (lob.getTableId() == LobStorageFrontend.TABLE_RESULT
         || lob.getTableId() == LobStorageFrontend.TABLE_ID_SESSION_VARIABLE) {
       throw DbException.get(
           ErrorCode.LOB_CLOSED_ON_TIMEOUT_1, "" + lob.getLobId() + "/" + lob.getTableId());
     }
     throw DbException.throwInternalError(
         "Lob not found: " + lob.getLobId() + "/" + lob.getTableId());
   }
   byte[] streamStoreId = (byte[]) value[0];
   return streamStore.get(streamStoreId);
 }
 @Override
 public void removeAllForTable(int tableId) {
   init();
   if (database.getMvStore().getStore().isClosed()) {
     return;
   }
   // this might not be very efficient -
   // to speed it up, we would need yet another map
   ArrayList<Long> list = New.arrayList();
   for (Entry<Long, Object[]> e : lobMap.entrySet()) {
     Object[] value = e.getValue();
     int t = (Integer) value[1];
     if (t == tableId) {
       list.add(e.getKey());
     }
   }
   for (long lobId : list) {
     removeLob(tableId, lobId);
   }
   if (tableId == LobStorageFrontend.TABLE_ID_SESSION_VARIABLE) {
     removeAllForTable(LobStorageFrontend.TABLE_TEMP);
     removeAllForTable(LobStorageFrontend.TABLE_RESULT);
   }
 }
 /**
  * Get the largest key that is smaller than the given key, or null if no such key exists.
  *
  * @param key the key (may not be null)
  * @return the result
  */
 public K lowerKey(K key) {
   // TODO Auto-generated method stub
   return map.lowerKey(key);
 }
 /**
  * Get the smallest key that is larger than the given key, or null if no such key exists.
  *
  * @param key the key (may not be null)
  * @return the result
  */
 public K higherKey(K key) {
   // TODO transactional higherKey
   return map.higherKey(key);
 }
 /**
  * Get the smallest key that is larger or equal to this key.
  *
  * @param key the key (may not be null)
  * @return the result
  */
 public K ceilingKey(K key) {
   // TODO transactional ceilingKey
   return map.ceilingKey(key);
 }
 /**
  * Get the last key.
  *
  * @return the last key, or null if empty
  */
 public K lastKey() {
   // TODO transactional lastKey
   return map.lastKey();
 }
 /**
  * Get the first key.
  *
  * @return the first key, or null if empty
  */
 public K firstKey() {
   // TODO transactional firstKey
   return map.firstKey();
 }