private void testFormat() throws IOException {
    Map<Long, byte[]> map = New.hashMap();
    StreamStore store = new StreamStore(map);
    store.setMinBlockSize(10);
    store.setMaxBlockSize(20);
    store.setNextKey(123);

    byte[] id;

    id = store.put(new ByteArrayInputStream(new byte[200]));
    assertEquals(200, store.length(id));
    assertEquals("02c8018801", StringUtils.convertBytesToHex(id));

    id = store.put(new ByteArrayInputStream(new byte[0]));
    assertEquals("", StringUtils.convertBytesToHex(id));

    id = store.put(new ByteArrayInputStream(new byte[1]));
    assertEquals("000100", StringUtils.convertBytesToHex(id));

    id = store.put(new ByteArrayInputStream(new byte[3]));
    assertEquals("0003000000", StringUtils.convertBytesToHex(id));

    id = store.put(new ByteArrayInputStream(new byte[10]));
    assertEquals("010a8901", StringUtils.convertBytesToHex(id));

    byte[] combined = StringUtils.convertHexToBytes("0001aa0002bbcc");
    assertEquals(3, store.length(combined));
    InputStream in = store.get(combined);
    assertEquals(1, in.skip(1));
    assertEquals(0xbb, in.read());
    assertEquals(1, in.skip(1));
  }
Пример #2
0
 private ByteArrayInputStream nextBuffer() {
   while (idBuffer.hasRemaining()) {
     switch (idBuffer.get()) {
       case 0:
         {
           int len = DataUtils.readVarInt(idBuffer);
           if (skip >= len) {
             skip -= len;
             idBuffer.position(idBuffer.position() + len);
             continue;
           }
           int p = (int) (idBuffer.position() + skip);
           int l = (int) (len - skip);
           idBuffer.position(p + l);
           return new ByteArrayInputStream(idBuffer.array(), p, l);
         }
       case 1:
         {
           int len = DataUtils.readVarInt(idBuffer);
           long key = DataUtils.readVarLong(idBuffer);
           if (skip >= len) {
             skip -= len;
             continue;
           }
           byte[] data = store.getBlock(key);
           int s = (int) skip;
           skip = 0;
           return new ByteArrayInputStream(data, s, data.length - s);
         }
       case 2:
         {
           long len = DataUtils.readVarLong(idBuffer);
           long key = DataUtils.readVarLong(idBuffer);
           if (skip >= len) {
             skip -= len;
             continue;
           }
           byte[] k = store.getBlock(key);
           ByteBuffer newBuffer =
               ByteBuffer.allocate(k.length + idBuffer.limit() - idBuffer.position());
           newBuffer.put(k);
           newBuffer.put(idBuffer);
           newBuffer.flip();
           idBuffer = newBuffer;
           return nextBuffer();
         }
       default:
         throw DataUtils.newIllegalArgumentException(
             "Unsupported id {0}", Arrays.toString(idBuffer.array()));
     }
   }
   return null;
 }
Пример #3
0
 @Override
 public void init() {
   if (init) {
     return;
   }
   init = true;
   Store s = database.getMvStore();
   MVStore mvStore;
   if (s == null) {
     // in-memory database
     mvStore = MVStore.open(null);
   } else {
     mvStore = s.getStore();
   }
   lobMap = mvStore.openMap("lobMap");
   refMap = mvStore.openMap("lobRef");
   dataMap = mvStore.openMap("lobData");
   streamStore = new StreamStore(dataMap);
   // garbage collection of the last blocks
   if (database.isReadOnly()) {
     return;
   }
   if (dataMap.isEmpty()) {
     return;
   }
   // search the last referenced block
   // (a lob may not have any referenced blocks if data is kept inline,
   // so we need to loop)
   long lastUsedKey = -1;
   Long lobId = lobMap.lastKey();
   while (lobId != null) {
     Object[] v = lobMap.get(lobId);
     byte[] id = (byte[]) v[0];
     lastUsedKey = streamStore.getMaxBlockKey(id);
     if (lastUsedKey >= 0) {
       break;
     }
     lobId = lobMap.lowerKey(lobId);
   }
   // delete all blocks that are newer
   while (true) {
     Long last = dataMap.lastKey();
     if (last == null || last <= lastUsedKey) {
       break;
     }
     dataMap.remove(last);
   }
   // don't re-use block ids, except at the very end
   Long last = dataMap.lastKey();
   if (last != null) {
     streamStore.setNextKey(last + 1);
   }
 }
Пример #4
0
 private void removeLob(int tableId, long lobId) {
   if (TRACE) {
     trace("remove " + tableId + "/" + lobId);
   }
   Object[] value = lobMap.remove(lobId);
   if (value == null) {
     // already removed
     return;
   }
   byte[] streamStoreId = (byte[]) value[0];
   Object[] key = new Object[] {streamStoreId, lobId};
   refMap.remove(key);
   // check if there are more entries for this streamStoreId
   key = new Object[] {streamStoreId, 0L};
   value = refMap.ceilingKey(key);
   boolean hasMoreEntries = false;
   if (value != null) {
     byte[] s2 = (byte[]) value[0];
     if (Arrays.equals(streamStoreId, s2)) {
       hasMoreEntries = true;
     }
   }
   if (!hasMoreEntries) {
     streamStore.remove(streamStoreId);
   }
 }
 private void testLoop() throws IOException {
   Map<Long, byte[]> map = New.hashMap();
   StreamStore store = new StreamStore(map);
   assertEquals(256 * 1024, store.getMaxBlockSize());
   assertEquals(256, store.getMinBlockSize());
   store.setNextKey(0);
   assertEquals(0, store.getNextKey());
   test(store, 10, 20, 1000);
   for (int i = 0; i < 20; i++) {
     test(store, 0, 128, i);
     test(store, 10, 128, i);
   }
   for (int i = 20; i < 200; i += 10) {
     test(store, 0, 128, i);
     test(store, 10, 128, i);
   }
 }
Пример #6
0
 private ValueLobDb createLob(InputStream in, int type) throws IOException {
   byte[] streamStoreId;
   try {
     streamStoreId = streamStore.put(in);
   } catch (Exception e) {
     throw DbException.convertToIOException(e);
   }
   long lobId = generateLobId();
   long length = streamStore.length(streamStoreId);
   int tableId = LobStorageFrontend.TABLE_TEMP;
   Object[] value = new Object[] {streamStoreId, tableId, length, 0};
   lobMap.put(lobId, value);
   Object[] key = new Object[] {streamStoreId, lobId};
   refMap.put(key, Boolean.TRUE);
   ValueLobDb lob = ValueLobDb.create(type, database, tableId, lobId, null, length);
   if (TRACE) {
     trace("create " + tableId + "/" + lobId);
   }
   return lob;
 }
  private void testWithFullMap() throws IOException {
    final AtomicInteger tests = new AtomicInteger();
    Map<Long, byte[]> map =
        new HashMap<Long, byte[]>() {

          private static final long serialVersionUID = 1L;

          public boolean containsKey(Object k) {
            tests.incrementAndGet();
            if (((Long) k) < Long.MAX_VALUE / 2) {
              // simulate a *very* full map
              return true;
            }
            return super.containsKey(k);
          }
        };
    StreamStore store = new StreamStore(map);
    store.setMinBlockSize(20);
    store.setMaxBlockSize(100);
    store.setNextKey(0);
    store.put(new ByteArrayInputStream(new byte[100]));
    assertEquals(1, map.size());
    assertEquals(64, tests.get());
    assertEquals(Long.MAX_VALUE / 2 + 1, store.getNextKey());
  }
  private void testTreeStructure() throws IOException {

    final AtomicInteger reads = new AtomicInteger();
    Map<Long, byte[]> map =
        new HashMap<Long, byte[]>() {

          private static final long serialVersionUID = 1L;

          public byte[] get(Object k) {
            reads.incrementAndGet();
            return super.get(k);
          }
        };

    StreamStore store = new StreamStore(map);
    store.setMinBlockSize(10);
    store.setMaxBlockSize(100);
    byte[] id = store.put(new ByteArrayInputStream(new byte[10000]));
    InputStream in = store.get(id);
    assertEquals(0, in.read());
    assertEquals(3, reads.get());
  }
 private void testDetectIllegalId() throws IOException {
   Map<Long, byte[]> map = New.hashMap();
   StreamStore store = new StreamStore(map);
   try {
     store.length(new byte[] {3, 0, 0});
     fail();
   } catch (IllegalArgumentException e) {
     // expected
   }
   try {
     store.remove(new byte[] {3, 0, 0});
     fail();
   } catch (IllegalArgumentException e) {
     // expected
   }
   map.put(0L, new byte[] {3, 0, 0});
   InputStream in = store.get(new byte[] {2, 1, 0});
   try {
     in.read();
     fail();
   } catch (IllegalArgumentException e) {
     // expected
   }
 }
Пример #10
0
 @Override
 public InputStream getInputStream(ValueLobDb lob, byte[] hmac, long byteCount)
     throws IOException {
   init();
   Object[] value = lobMap.get(lob.getLobId());
   if (value == null) {
     if (lob.getTableId() == LobStorageFrontend.TABLE_RESULT
         || lob.getTableId() == LobStorageFrontend.TABLE_ID_SESSION_VARIABLE) {
       throw DbException.get(
           ErrorCode.LOB_CLOSED_ON_TIMEOUT_1, "" + lob.getLobId() + "/" + lob.getTableId());
     }
     throw DbException.throwInternalError(
         "Lob not found: " + lob.getLobId() + "/" + lob.getTableId());
   }
   byte[] streamStoreId = (byte[]) value[0];
   return streamStore.get(streamStoreId);
 }
Пример #11
0
 Stream(StreamStore store, byte[] id) {
   this.store = store;
   this.length = store.length(id);
   this.idBuffer = ByteBuffer.wrap(id);
 }
  private void test(StreamStore store, int minBlockSize, int maxBlockSize, int length)
      throws IOException {
    store.setMinBlockSize(minBlockSize);
    assertEquals(minBlockSize, store.getMinBlockSize());
    store.setMaxBlockSize(maxBlockSize);
    assertEquals(maxBlockSize, store.getMaxBlockSize());
    long next = store.getNextKey();
    Random r = new Random(length);
    byte[] data = new byte[length];
    r.nextBytes(data);
    byte[] id = store.put(new ByteArrayInputStream(data));
    if (length > 0 && length >= minBlockSize) {
      assertFalse(store.isInPlace(id));
    } else {
      assertTrue(store.isInPlace(id));
    }
    long next2 = store.getNextKey();
    if (length > 0 && length >= minBlockSize) {
      assertTrue(next2 > next);
    } else {
      assertEquals(next, next2);
    }
    if (length == 0) {
      assertEquals(0, id.length);
    }

    assertEquals(length, store.length(id));

    InputStream in = store.get(id);
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    IOUtils.copy(in, out);
    assertTrue(Arrays.equals(data, out.toByteArray()));

    in = store.get(id);
    in.close();
    assertEquals(-1, in.read());

    in = store.get(id);
    assertEquals(0, in.skip(0));
    if (length > 0) {
      assertEquals(1, in.skip(1));
      if (length > 1) {
        assertEquals(data[1] & 255, in.read());
        if (length > 2) {
          assertEquals(1, in.skip(1));
          if (length > 3) {
            assertEquals(data[3] & 255, in.read());
          }
        } else {
          assertEquals(0, in.skip(1));
        }
      } else {
        assertEquals(-1, in.read());
      }
    } else {
      assertEquals(0, in.skip(1));
    }

    if (length > 12) {
      in = store.get(id);
      assertEquals(12, in.skip(12));
      assertEquals(data[12] & 255, in.read());
      long skipped = 0;
      while (true) {
        long s = in.skip(Integer.MAX_VALUE);
        if (s == 0) {
          break;
        }
        skipped += s;
      }
      assertEquals(length - 13, skipped);
      assertEquals(-1, in.read());
    }

    store.remove(id);
    assertEquals(0, store.getMap().size());
  }
  private void testWithExistingData() throws IOException {

    final AtomicInteger tests = new AtomicInteger();
    Map<Long, byte[]> map =
        new HashMap<Long, byte[]>() {

          private static final long serialVersionUID = 1L;

          public boolean containsKey(Object k) {
            tests.incrementAndGet();
            return super.containsKey(k);
          }
        };
    StreamStore store = new StreamStore(map);
    store.setMinBlockSize(10);
    store.setMaxBlockSize(20);
    store.setNextKey(0);
    for (int i = 0; i < 10; i++) {
      store.put(new ByteArrayInputStream(new byte[20]));
    }
    assertEquals(10, map.size());
    assertEquals(10, tests.get());
    for (int i = 0; i < 10; i++) {
      map.containsKey(i);
    }
    assertEquals(20, tests.get());
    store = new StreamStore(map);
    store.setMinBlockSize(10);
    store.setMaxBlockSize(20);
    store.setNextKey(0);
    assertEquals(0, store.getNextKey());
    for (int i = 0; i < 5; i++) {
      store.put(new ByteArrayInputStream(new byte[20]));
    }
    assertEquals(88, tests.get());
    assertEquals(15, store.getNextKey());
    assertEquals(15, map.size());
    for (int i = 0; i < 15; i++) {
      map.containsKey(i);
    }
  }