Beispiel #1
0
 @Test
 public void testPutGetDelete() throws IOException {
   assertEquals("Empty store", 0, store.count());
   assertNull("Should find no previous record", store.put(record));
   assertEquals("Empty store", 1, store.count());
   assertEquals("Should find the record by key.", record, this.store.get(key));
   assertEquals("Should delete the record and return previous", record, this.store.delete(key));
   assertEquals("Empty store", 0, store.count());
   assertNull("Should not find the record by key.", this.store.get(key));
 }
Beispiel #2
0
 @Test
 public void testOverwrite() throws IOException {
   assertNull("Should find no previous record", store.put(record));
   assertEquals("Should find the record by key.", record, this.store.get(key));
   assertEquals("Should find the record by key.", record, this.store.put(record2));
   assertEquals("Should find the record by key.", record2, this.store.get(key));
 }
Beispiel #3
0
 private List<Record> records(HashStore store) {
   ClosableIterator<Record> iter = store.iterator();
   List<Record> l = new ArrayList<Record>();
   while (iter.hasNext()) l.add(iter.next());
   iter.close();
   return l;
 }
Beispiel #4
0
  @Test
  public void testRecoveryCorruptMessage() throws IOException {
    List<Record> records = StoreTestUtils.randomRecords(10);
    putAll(records);

    // append a message with a crc that won't possibly validate
    Record invalid = new Record(ByteBuffer.wrap(StoreTestUtils.randomBytes(10)));
    store.log().append(invalid);
    this.store.close();
    this.store = new HashStore(config);
    // assertEquals("Same records should be present after close and re-open", records,
    // records(store));
  }
Beispiel #5
0
  /**
   * Returns a hex enccoded SHA1 hash of the whole file. This can be used to locate the files bytes
   * again
   *
   * @param in
   * @param hashStore
   * @param blobStore
   * @return
   * @throws IOException
   */
  public String parse(InputStream in, HashStore hashStore, BlobStore blobStore) throws IOException {
    if (log.isInfoEnabled()) {
      log.info("parse. inputstream: " + in);
    }
    Rsum rsum = new Rsum(128);
    int numBlobs = 0;
    byte[] arr = new byte[1024];
    ByteArrayOutputStream bout = new ByteArrayOutputStream();

    List<String> blobHashes = new ArrayList<>();
    MessageDigest blobCrc = getCrypt();
    MessageDigest fanoutCrc = getCrypt();
    MessageDigest fileCrc = getCrypt();

    long fanoutLength = 0;
    long fileLength = 0;

    int s = in.read(arr, 0, 1024);
    if (log.isTraceEnabled()) {
      log.trace("initial block size: " + s);
    }

    List<String> fanoutHashes = new ArrayList<>();
    while (s >= 0) {
      numBytes += s;
      // log.trace("numBytes: {}", numBytes);
      if (cancelled) {
        throw new IOException("operation cancelled");
      }
      for (int i = 0; i < s; i++) {
        byte b = arr[i];
        rsum.roll(b);
        blobCrc.update(b);
        fanoutCrc.update(b);
        fileCrc.update(b);
        fanoutLength++;
        fileLength++;
        bout.write(b);
        int x = rsum.getValue();

        // System.out.println("x=" + x);
        // System.out.println("check mask: " + (x & MASK) + " == " + MASK);
        boolean limited;
        if (MAX_BLOB_SIZE != null) {
          limited = bout.size() > MAX_BLOB_SIZE;
          if (limited) {
            log.warn("HIT BLOB LIMIT: " + bout.size());
          }
        } else {
          limited = false;
        }
        if (((x & MASK) == MASK) || limited) {
          String blobCrcHex = toHex(blobCrc);
          byte[] blobBytes = bout.toByteArray();
          if (log.isInfoEnabled()) {
            log.info(
                "Store blob: "
                    + blobCrcHex
                    + " length="
                    + blobBytes.length
                    + " hash: "
                    + x
                    + " mask: "
                    + MASK);
          }
          blobStore.setBlob(blobCrcHex, blobBytes);
          bout.reset();
          blobHashes.add(blobCrcHex);
          blobCrc.reset();
          if ((x & FANOUT_MASK) == FANOUT_MASK) {
            String fanoutCrcVal = toHex(fanoutCrc);
            fanoutHashes.add(fanoutCrcVal);
            // log.info("set chunk fanout: {} length={}", fanoutCrcVal, fanoutLength);
            hashStore.setChunkFanout(fanoutCrcVal, blobHashes, fanoutLength);
            fanoutLength = 0;
            fanoutCrc.reset();
            blobHashes = new ArrayList<>();
          }
          numBlobs++;
          rsum.reset();
        }
      }

      s = in.read(arr, 0, 1024);
    }
    // Need to store terminal data, ie data which has been accumulated since the last boundary
    String blobCrcHex = toHex(blobCrc);
    // System.out.println("Store terminal blob: " + blobCrcHex);
    blobStore.setBlob(blobCrcHex, bout.toByteArray());
    numBlobs++;
    blobHashes.add(blobCrcHex);
    String fanoutCrcVal = toHex(fanoutCrc);
    // log.info("set terminal chunk fanout: {} length={}" ,fanoutCrcVal, fanoutLength);

    hashStore.setChunkFanout(fanoutCrcVal, blobHashes, fanoutLength);
    fanoutHashes.add(fanoutCrcVal);

    // Now store a fanout for the whole file. The contained hashes locate other fanouts
    String fileCrcVal = toHex(fileCrc);
    if (log.isInfoEnabled()) {
      log.info(
          "set file fanout: "
              + fanoutCrcVal
              + "  length="
              + fileLength
              + " avg blob size="
              + fileLength / numBlobs);
    }
    hashStore.setFileFanout(fileCrcVal, fanoutHashes, fileLength);
    return fileCrcVal;
  }