コード例 #1
0
 @Before
 public void beforeClass() {
   directory = new File("target/fsbs/");
   FileUtil.delete(directory);
   directory.mkdirs();
   trash = new File(directory, FileSystemBinaryStore.TRASH_DIRECTORY_NAME);
   store = new FileSystemBinaryStore(directory);
   store.setMinimumBinarySizeInBytes(MIN_BINARY_SIZE);
   store.setMimeTypeDetector(DEFAULT_DETECTOR);
   print = false;
 }
コード例 #2
0
  protected Binary storeAndCheck(int contentIndex, Class<? extends Binary> valueClass)
      throws Exception {
    String content = CONTENT[contentIndex];
    String sha1 = CONTENT_HASHES[contentIndex];
    InputStream stream = new ByteArrayInputStream(content.getBytes());

    Stopwatch sw = new Stopwatch();
    sw.start();
    Binary binary = store.storeValue(stream, false);
    sw.stop();
    if (print) System.out.println("Time to store 18MB file: " + sw.getTotalDuration());

    if (valueClass != null) {
      assertThat(binary, is(instanceOf(valueClass)));
    }
    if (content.length() == 0) {
      assertThat(binary, is(instanceOf(EmptyBinaryValue.class)));
    } else if (content.length() < MIN_BINARY_SIZE) {
      assertThat(binary, is(instanceOf(InMemoryBinaryValue.class)));
    } else {
      assertThat(binary, is(instanceOf(StoredBinaryValue.class)));
    }
    assertThat(binary.getHexHash(), is(sha1));
    String binaryContent = IoUtil.read(binary.getStream());
    assertThat(binaryContent, is(content));
    return binary;
  }
コード例 #3
0
  @Test
  public void shouldCleanTrashFilesWhenFilesBecomeUsed() throws Exception {
    Set<Binary> binaries = new HashSet<Binary>();
    int storedCount = 0;
    for (int i = 0; i != CONTENT.length; ++i) {
      Binary binary = storeAndCheck(i);
      assertThat(binary, is(notNullValue()));
      binaries.add(binary);
      if (binary instanceof StoredBinaryValue) storedCount++;
    }

    // Make sure there are files for all stored values ...
    assertThat(countStoredFiles(), is(storedCount));
    assertThat(countTrashFiles(), is(0));

    // Mark one of the files as being unused ...
    String unused = binaries.iterator().next().getHexHash();
    BinaryKey unusedKey = new BinaryKey(unused);
    store.markAsUnused(Collections.singleton(unusedKey));

    // Make sure the file was moved to the trash ...
    assertThat(countStoredFiles(), is(storedCount));
    assertThat(countTrashFiles(), is(1));

    // Now access all the binary files which will not change there used/unused state
    for (Binary binary : binaries) {
      InputStream stream = binary.getStream();
      String content = IoUtil.read(stream);
      assertThat(content.length() != 0, is(true));
    }

    // Make sure there are files for all stored values ...
    assertThat(countStoredFiles(), is(storedCount));
    assertThat(countTrashFiles(), is(1));

    // Now mark the file explicitly as used and check that the file from the trash was removed
    store.markAsUsed(Collections.singleton(unusedKey));
    assertThat(countTrashFiles(), is(0));
  }
コード例 #4
0
  @Override
  public BinaryValue storeValue(InputStream stream) throws BinaryStoreException {
    // store into temporary file system store and get SHA-1
    BinaryValue temp = cache.storeValue(stream);
    try {
      // prepare new binary key based on SHA-1
      BinaryKey key = new BinaryKey(temp.getKey().toString());

      // check for duplicate content
      if (this.contentExists(key, ALIVE)) {
        return new StoredBinaryValue(this, key, temp.getSize());
      }

      // check unused content
      if (this.contentExists(key, UNUSED)) {
        session.execute("UPDATE modeshape.binary SET usage=1 WHERE cid='" + key + "';");
        return new StoredBinaryValue(this, key, temp.getSize());
      }

      // store content
      PreparedStatement query =
          session.prepare(
              "INSERT INTO modeshape.binary (cid, usage_time, payload, usage) VALUES ( ?,?,?,1 );");
      BoundStatement statement = new BoundStatement(query);
      session.execute(statement.bind(key.toString(), new Date(), buffer(stream)));
      return new StoredBinaryValue(this, key, temp.getSize());
    } catch (BinaryStoreException e) {
      throw e;
    } catch (IOException e) {
      throw new BinaryStoreException(e);
    } catch (RuntimeException e) {
      throw new BinaryStoreException(e);
    } finally {
      // remove content from temp store
      cache.markAsUnused(temp.getKey());
    }
  }
コード例 #5
0
  @Test
  public void shouldCreateTrashFilesForUnusedBinaries() throws Exception {
    Set<String> storedSha1s = new HashSet<String>();
    for (int i = 0; i != CONTENT.length; ++i) {
      Binary binary = storeAndCheck(i);
      if (binary instanceof StoredBinaryValue) storedSha1s.add(binary.getHexHash());
    }

    // Make sure there are files for all stored values ...
    assertThat(countStoredFiles(), is(storedSha1s.size()));
    assertThat(countTrashFiles(), is(0));

    // Mark one of the files as being unused ...
    String unused = storedSha1s.iterator().next();
    store.markAsUnused(Collections.singleton(new BinaryKey(unused)));

    // Make sure the trash file was created
    assertThat(countStoredFiles(), is(storedSha1s.size()));
    assertThat(countTrashFiles(), is(1));
    // Check that the name of the trash file is the SHA1
    File trashFile = collectFiles(trash).get(0);
    assertNotNull(trashFile);
    assertEquals(unused, trashFile.getName());

    Thread.sleep(
        1100L); // Sleep more than a second, since modified times may only be accurate to nearest
                // second ...
    store.removeValuesUnusedLongerThan(1, TimeUnit.SECONDS);

    // Make sure the file was removed from the trash ...
    assertThat(countStoredFiles(), is(storedSha1s.size() - 1));
    assertThat(countTrashFiles(), is(0));

    // And that all directories in the trash were removed (since they should be empty) ...
    assertThat(trash.listFiles().length, is(0));
  }
コード例 #6
0
  protected void storeAndCheckResource(
      String resourcePath, String expectedSha1, String desc, long numBytes) throws Exception {
    InputStream content = getClass().getClassLoader().getResourceAsStream(resourcePath);
    assertThat(content, is(notNullValue()));

    Stopwatch sw = new Stopwatch();
    sw.start();
    Binary binary = store.storeValue(content, false);
    sw.stop();
    if (print) System.out.println("Time to store " + desc + ": " + sw.getTotalDuration());

    if (numBytes == 0) {
      assertThat(binary, is(instanceOf(EmptyBinaryValue.class)));
    } else if (numBytes < MIN_BINARY_SIZE) {
      assertThat(binary, is(instanceOf(InMemoryBinaryValue.class)));
    } else {
      assertThat(binary, is(instanceOf(StoredBinaryValue.class)));
    }
    assertThat(binary.getHexHash(), is(expectedSha1));
    assertThat(binary.getSize(), is(numBytes));

    // Now try reading and comparing the two streams ...
    InputStream expected = getClass().getClassLoader().getResourceAsStream(resourcePath);
    InputStream actual = binary.getStream();
    byte[] buffer1 = new byte[1024];
    byte[] buffer2 = new byte[1024];
    int numRead = 0;
    while ((numRead = expected.read(buffer1)) == actual.read(buffer2)) {
      if (numRead == -1) break;
      for (int i = 0; i != numRead; ++i) {
        assertThat(buffer1[i], is(buffer2[i]));
      }
    }

    if (print) {
      // And try measuring how fast we can read the file ...
      sw = new Stopwatch();
      sw.start();
      while (-1 != actual.read(buffer2)) {}
      sw.stop();
      System.out.println("Time to read " + desc + ": " + sw.getTotalDuration());
    }
  }
コード例 #7
0
  @Test
  public void multipleThreadsShouldReadTheSameFile() throws Exception {
    final String textBase = "The quick brown fox jumps over the lazy dog";
    StringBuilder builder = new StringBuilder();
    Random rand = new Random();
    while (builder.length() <= MIN_BINARY_SIZE) {
      builder.append(textBase.substring(0, rand.nextInt(textBase.length())));
    }
    final String text = builder.toString();
    final Binary storedValue = store.storeValue(new ByteArrayInputStream(text.getBytes()), false);
    ExecutorService executor = Executors.newFixedThreadPool(3);
    Callable<String> readingTask =
        new Callable<String>() {
          @Override
          public String call() throws Exception {
            File tempFile = File.createTempFile("test-binary-store", "bin");
            try {
              FileOutputStream fos = new FileOutputStream(tempFile);
              InputStream is = storedValue.getStream();
              byte[] buff = new byte[100];
              int available;
              while ((available = is.read(buff)) != -1) {
                fos.write(buff, 0, available);
              }
              fos.close();

              return IoUtil.read(tempFile);
            } finally {
              tempFile.delete();
            }
          }
        };
    List<Callable<String>> tasks = Arrays.asList(readingTask, readingTask, readingTask);
    List<Future<String>> futures = executor.invokeAll(tasks, 5, TimeUnit.SECONDS);
    for (Future<String> future : futures) {
      assertEquals(text, future.get());
    }
  }