@Before public void beforeClass() { directory = new File("target/fsbs/"); FileUtil.delete(directory); directory.mkdirs(); trash = new File(directory, FileSystemBinaryStore.TRASH_DIRECTORY_NAME); store = new FileSystemBinaryStore(directory); store.setMinimumBinarySizeInBytes(MIN_BINARY_SIZE); store.setMimeTypeDetector(DEFAULT_DETECTOR); print = false; }
protected Binary storeAndCheck(int contentIndex, Class<? extends Binary> valueClass) throws Exception { String content = CONTENT[contentIndex]; String sha1 = CONTENT_HASHES[contentIndex]; InputStream stream = new ByteArrayInputStream(content.getBytes()); Stopwatch sw = new Stopwatch(); sw.start(); Binary binary = store.storeValue(stream, false); sw.stop(); if (print) System.out.println("Time to store 18MB file: " + sw.getTotalDuration()); if (valueClass != null) { assertThat(binary, is(instanceOf(valueClass))); } if (content.length() == 0) { assertThat(binary, is(instanceOf(EmptyBinaryValue.class))); } else if (content.length() < MIN_BINARY_SIZE) { assertThat(binary, is(instanceOf(InMemoryBinaryValue.class))); } else { assertThat(binary, is(instanceOf(StoredBinaryValue.class))); } assertThat(binary.getHexHash(), is(sha1)); String binaryContent = IoUtil.read(binary.getStream()); assertThat(binaryContent, is(content)); return binary; }
@Test public void shouldCleanTrashFilesWhenFilesBecomeUsed() throws Exception { Set<Binary> binaries = new HashSet<Binary>(); int storedCount = 0; for (int i = 0; i != CONTENT.length; ++i) { Binary binary = storeAndCheck(i); assertThat(binary, is(notNullValue())); binaries.add(binary); if (binary instanceof StoredBinaryValue) storedCount++; } // Make sure there are files for all stored values ... assertThat(countStoredFiles(), is(storedCount)); assertThat(countTrashFiles(), is(0)); // Mark one of the files as being unused ... String unused = binaries.iterator().next().getHexHash(); BinaryKey unusedKey = new BinaryKey(unused); store.markAsUnused(Collections.singleton(unusedKey)); // Make sure the file was moved to the trash ... assertThat(countStoredFiles(), is(storedCount)); assertThat(countTrashFiles(), is(1)); // Now access all the binary files which will not change there used/unused state for (Binary binary : binaries) { InputStream stream = binary.getStream(); String content = IoUtil.read(stream); assertThat(content.length() != 0, is(true)); } // Make sure there are files for all stored values ... assertThat(countStoredFiles(), is(storedCount)); assertThat(countTrashFiles(), is(1)); // Now mark the file explicitly as used and check that the file from the trash was removed store.markAsUsed(Collections.singleton(unusedKey)); assertThat(countTrashFiles(), is(0)); }
@Override public BinaryValue storeValue(InputStream stream) throws BinaryStoreException { // store into temporary file system store and get SHA-1 BinaryValue temp = cache.storeValue(stream); try { // prepare new binary key based on SHA-1 BinaryKey key = new BinaryKey(temp.getKey().toString()); // check for duplicate content if (this.contentExists(key, ALIVE)) { return new StoredBinaryValue(this, key, temp.getSize()); } // check unused content if (this.contentExists(key, UNUSED)) { session.execute("UPDATE modeshape.binary SET usage=1 WHERE cid='" + key + "';"); return new StoredBinaryValue(this, key, temp.getSize()); } // store content PreparedStatement query = session.prepare( "INSERT INTO modeshape.binary (cid, usage_time, payload, usage) VALUES ( ?,?,?,1 );"); BoundStatement statement = new BoundStatement(query); session.execute(statement.bind(key.toString(), new Date(), buffer(stream))); return new StoredBinaryValue(this, key, temp.getSize()); } catch (BinaryStoreException e) { throw e; } catch (IOException e) { throw new BinaryStoreException(e); } catch (RuntimeException e) { throw new BinaryStoreException(e); } finally { // remove content from temp store cache.markAsUnused(temp.getKey()); } }
@Test public void shouldCreateTrashFilesForUnusedBinaries() throws Exception { Set<String> storedSha1s = new HashSet<String>(); for (int i = 0; i != CONTENT.length; ++i) { Binary binary = storeAndCheck(i); if (binary instanceof StoredBinaryValue) storedSha1s.add(binary.getHexHash()); } // Make sure there are files for all stored values ... assertThat(countStoredFiles(), is(storedSha1s.size())); assertThat(countTrashFiles(), is(0)); // Mark one of the files as being unused ... String unused = storedSha1s.iterator().next(); store.markAsUnused(Collections.singleton(new BinaryKey(unused))); // Make sure the trash file was created assertThat(countStoredFiles(), is(storedSha1s.size())); assertThat(countTrashFiles(), is(1)); // Check that the name of the trash file is the SHA1 File trashFile = collectFiles(trash).get(0); assertNotNull(trashFile); assertEquals(unused, trashFile.getName()); Thread.sleep( 1100L); // Sleep more than a second, since modified times may only be accurate to nearest // second ... store.removeValuesUnusedLongerThan(1, TimeUnit.SECONDS); // Make sure the file was removed from the trash ... assertThat(countStoredFiles(), is(storedSha1s.size() - 1)); assertThat(countTrashFiles(), is(0)); // And that all directories in the trash were removed (since they should be empty) ... assertThat(trash.listFiles().length, is(0)); }
protected void storeAndCheckResource( String resourcePath, String expectedSha1, String desc, long numBytes) throws Exception { InputStream content = getClass().getClassLoader().getResourceAsStream(resourcePath); assertThat(content, is(notNullValue())); Stopwatch sw = new Stopwatch(); sw.start(); Binary binary = store.storeValue(content, false); sw.stop(); if (print) System.out.println("Time to store " + desc + ": " + sw.getTotalDuration()); if (numBytes == 0) { assertThat(binary, is(instanceOf(EmptyBinaryValue.class))); } else if (numBytes < MIN_BINARY_SIZE) { assertThat(binary, is(instanceOf(InMemoryBinaryValue.class))); } else { assertThat(binary, is(instanceOf(StoredBinaryValue.class))); } assertThat(binary.getHexHash(), is(expectedSha1)); assertThat(binary.getSize(), is(numBytes)); // Now try reading and comparing the two streams ... InputStream expected = getClass().getClassLoader().getResourceAsStream(resourcePath); InputStream actual = binary.getStream(); byte[] buffer1 = new byte[1024]; byte[] buffer2 = new byte[1024]; int numRead = 0; while ((numRead = expected.read(buffer1)) == actual.read(buffer2)) { if (numRead == -1) break; for (int i = 0; i != numRead; ++i) { assertThat(buffer1[i], is(buffer2[i])); } } if (print) { // And try measuring how fast we can read the file ... sw = new Stopwatch(); sw.start(); while (-1 != actual.read(buffer2)) {} sw.stop(); System.out.println("Time to read " + desc + ": " + sw.getTotalDuration()); } }
@Test public void multipleThreadsShouldReadTheSameFile() throws Exception { final String textBase = "The quick brown fox jumps over the lazy dog"; StringBuilder builder = new StringBuilder(); Random rand = new Random(); while (builder.length() <= MIN_BINARY_SIZE) { builder.append(textBase.substring(0, rand.nextInt(textBase.length()))); } final String text = builder.toString(); final Binary storedValue = store.storeValue(new ByteArrayInputStream(text.getBytes()), false); ExecutorService executor = Executors.newFixedThreadPool(3); Callable<String> readingTask = new Callable<String>() { @Override public String call() throws Exception { File tempFile = File.createTempFile("test-binary-store", "bin"); try { FileOutputStream fos = new FileOutputStream(tempFile); InputStream is = storedValue.getStream(); byte[] buff = new byte[100]; int available; while ((available = is.read(buff)) != -1) { fos.write(buff, 0, available); } fos.close(); return IoUtil.read(tempFile); } finally { tempFile.delete(); } } }; List<Callable<String>> tasks = Arrays.asList(readingTask, readingTask, readingTask); List<Future<String>> futures = executor.invokeAll(tasks, 5, TimeUnit.SECONDS); for (Future<String> future : futures) { assertEquals(text, future.get()); } }