public void testAutocommit() throws Exception { Writer file = store.open("urn:test:file").openWriter(); file.append("blob store test"); file.close(); CharSequence str = store.open("urn:test:file").getCharContent(true); assertEquals("blob store test", str.toString()); }
public void testReuseVersion() throws Exception { BlobVersion trx1 = store.newVersion("urn:test:trx1"); Writer file1 = trx1.open("urn:test:file1").openWriter(); file1.append("blob store test"); file1.close(); trx1.commit(); BlobVersion trx2 = store.newVersion("urn:test:trx2"); file1 = trx2.open("urn:test:file1").openWriter(); file1.append("blob store test"); file1.close(); trx2.commit(); Writer file2 = trx2.open("urn:test:file2").openWriter(); file2.append("blob store test"); file2.close(); trx2.commit(); trx2 = store.newVersion("urn:test:trx2"); file2 = trx2.open("urn:test:file2").openWriter(); file2.append("blob store test"); file2.close(); trx2.commit(); BlobVersion trx3 = store.newVersion("urn:test:trx3"); CharSequence str1 = trx3.open("urn:test:file1").getCharContent(true); assertEquals("blob store test", str1.toString()); CharSequence str2 = trx3.open("urn:test:file2").getCharContent(true); assertEquals("blob store test", str2.toString()); }
public void testEraseSingleBlob() throws Exception { BlobVersion trx1 = store.newVersion("urn:test:trx1"); Writer file = trx1.open("urn:test:file").openWriter(); file.append("blob store test"); file.close(); trx1.commit(); store.erase(); assertEmpty(dir); }
public void testEraseMultiVersionSingleBlob() throws Exception { Writer file = store.open("urn:test:file").openWriter(); file.append("blob store test1"); file.close(); file = store.open("urn:test:file").openWriter(); file.append("blob store test2"); file.close(); store.erase(); assertEmpty(dir); }
public void testRoundTripString() throws Exception { BlobVersion trx1 = store.newVersion("urn:test:trx1"); Writer file = trx1.open("urn:test:file").openWriter(); file.append("blob store test"); file.close(); trx1.commit(); BlobVersion trx2 = store.newVersion("urn:test:trx2"); CharSequence str = trx2.open("urn:test:file").getCharContent(true); assertEquals("blob store test", str.toString()); }
public void testConcurrency() throws Exception { Writer test1 = store.open("urn:test:file").openWriter(); test1.append("test1"); test1.close(); Writer test2 = store.open("urn:test:file").openWriter(); test2.append("test2"); test2.flush(); assertEquals("test1", store.open("urn:test:file").getCharContent(true).toString()); test2.close(); assertEquals("test2", store.open("urn:test:file").getCharContent(true).toString()); }
private void recalcHashes(DataNode item) throws IOException { if (item.dirty == null) { return; // not dirty, which means no children are dirty } // only directories have derived hashes if (item instanceof DirectoryNode) { DirectoryNode dirNode = (DirectoryNode) item; for (DataNode child : dirNode) { recalcHashes(child); } ByteArrayOutputStream bout = new ByteArrayOutputStream(); hashCalc.sort(dirNode.getChildren()); String newHash = hashCalc.calcHash(dirNode, bout); item.setHash(newHash); byte[] arrTriplets = bout.toByteArray(); blobStore.setBlob(newHash, arrTriplets); log.info( "recalcHashes: " + item.name + " children:" + dirNode.members.size() + " hash=" + newHash); } }
public void testAtomicity() throws Exception { BlobVersion trx1 = store.newVersion("urn:test:trx1"); Writer file1 = trx1.open("urn:test:file1").openWriter(); file1.append("blob store test"); file1.close(); Writer file2 = trx1.open("urn:test:file2").openWriter(); file2.append("blob store test"); file2.close(); BlobVersion trx2 = store.newVersion("urn:test:trx2"); assertNull(trx2.open("urn:test:file1").getCharContent(true)); assertNull(trx2.open("urn:test:file2").getCharContent(true)); trx1.commit(); BlobVersion trx3 = store.newVersion("urn:test:trx3"); assertEquals("blob store test", trx3.open("urn:test:file1").getCharContent(true).toString()); assertEquals("blob store test", trx3.open("urn:test:file2").getCharContent(true).toString()); }
public void testReopenInvalid() throws Exception { try { store.openVersion("urn:test:nothing"); fail(); } catch (IllegalArgumentException e) { // pass } }
protected void initialize( BlobStore blobStore, ClusterName clusterName, @Nullable ByteSizeValue defaultChunkSize) throws IOException { this.blobStore = blobStore; this.chunkSize = componentSettings.getAsBytesSize("chunk_size", defaultChunkSize); this.basePath = BlobPath.cleanPath().add(clusterName.value()); this.metaDataBlobContainer = blobStore.immutableBlobContainer(basePath.add("metadata")); this.currentIndex = findLatestIndex(); this.compress = componentSettings.getAsBoolean("compress", true); logger.debug("Latest metadata found at index [" + currentIndex + "]"); }
public List<ITriplet> find(String hash) { // return DataItem.findByHash(hash, session); byte[] arr = blobStore.getBlob(hash); if (arr == null) { return null; } ByteArrayInputStream bin = new ByteArrayInputStream(arr); try { return hashCalc.parseTriplets(bin); } catch (IOException ex) { throw new RuntimeException(ex); } }
protected BlobStoreIndexShardGateway( ShardId shardId, @IndexSettings Settings indexSettings, ThreadPool threadPool, IndexGateway indexGateway, IndexShard indexShard, Store store) { super(shardId, indexSettings); this.threadPool = threadPool; this.indexShard = (InternalIndexShard) indexShard; this.store = store; BlobStoreIndexGateway blobStoreIndexGateway = (BlobStoreIndexGateway) indexGateway; this.chunkSize = blobStoreIndexGateway.chunkSize(); // can be null -> no chunking this.blobStore = blobStoreIndexGateway.blobStore(); this.shardPath = blobStoreIndexGateway.shardPath(shardId.id()); this.blobContainer = blobStore.immutableBlobContainer(shardPath); this.recoveryStatus = new RecoveryStatus(); }
public CommitPoint findCommitPoint(String index, int shardId) throws IOException { BlobPath path = BlobStoreIndexGateway.shardPath(basePath, index, shardId); ImmutableBlobContainer container = blobStore.immutableBlobContainer(path); ImmutableMap<String, BlobMetaData> blobs = container.listBlobs(); List<CommitPoint> commitPointsList = Lists.newArrayList(); for (BlobMetaData md : blobs.values()) { if (md.length() == 0) { // a commit point that was not flushed yet... continue; } if (md.name().startsWith("commit-")) { try { commitPointsList.add(CommitPoints.fromXContent(container.readBlobFully(md.name()))); } catch (Exception e) { logger.warn("failed to read commit point at path {} with name [{}]", e, path, md.name()); } } } CommitPoints commitPoints = new CommitPoints(commitPointsList); if (commitPoints.commits().isEmpty()) { return null; } return commitPoints.commits().get(0); }
@Override public void close(boolean delete) throws ElasticSearchException { if (delete) { blobStore.delete(shardPath); } }
public void testEraseEmpty() throws Exception { store.erase(); assertEmpty(dir); }
public void tearDown() throws Exception { store.erase(); dir.delete(); }
public void testIsolation() throws Exception { BlobVersion trx1 = store.newVersion("urn:test:trx1"); Writer file1 = trx1.open("urn:test:file1").openWriter(); file1.append("blob store test"); file1.close(); final CountDownLatch latch1 = new CountDownLatch(1); new Thread( new Runnable() { public void run() { try { error = null; try { BlobVersion trx2 = store.newVersion("urn:test:trx2"); BlobObject blob = trx2.open("urn:test:file1"); assertNull(blob.getCharContent(true)); } catch (Exception e) { e.printStackTrace(); fail(); } finally { latch1.countDown(); } } catch (AssertionFailedError e) { error = e; } } }) .start(); latch1.await(); if (error != null) throw error; trx1.prepare(); final CountDownLatch latch2 = new CountDownLatch(1); final CountDownLatch latch3 = new CountDownLatch(1); new Thread( new Runnable() { public void run() { try { error = null; try { latch2.countDown(); BlobVersion trx3 = store.newVersion("urn:test:trx3"); BlobObject blob = trx3.open("urn:test:file1"); CharSequence str = blob.getCharContent(true); assertNotNull(str); assertEquals("blob store test", str.toString()); } catch (Exception e) { e.printStackTrace(); fail(); } finally { latch3.countDown(); } } catch (AssertionFailedError e) { error = e; } } }) .start(); latch2.await(); assertFalse(latch3.await(1, TimeUnit.SECONDS)); trx1.commit(); latch3.await(); if (error != null) throw error; }
/** * Returns a hex enccoded SHA1 hash of the whole file. This can be used to locate the files bytes * again * * @param in * @param hashStore * @param blobStore * @return * @throws IOException */ public String parse(InputStream in, HashStore hashStore, BlobStore blobStore) throws IOException { if (log.isInfoEnabled()) { log.info("parse. inputstream: " + in); } Rsum rsum = new Rsum(128); int numBlobs = 0; byte[] arr = new byte[1024]; ByteArrayOutputStream bout = new ByteArrayOutputStream(); List<String> blobHashes = new ArrayList<>(); MessageDigest blobCrc = getCrypt(); MessageDigest fanoutCrc = getCrypt(); MessageDigest fileCrc = getCrypt(); long fanoutLength = 0; long fileLength = 0; int s = in.read(arr, 0, 1024); if (log.isTraceEnabled()) { log.trace("initial block size: " + s); } List<String> fanoutHashes = new ArrayList<>(); while (s >= 0) { numBytes += s; // log.trace("numBytes: {}", numBytes); if (cancelled) { throw new IOException("operation cancelled"); } for (int i = 0; i < s; i++) { byte b = arr[i]; rsum.roll(b); blobCrc.update(b); fanoutCrc.update(b); fileCrc.update(b); fanoutLength++; fileLength++; bout.write(b); int x = rsum.getValue(); // System.out.println("x=" + x); // System.out.println("check mask: " + (x & MASK) + " == " + MASK); boolean limited; if (MAX_BLOB_SIZE != null) { limited = bout.size() > MAX_BLOB_SIZE; if (limited) { log.warn("HIT BLOB LIMIT: " + bout.size()); } } else { limited = false; } if (((x & MASK) == MASK) || limited) { String blobCrcHex = toHex(blobCrc); byte[] blobBytes = bout.toByteArray(); if (log.isInfoEnabled()) { log.info( "Store blob: " + blobCrcHex + " length=" + blobBytes.length + " hash: " + x + " mask: " + MASK); } blobStore.setBlob(blobCrcHex, blobBytes); bout.reset(); blobHashes.add(blobCrcHex); blobCrc.reset(); if ((x & FANOUT_MASK) == FANOUT_MASK) { String fanoutCrcVal = toHex(fanoutCrc); fanoutHashes.add(fanoutCrcVal); // log.info("set chunk fanout: {} length={}", fanoutCrcVal, fanoutLength); hashStore.setChunkFanout(fanoutCrcVal, blobHashes, fanoutLength); fanoutLength = 0; fanoutCrc.reset(); blobHashes = new ArrayList<>(); } numBlobs++; rsum.reset(); } } s = in.read(arr, 0, 1024); } // Need to store terminal data, ie data which has been accumulated since the last boundary String blobCrcHex = toHex(blobCrc); // System.out.println("Store terminal blob: " + blobCrcHex); blobStore.setBlob(blobCrcHex, bout.toByteArray()); numBlobs++; blobHashes.add(blobCrcHex); String fanoutCrcVal = toHex(fanoutCrc); // log.info("set terminal chunk fanout: {} length={}" ,fanoutCrcVal, fanoutLength); hashStore.setChunkFanout(fanoutCrcVal, blobHashes, fanoutLength); fanoutHashes.add(fanoutCrcVal); // Now store a fanout for the whole file. The contained hashes locate other fanouts String fileCrcVal = toHex(fileCrc); if (log.isInfoEnabled()) { log.info( "set file fanout: " + fanoutCrcVal + " length=" + fileLength + " avg blob size=" + fileLength / numBlobs); } hashStore.setFileFanout(fileCrcVal, fanoutHashes, fileLength); return fileCrcVal; }
public boolean dirExists(String hash) { return blobStore.hasBlob(hash); }
@Override public void reset() throws Exception { blobStore.delete(BlobPath.cleanPath()); }
@Override protected void delete(IndexMetaData indexMetaData) throws ElasticSearchException { BlobPath indexPath = basePath().add("indices").add(indexMetaData.index()); blobStore.delete(indexPath); }