private int findLatestIndex() throws IOException { ImmutableMap<String, BlobMetaData> blobs = metaDataBlobContainer.listBlobsByPrefix("metadata-"); int index = -1; for (BlobMetaData md : blobs.values()) { if (logger.isTraceEnabled()) { logger.trace("[findLatestMetadata]: Processing [" + md.name() + "]"); } String name = md.name(); int fileIndex = Integer.parseInt(name.substring(name.indexOf('-') + 1)); if (fileIndex >= index) { // try and read the meta data byte[] data = null; try { data = metaDataBlobContainer.readBlobFully(name); readMetaData(data); index = fileIndex; } catch (IOException e) { logger.warn( "[findLatestMetadata]: failed to read metadata from [{}], data_length [{}] ignoring...", e, name, data == null ? "na" : data.length); } } } return index; }
@Override public void write(MetaData metaData) throws GatewayException { final String newMetaData = "metadata-" + (currentIndex + 1); CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry(); try { StreamOutput streamOutput; if (compress) { streamOutput = cachedEntry.bytes(CompressorFactory.defaultCompressor()); } else { streamOutput = cachedEntry.bytes(); } XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, streamOutput); builder.startObject(); MetaData.Builder.toXContent(metaData, builder, ToXContent.EMPTY_PARAMS); builder.endObject(); builder.close(); metaDataBlobContainer.writeBlob( newMetaData, new ByteArrayInputStream( cachedEntry.bytes().underlyingBytes(), 0, cachedEntry.bytes().size()), cachedEntry.bytes().size()); } catch (IOException e) { throw new GatewayException("Failed to write metadata [" + newMetaData + "]", e); } finally { CachedStreamOutput.pushEntry(cachedEntry); } currentIndex++; try { metaDataBlobContainer.deleteBlobsByFilter( new BlobContainer.BlobNameFilter() { @Override public boolean accept(String blobName) { return blobName.startsWith("metadata-") && !newMetaData.equals(blobName); } }); } catch (IOException e) { logger.debug("Failed to delete old metadata, will do it next time", e); } }
public CommitPoint findCommitPoint(String index, int shardId) throws IOException { BlobPath path = BlobStoreIndexGateway.shardPath(basePath, index, shardId); ImmutableBlobContainer container = blobStore.immutableBlobContainer(path); ImmutableMap<String, BlobMetaData> blobs = container.listBlobs(); List<CommitPoint> commitPointsList = Lists.newArrayList(); for (BlobMetaData md : blobs.values()) { if (md.length() == 0) { // a commit point that was not flushed yet... continue; } if (md.name().startsWith("commit-")) { try { commitPointsList.add(CommitPoints.fromXContent(container.readBlobFully(md.name()))); } catch (Exception e) { logger.warn("failed to read commit point at path {} with name [{}]", e, path, md.name()); } } } CommitPoints commitPoints = new CommitPoints(commitPointsList); if (commitPoints.commits().isEmpty()) { return null; } return commitPoints.commits().get(0); }
@Override public MetaData read() throws GatewayException { try { this.currentIndex = findLatestIndex(); } catch (IOException e) { throw new GatewayException("Failed to find latest metadata to read from", e); } if (currentIndex == -1) return null; String metaData = "metadata-" + currentIndex; try { return readMetaData(metaDataBlobContainer.readBlobFully(metaData)); } catch (GatewayException e) { throw e; } catch (Exception e) { throw new GatewayException("Failed to read metadata [" + metaData + "] from gateway", e); } }