示例#1
0
  private int findLatestIndex() throws IOException {
    ImmutableMap<String, BlobMetaData> blobs = metaDataBlobContainer.listBlobsByPrefix("metadata-");

    int index = -1;
    for (BlobMetaData md : blobs.values()) {
      if (logger.isTraceEnabled()) {
        logger.trace("[findLatestMetadata]: Processing [" + md.name() + "]");
      }
      String name = md.name();
      int fileIndex = Integer.parseInt(name.substring(name.indexOf('-') + 1));
      if (fileIndex >= index) {
        // try and read the meta data
        byte[] data = null;
        try {
          data = metaDataBlobContainer.readBlobFully(name);
          readMetaData(data);
          index = fileIndex;
        } catch (IOException e) {
          logger.warn(
              "[findLatestMetadata]: failed to read metadata from [{}], data_length [{}] ignoring...",
              e,
              name,
              data == null ? "na" : data.length);
        }
      }
    }

    return index;
  }
示例#2
0
  @Override
  public void write(MetaData metaData) throws GatewayException {
    final String newMetaData = "metadata-" + (currentIndex + 1);
    CachedStreamOutput.Entry cachedEntry = CachedStreamOutput.popEntry();
    try {
      StreamOutput streamOutput;
      if (compress) {
        streamOutput = cachedEntry.bytes(CompressorFactory.defaultCompressor());
      } else {
        streamOutput = cachedEntry.bytes();
      }
      XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, streamOutput);
      builder.startObject();
      MetaData.Builder.toXContent(metaData, builder, ToXContent.EMPTY_PARAMS);
      builder.endObject();
      builder.close();
      metaDataBlobContainer.writeBlob(
          newMetaData,
          new ByteArrayInputStream(
              cachedEntry.bytes().underlyingBytes(), 0, cachedEntry.bytes().size()),
          cachedEntry.bytes().size());
    } catch (IOException e) {
      throw new GatewayException("Failed to write metadata [" + newMetaData + "]", e);
    } finally {
      CachedStreamOutput.pushEntry(cachedEntry);
    }

    currentIndex++;

    try {
      metaDataBlobContainer.deleteBlobsByFilter(
          new BlobContainer.BlobNameFilter() {
            @Override
            public boolean accept(String blobName) {
              return blobName.startsWith("metadata-") && !newMetaData.equals(blobName);
            }
          });
    } catch (IOException e) {
      logger.debug("Failed to delete old metadata, will do it next time", e);
    }
  }
示例#3
0
 public CommitPoint findCommitPoint(String index, int shardId) throws IOException {
   BlobPath path = BlobStoreIndexGateway.shardPath(basePath, index, shardId);
   ImmutableBlobContainer container = blobStore.immutableBlobContainer(path);
   ImmutableMap<String, BlobMetaData> blobs = container.listBlobs();
   List<CommitPoint> commitPointsList = Lists.newArrayList();
   for (BlobMetaData md : blobs.values()) {
     if (md.length() == 0) { // a commit point that was not flushed yet...
       continue;
     }
     if (md.name().startsWith("commit-")) {
       try {
         commitPointsList.add(CommitPoints.fromXContent(container.readBlobFully(md.name())));
       } catch (Exception e) {
         logger.warn("failed to read commit point at path {} with name [{}]", e, path, md.name());
       }
     }
   }
   CommitPoints commitPoints = new CommitPoints(commitPointsList);
   if (commitPoints.commits().isEmpty()) {
     return null;
   }
   return commitPoints.commits().get(0);
 }
示例#4
0
  @Override
  public MetaData read() throws GatewayException {
    try {
      this.currentIndex = findLatestIndex();
    } catch (IOException e) {
      throw new GatewayException("Failed to find latest metadata to read from", e);
    }
    if (currentIndex == -1) return null;
    String metaData = "metadata-" + currentIndex;

    try {
      return readMetaData(metaDataBlobContainer.readBlobFully(metaData));
    } catch (GatewayException e) {
      throw e;
    } catch (Exception e) {
      throw new GatewayException("Failed to read metadata [" + metaData + "] from gateway", e);
    }
  }