Exemplo n.º 1
0
  private int findLatestIndex() throws IOException {
    ImmutableMap<String, BlobMetaData> blobs = metaDataBlobContainer.listBlobsByPrefix("metadata-");

    int index = -1;
    for (BlobMetaData md : blobs.values()) {
      if (logger.isTraceEnabled()) {
        logger.trace("[findLatestMetadata]: Processing [" + md.name() + "]");
      }
      String name = md.name();
      int fileIndex = Integer.parseInt(name.substring(name.indexOf('-') + 1));
      if (fileIndex >= index) {
        // try and read the meta data
        byte[] data = null;
        try {
          data = metaDataBlobContainer.readBlobFully(name);
          readMetaData(data);
          index = fileIndex;
        } catch (IOException e) {
          logger.warn(
              "[findLatestMetadata]: failed to read metadata from [{}], data_length [{}] ignoring...",
              e,
              name,
              data == null ? "na" : data.length);
        }
      }
    }

    return index;
  }
 private boolean commitPointFileExistsInBlobs(
     CommitPoint.FileInfo fileInfo, ImmutableMap<String, BlobMetaData> blobs) {
   BlobMetaData blobMetaData = blobs.get(fileInfo.name());
   if (blobMetaData != null) {
     if (blobMetaData.length() != fileInfo.length()) {
       return false;
     }
   } else if (blobs.containsKey(fileInfo.name() + ".part0")) {
     // multi part file sum up the size and check
     int part = 0;
     long totalSize = 0;
     while (true) {
       blobMetaData = blobs.get(fileInfo.name() + ".part" + part++);
       if (blobMetaData == null) {
         break;
       }
       totalSize += blobMetaData.length();
     }
     if (totalSize != fileInfo.length()) {
       return false;
     }
   } else {
     // no file, not exact and not multipart
     return false;
   }
   return true;
 }
 @Override
 public void deleteBlobsByFilter(BlobNameFilter filter) throws IOException {
   ImmutableMap<String, BlobMetaData> blobs = listBlobs();
   for (BlobMetaData blob : blobs.values()) {
     if (filter.accept(blob.name())) {
       deleteBlob(blob.name());
     }
   }
 }
 @Override
 public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(String blobNamePrefix)
     throws IOException {
   ImmutableMap<String, BlobMetaData> allBlobs = listBlobs();
   ImmutableMap.Builder<String, BlobMetaData> blobs = ImmutableMap.builder();
   for (BlobMetaData blob : allBlobs.values()) {
     if (blob.name().startsWith(blobNamePrefix)) {
       blobs.put(blob.name(), blob);
     }
   }
   return blobs.build();
 }
Exemplo n.º 5
0
 public CommitPoint findCommitPoint(String index, int shardId) throws IOException {
   BlobPath path = BlobStoreIndexGateway.shardPath(basePath, index, shardId);
   ImmutableBlobContainer container = blobStore.immutableBlobContainer(path);
   ImmutableMap<String, BlobMetaData> blobs = container.listBlobs();
   List<CommitPoint> commitPointsList = Lists.newArrayList();
   for (BlobMetaData md : blobs.values()) {
     if (md.length() == 0) { // a commit point that was not flushed yet...
       continue;
     }
     if (md.name().startsWith("commit-")) {
       try {
         commitPointsList.add(CommitPoints.fromXContent(container.readBlobFully(md.name())));
       } catch (Exception e) {
         logger.warn("failed to read commit point at path {} with name [{}]", e, path, md.name());
       }
     }
   }
   CommitPoints commitPoints = new CommitPoints(commitPointsList);
   if (commitPoints.commits().isEmpty()) {
     return null;
   }
   return commitPoints.commits().get(0);
 }