コード例 #1
0
ファイル: ceph.java プロジェクト: XiaoqingWang/MovieSite
 // 列出某个bucket的所有objects
 public static void listObjectInBucket(Bucket bucket) {
   List<Bucket> buckets = conn.listBuckets();
   ObjectListing objects = conn.listObjects(bucket.getName());
   do {
     for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
       System.out.println(
           objectSummary.getKey()
               + "\t"
               + objectSummary.getSize()
               + "\t"
               + StringUtils.fromDate(objectSummary.getLastModified()));
     }
     objects = conn.listNextBatchOfObjects(objects);
   } while (objects.isTruncated());
 }
コード例 #2
0
ファイル: ceph.java プロジェクト: XiaoqingWang/MovieSite
 // 列出有prefix前缀的所有objects
 public static void listObjectInBucketWithPrefix(Bucket bucket, String prefix) {
   ListObjectsRequest listObjectsRequest =
       new ListObjectsRequest().withBucketName(bucket.getName()).withPrefix(prefix);
   ObjectListing objects = conn.listObjects(listObjectsRequest);
   do {
     for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
       System.out.println(
           objectSummary.getKey()
               + "\t"
               + objectSummary.getSize()
               + "\t"
               + StringUtils.fromDate(objectSummary.getLastModified()));
     }
     objects = conn.listNextBatchOfObjects(objects);
   } while (objects.isTruncated());
 }
コード例 #3
0
ファイル: S3NotebookRepo.java プロジェクト: conker84/zeppelin
  @Override
  public void remove(String noteId, AuthenticationInfo subject) throws IOException {
    String key = user + "/" + "notebook" + "/" + noteId;
    final ListObjectsRequest listObjectsRequest =
        new ListObjectsRequest().withBucketName(bucketName).withPrefix(key);

    try {
      ObjectListing objects = s3client.listObjects(listObjectsRequest);
      do {
        for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
          s3client.deleteObject(bucketName, objectSummary.getKey());
        }
        objects = s3client.listNextBatchOfObjects(objects);
      } while (objects.isTruncated());
    } catch (AmazonClientException ace) {
      throw new IOException("Unable to remove note in S3: " + ace, ace);
    }
  }
コード例 #4
0
ファイル: ImportS3.java プロジェクト: pragnesh/h2o
 @Override
 protected Response serve() {
   JsonObject json = new JsonObject();
   JsonArray succ = new JsonArray();
   JsonArray fail = new JsonArray();
   String bucket = _bucket.value();
   AmazonS3 s3 = PersistS3.getClient();
   ObjectListing currentList = s3.listObjects(bucket);
   processListing(currentList, succ, fail);
   while (currentList.isTruncated()) {
     currentList = s3.listNextBatchOfObjects(currentList);
     processListing(currentList, succ, fail);
   }
   json.add(NUM_SUCCEEDED, new JsonPrimitive(succ.size()));
   json.add(SUCCEEDED, succ);
   json.add(NUM_FAILED, new JsonPrimitive(fail.size()));
   json.add(FAILED, fail);
   DKV.write_barrier();
   Response r = Response.done(json);
   r.setBuilder(SUCCEEDED + "." + KEY, new KeyCellBuilder());
   return r;
 }
コード例 #5
0
  public static void main(String[] args) {

    if (args.length < 2) {
      System.out.println("USAGE: localPath bucketname <bucketPrefix>");
      System.exit(1);
    }

    String localPath = args[0];
    String bucketName = args[1];
    String bucketPrefix = "";

    // add on extra slash
    if (!localPath.endsWith("/")) localPath += "/";

    if (args.length == 3) bucketPrefix = args[2];

    // check local dir, make if it does not exist
    File localDir = new File(localPath);
    if (!localDir.exists()) localDir.mkdirs();

    if (!localDir.isDirectory()) {
      System.out.println("Local Dir is not a dir: " + localPath);
      System.exit(1);
    }

    long totalBytes = 0;
    long start = System.currentTimeMillis();

    AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());

    ObjectListing listObjects = s3.listObjects(bucketName, bucketPrefix);
    do {
      for (S3ObjectSummary objectSummary : listObjects.getObjectSummaries()) {
        S3Object object = s3.getObject(bucketName, objectSummary.getKey());
        S3ObjectInputStream inputStream = object.getObjectContent();
        if ("gzip".equals(object.getObjectMetadata().getContentEncoding())) {
          InputStream in = null;
          try {

            totalBytes += object.getObjectMetadata().getContentLength();

            in = new GZIPInputStream(inputStream);
            // write this sucker out
            String path = localPath + object.getKey();
            // have to take the gz off since this is not downloading compressed!
            if (path.endsWith(".gz")) path = path.substring(0, path.length() - 3);
            System.out.print("Writing file: " + path);
            File check = new File(path);
            File parentFile = check.getParentFile();
            if (!parentFile.exists()) parentFile.mkdirs();

            FileOutputStream out = new FileOutputStream(path);
            IOUtils.copy(in, out);
            System.out.println(" written.");

          } catch (IOException e) {
            System.out.println("crap");
            e.printStackTrace();
            throw new IllegalStateException("files are too hard", e);
          } finally {
            IOUtils.closeQuietly(in);
          }
        } else {
          System.out.println(
              "unhandled content encoding: " + object.getObjectMetadata().getContentEncoding());
        }
      }

      listObjects = s3.listNextBatchOfObjects(listObjects);
    } while (listObjects.isTruncated());

    long now = System.currentTimeMillis();
    System.out.println(
        (totalBytes / 1000.0 / 1000.0)
            + " mb downloaded in "
            + ((now - start) / 1000)
            + " seconds.");
  }
コード例 #6
0
  /** Deletes content of the repository files in the bucket */
  public void cleanRepositoryFiles(String basePath) {
    Settings settings = internalCluster().getInstance(Settings.class);
    Settings[] buckets = {
      settings.getByPrefix("repositories.s3."),
      settings.getByPrefix("repositories.s3.private-bucket."),
      settings.getByPrefix("repositories.s3.remote-bucket."),
      settings.getByPrefix("repositories.s3.external-bucket.")
    };
    for (Settings bucket : buckets) {
      String endpoint = bucket.get("endpoint", settings.get("repositories.s3.endpoint"));
      String protocol = bucket.get("protocol", settings.get("repositories.s3.protocol"));
      String region = bucket.get("region", settings.get("repositories.s3.region"));
      String accessKey = bucket.get("access_key", settings.get("cloud.aws.access_key"));
      String secretKey = bucket.get("secret_key", settings.get("cloud.aws.secret_key"));
      String bucketName = bucket.get("bucket");

      // We check that settings has been set in elasticsearch.yml integration test file
      // as described in README
      assertThat(
          "Your settings in elasticsearch.yml are incorrects. Check README file.",
          bucketName,
          notNullValue());
      AmazonS3 client =
          internalCluster()
              .getInstance(AwsS3Service.class)
              .client(endpoint, protocol, region, accessKey, secretKey);
      try {
        ObjectListing prevListing = null;
        // From
        // http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
        // we can do at most 1K objects per delete
        // We don't know the bucket name until first object listing
        DeleteObjectsRequest multiObjectDeleteRequest = null;
        ArrayList<DeleteObjectsRequest.KeyVersion> keys =
            new ArrayList<DeleteObjectsRequest.KeyVersion>();
        while (true) {
          ObjectListing list;
          if (prevListing != null) {
            list = client.listNextBatchOfObjects(prevListing);
          } else {
            list = client.listObjects(bucketName, basePath);
            multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
          }
          for (S3ObjectSummary summary : list.getObjectSummaries()) {
            keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
            // Every 500 objects batch the delete request
            if (keys.size() > 500) {
              multiObjectDeleteRequest.setKeys(keys);
              client.deleteObjects(multiObjectDeleteRequest);
              multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
              keys.clear();
            }
          }
          if (list.isTruncated()) {
            prevListing = list;
          } else {
            break;
          }
        }
        if (!keys.isEmpty()) {
          multiObjectDeleteRequest.setKeys(keys);
          client.deleteObjects(multiObjectDeleteRequest);
        }
      } catch (Throwable ex) {
        logger.warn("Failed to delete S3 repository [{}] in [{}]", ex, bucketName, region);
      }
    }
  }