// 列出某个bucket的所有objects public static void listObjectInBucket(Bucket bucket) { List<Bucket> buckets = conn.listBuckets(); ObjectListing objects = conn.listObjects(bucket.getName()); do { for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) { System.out.println( objectSummary.getKey() + "\t" + objectSummary.getSize() + "\t" + StringUtils.fromDate(objectSummary.getLastModified())); } objects = conn.listNextBatchOfObjects(objects); } while (objects.isTruncated()); }
// 列出有prefix前缀的所有objects public static void listObjectInBucketWithPrefix(Bucket bucket, String prefix) { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket.getName()).withPrefix(prefix); ObjectListing objects = conn.listObjects(listObjectsRequest); do { for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) { System.out.println( objectSummary.getKey() + "\t" + objectSummary.getSize() + "\t" + StringUtils.fromDate(objectSummary.getLastModified())); } objects = conn.listNextBatchOfObjects(objects); } while (objects.isTruncated()); }
@Override public void remove(String noteId, AuthenticationInfo subject) throws IOException { String key = user + "/" + "notebook" + "/" + noteId; final ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(key); try { ObjectListing objects = s3client.listObjects(listObjectsRequest); do { for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) { s3client.deleteObject(bucketName, objectSummary.getKey()); } objects = s3client.listNextBatchOfObjects(objects); } while (objects.isTruncated()); } catch (AmazonClientException ace) { throw new IOException("Unable to remove note in S3: " + ace, ace); } }
@Override protected Response serve() { JsonObject json = new JsonObject(); JsonArray succ = new JsonArray(); JsonArray fail = new JsonArray(); String bucket = _bucket.value(); AmazonS3 s3 = PersistS3.getClient(); ObjectListing currentList = s3.listObjects(bucket); processListing(currentList, succ, fail); while (currentList.isTruncated()) { currentList = s3.listNextBatchOfObjects(currentList); processListing(currentList, succ, fail); } json.add(NUM_SUCCEEDED, new JsonPrimitive(succ.size())); json.add(SUCCEEDED, succ); json.add(NUM_FAILED, new JsonPrimitive(fail.size())); json.add(FAILED, fail); DKV.write_barrier(); Response r = Response.done(json); r.setBuilder(SUCCEEDED + "." + KEY, new KeyCellBuilder()); return r; }
public static void main(String[] args) { if (args.length < 2) { System.out.println("USAGE: localPath bucketname <bucketPrefix>"); System.exit(1); } String localPath = args[0]; String bucketName = args[1]; String bucketPrefix = ""; // add on extra slash if (!localPath.endsWith("/")) localPath += "/"; if (args.length == 3) bucketPrefix = args[2]; // check local dir, make if it does not exist File localDir = new File(localPath); if (!localDir.exists()) localDir.mkdirs(); if (!localDir.isDirectory()) { System.out.println("Local Dir is not a dir: " + localPath); System.exit(1); } long totalBytes = 0; long start = System.currentTimeMillis(); AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); ObjectListing listObjects = s3.listObjects(bucketName, bucketPrefix); do { for (S3ObjectSummary objectSummary : listObjects.getObjectSummaries()) { S3Object object = s3.getObject(bucketName, objectSummary.getKey()); S3ObjectInputStream inputStream = object.getObjectContent(); if ("gzip".equals(object.getObjectMetadata().getContentEncoding())) { InputStream in = null; try { totalBytes += object.getObjectMetadata().getContentLength(); in = new GZIPInputStream(inputStream); // write this sucker out String path = localPath + object.getKey(); // have to take the gz off since this is not downloading compressed! if (path.endsWith(".gz")) path = path.substring(0, path.length() - 3); System.out.print("Writing file: " + path); File check = new File(path); File parentFile = check.getParentFile(); if (!parentFile.exists()) parentFile.mkdirs(); FileOutputStream out = new FileOutputStream(path); IOUtils.copy(in, out); System.out.println(" written."); } catch (IOException e) { System.out.println("crap"); e.printStackTrace(); throw new IllegalStateException("files are too hard", e); } finally { IOUtils.closeQuietly(in); } } else { System.out.println( "unhandled content encoding: " + object.getObjectMetadata().getContentEncoding()); } } listObjects = s3.listNextBatchOfObjects(listObjects); } while (listObjects.isTruncated()); long now = System.currentTimeMillis(); System.out.println( (totalBytes / 1000.0 / 1000.0) + " mb downloaded in " + ((now - start) / 1000) + " seconds."); }
/** Deletes content of the repository files in the bucket */ public void cleanRepositoryFiles(String basePath) { Settings settings = internalCluster().getInstance(Settings.class); Settings[] buckets = { settings.getByPrefix("repositories.s3."), settings.getByPrefix("repositories.s3.private-bucket."), settings.getByPrefix("repositories.s3.remote-bucket."), settings.getByPrefix("repositories.s3.external-bucket.") }; for (Settings bucket : buckets) { String endpoint = bucket.get("endpoint", settings.get("repositories.s3.endpoint")); String protocol = bucket.get("protocol", settings.get("repositories.s3.protocol")); String region = bucket.get("region", settings.get("repositories.s3.region")); String accessKey = bucket.get("access_key", settings.get("cloud.aws.access_key")); String secretKey = bucket.get("secret_key", settings.get("cloud.aws.secret_key")); String bucketName = bucket.get("bucket"); // We check that settings has been set in elasticsearch.yml integration test file // as described in README assertThat( "Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue()); AmazonS3 client = internalCluster() .getInstance(AwsS3Service.class) .client(endpoint, protocol, region, accessKey, secretKey); try { ObjectListing prevListing = null; // From // http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html // we can do at most 1K objects per delete // We don't know the bucket name until first object listing DeleteObjectsRequest multiObjectDeleteRequest = null; ArrayList<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<DeleteObjectsRequest.KeyVersion>(); while (true) { ObjectListing list; if (prevListing != null) { list = client.listNextBatchOfObjects(prevListing); } else { list = client.listObjects(bucketName, basePath); multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName()); } for (S3ObjectSummary summary : list.getObjectSummaries()) { keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey())); // Every 500 objects batch the delete request if (keys.size() > 500) { multiObjectDeleteRequest.setKeys(keys); client.deleteObjects(multiObjectDeleteRequest); multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName()); keys.clear(); } } if (list.isTruncated()) { prevListing = list; } else { break; } } if (!keys.isEmpty()) { multiObjectDeleteRequest.setKeys(keys); client.deleteObjects(multiObjectDeleteRequest); } } catch (Throwable ex) { logger.warn("Failed to delete S3 repository [{}] in [{}]", ex, bucketName, region); } } }