/**
* Reads next key from S3
*
* @return {@link S3ObjectSummary} of the key or <code>null</code> if there are no more keys to
* read
*/
public S3ObjectSummary getNextKey() {
if (objectListing == null) {
LOG.debug("Listing objects");
objectListing = s3Client.listObjects(listObjectsRequest);
}
// we have more elements in list
if (currentPosition < objectListing.getObjectSummaries().size()) {
currentObject = objectListing.getObjectSummaries().get(currentPosition++);
} else if (objectListing.isTruncated()) {
LOG.debug("Current batch reached its end. Fetching next page.");
// when we're at the end, check if there's more data to read
listObjectsRequest.setMarker(objectListing.getNextMarker());
LOG.debug("New marker is set to {}", listObjectsRequest.getMarker());
objectListing = s3Client.listObjects(listObjectsRequest);
currentPosition = 0;
if (currentPosition < objectListing.getObjectSummaries().size()) {
currentObject = objectListing.getObjectSummaries().get(currentPosition++);
} else {
LOG.debug("No more objects to read");
currentObject = null;
}
} else {
currentObject = null;
}
return currentObject;
}
/**
* get the maps
*
* @return List<String> list of name of maps
*/
public List<String> getMapsList(String userName) {
List<String> maps = new ArrayList<String>();
ListObjectsRequest listObjectsRequest =
new ListObjectsRequest()
.withBucketName(bucketName)
.withPrefix("maps/public/")
.withDelimiter("/");
ObjectListing objects = s3.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
if (!objectSummary.getKey().equals("maps/public")) {
System.out.println("get map: " + objectSummary.getKey());
maps.add(objectSummary.getKey());
}
}
ListObjectsRequest listObjectsRequest2 =
new ListObjectsRequest()
.withBucketName(bucketName)
.withPrefix("maps/" + userName + "/")
.withDelimiter("/");
ObjectListing objects2 = s3.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
if (!objectSummary.getKey().equals("maps/" + userName)) {
System.out.println("get map: " + objectSummary.getKey());
maps.add(objectSummary.getKey());
}
}
return maps;
}
@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
List<NoteInfo> infos = new LinkedList<>();
NoteInfo info;
try {
ListObjectsRequest listObjectsRequest =
new ListObjectsRequest().withBucketName(bucketName).withPrefix(user + "/" + "notebook");
ObjectListing objectListing;
do {
objectListing = s3client.listObjects(listObjectsRequest);
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
if (objectSummary.getKey().endsWith("note.json")) {
info = getNoteInfo(objectSummary.getKey());
if (info != null) {
infos.add(info);
}
}
}
listObjectsRequest.setMarker(objectListing.getNextMarker());
} while (objectListing.isTruncated());
} catch (AmazonClientException ace) {
throw new IOException("Unable to list objects in S3: " + ace, ace);
}
return infos;
}
@Override
public boolean exists(String key, boolean isFile) throws IOException {
if (isFile) {
Preconditions.checkArgument(
key.charAt(key.length() - 1) != '/', "File should not end in %s: %s", '/', key);
} else {
Preconditions.checkArgument(
key.charAt(key.length() - 1) == '/', "Dir should end in %s: %s", '/', key);
}
ObjectListing listing;
try {
listing = s3Client.listObjects(Namespaces.get().getBucket(), key);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
if (listing != null) {
for (S3ObjectSummary s3os : listing.getObjectSummaries()) {
String object = s3os.getKey();
if (object.equals(key) || (!isFile && object.startsWith(key))) {
// Counts if the keys match exactly, or, the match continues with a subdir path in which
// case this must be a directory
return true;
}
}
}
return false;
}
public BucketIterator(String bucket, S3Downloader s3) {
this.bucket = bucket;
if (s3 == null) s3 = new S3Downloader();
currList = s3.listObjects(bucket);
currObjects = currList.getObjectSummaries();
}
public LinkedList<String> listNames(String prefix, String bucketName, String[] canonicalIDs)
throws StorageCloudException {
LinkedList<String> find = new LinkedList<String>();
try {
ObjectListing objectListing = null;
if (bucketName == null)
objectListing =
conn.listObjects(
new ListObjectsRequest().withBucketName(defaultBucketName).withPrefix(prefix));
else {
bucketName = bucketName.concat(location);
objectListing =
conn.listObjects(
new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix));
}
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
find.add(objectSummary.getKey());
}
} catch (AmazonServiceException e1) {
throw new ServiceSiteException("AWSS3Exception::" + e1.getMessage());
} catch (AmazonClientException e2) {
throw new ClientServiceException("AWSS3Exception::" + e2.getMessage());
}
return find;
}
public String[] setAcl(String bucketNameToShare, String[] canonicalId, String permission)
throws StorageCloudException {
boolean withRead = false;
if (bucketNameToShare != null) {
bucketNameToShare = bucketNameToShare.concat(location);
if (!conn.doesBucketExist(bucketNameToShare)) {
conn.createBucket(bucketNameToShare, region);
}
} else {
return null;
}
// set acl
AccessControlList acl = conn.getBucketAcl(bucketNameToShare);
for (int i = 0; i < canonicalId.length; i++) {
if (permission.equals("rw")) {
CanonicalGrantee grantee = new CanonicalGrantee(canonicalId[i]);
acl.grantPermission(grantee, Permission.Read);
acl.grantPermission(grantee, Permission.Write);
withRead = true;
} else if (permission.equals("r")) {
acl.grantPermission(new CanonicalGrantee(canonicalId[i]), Permission.Read);
withRead = true;
} else if (permission.equals("w")) {
acl.grantPermission(new CanonicalGrantee(canonicalId[i]), Permission.Write);
}
}
try {
if (withRead) {
ObjectListing objectListing = conn.listObjects(bucketNameToShare);
AccessControlList aclKeys = null;
for (S3ObjectSummary elem : objectListing.getObjectSummaries()) {
aclKeys = conn.getObjectAcl(bucketNameToShare, elem.getKey());
for (int i = 0; i < canonicalId.length; i++) {
aclKeys.grantPermission(new CanonicalGrantee(canonicalId[i]), Permission.Read);
}
conn.setObjectAcl(bucketNameToShare, elem.getKey(), aclKeys);
}
}
// confirm if acl well
conn.setBucketAcl(bucketNameToShare, acl);
AccessControlList newAcl = conn.getBucketAcl(bucketNameToShare);
Set<Grant> grants = newAcl.getGrants();
boolean flag = false;
for (Grant grant : grants) {
if (grant.getGrantee().getIdentifier().equals(canonicalId[0])) {
flag = true;
}
}
if (!flag) {
throw new ServiceSiteException("AWSS3Exception:: ACL");
}
} catch (AmazonServiceException e1) {
throw new ServiceSiteException("AWSS3Exception::" + e1.getMessage());
} catch (AmazonClientException e2) {
throw new ClientServiceException("AWSS3Exception::" + e2.getMessage());
}
return canonicalId;
}
private List<S3ObjectSummary> nestedResources(String resourceName) {
ObjectListing listing = S3.client().listObjects(S3.bucket(), resourceName);
List<S3ObjectSummary> resources = listing.getObjectSummaries();
// exclude the own resource
if (resources.size() > 0) resources = resources.subList(1, resources.size());
return resources;
}
public void DownloadUsingS3() throws Exception {
while (true) {
ObjectListing list =
s3api.ReadBucket(S3_ACCESS_KEY_ID, S3_SECRET_KEY, S3_ENDPOINT, null, S3_BUCKET);
System.out.println("bucket files count " + list.getObjectSummaries().size());
vLogger.LogInfo("emcWorldDownloader: bucket files count " + list.getObjectSummaries().size());
int s3Count = list.getObjectSummaries().size();
int localCount = new File(LOCAL_DIR).listFiles().length;
if (s3Count != localCount) {
for (S3ObjectSummary obj : list.getObjectSummaries()) {
if (!(new File(LOCAL_DIR, obj.getKey()).exists())) {
System.out.println("Downloading - " + obj.getKey());
vLogger.LogInfo("emcWorldDownloader: Downloading - " + obj.getKey());
S3ObjectInputStream in =
s3api.ReadObject(
S3_ACCESS_KEY_ID, S3_SECRET_KEY, S3_ENDPOINT, null, S3_BUCKET, obj.getKey());
File file = new File(LOCAL_DIR + obj.getKey());
int count = 0;
byte[] buf = new byte[1024];
OutputStream out = new FileOutputStream(file);
while ((count = in.read(buf)) != -1) {
if (Thread.interrupted()) {
throw new InterruptedException();
}
out.write(buf, 0, count);
}
out.close();
in.close();
} else {
System.out.println("Skipping - " + obj.getKey());
vLogger.LogInfo("emcWorldDownloader: Skipping - " + obj.getKey());
}
}
} else System.out.println("No New Files Yet");
vLogger.LogInfo("emcWorldDownloader: Skipping - Sleep 60 seconds");
System.out.println("Sleep for 1 minute");
Thread.sleep(60000);
}
}
public static List<String> listBucket(String bucketName, String prefix) {
List<String> ret = new ArrayList<String>();
ObjectListing listing =
getInstance()
.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix));
for (S3ObjectSummary s : listing.getObjectSummaries()) {
StringBuilder sb = new StringBuilder();
sb.append("s3://").append(bucketName).append("/").append(s.getKey());
ret.add(sb.toString());
}
while (listing.isTruncated()) {
listing = getInstance().listNextBatchOfObjects(listing);
for (S3ObjectSummary s : listing.getObjectSummaries()) {
StringBuilder sb = new StringBuilder();
sb.append("s3://").append(bucketName).append("/").append(s.getKey());
ret.add(sb.toString());
}
}
return ret;
}
/**
* Expect request with start marker and provide in response single item, notify that marker is a
* next marker to request.
*
* @param item Item to respond with
* @param start Start marker to expect
* @param marker Next marker
* @return This instance
*/
public RegionExpectations expectResponse(
final String item, final String start, final String marker) {
final ObjectListing response = new ObjectListing();
response.setNextMarker(marker);
final S3ObjectSummary summary = new S3ObjectSummary();
summary.setKey(item);
response.getObjectSummaries().add(summary);
this.responses.add(response);
this.markers.add(start);
return this;
}
private List<String> getResourceNames(ObjectListing objectListing, Pattern pattern) {
List<String> resourceNames = new ArrayList<String>();
for (String commonPrefix : objectListing.getCommonPrefixes()) {
resourceNames.add(getResourceName(commonPrefix, pattern));
}
for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) {
resourceNames.add(getResourceName(s3ObjectSummary.getKey(), pattern));
}
return resourceNames;
}
@Override
public List<BackupMetaData> getAvailableBackups(
Exhibitor exhibitor, Map<String, String> configValues) throws Exception {
ListObjectsRequest request = new ListObjectsRequest();
request.setBucketName(configValues.get(CONFIG_BUCKET.getKey()));
ObjectListing listing = s3Client.listObjects(request);
return Lists.transform(
listing.getObjectSummaries(),
new Function<S3ObjectSummary, BackupMetaData>() {
@Override
public BackupMetaData apply(S3ObjectSummary summary) {
return fromKey(summary.getKey());
}
});
}
@Override
protected int poll() throws Exception {
// must reset for each poll
shutdownRunningTask = null;
pendingExchanges = 0;
String fileName = getConfiguration().getFileName();
String bucketName = getConfiguration().getBucketName();
Queue<Exchange> exchanges = null;
if (fileName != null) {
LOG.trace("Getting object in bucket [{}] with file name [{}]...", bucketName, fileName);
S3Object s3Object = getAmazonS3Client().getObject(new GetObjectRequest(bucketName, fileName));
exchanges = createExchanges(s3Object);
} else {
LOG.trace("Queueing objects in bucket [{}]...", bucketName);
ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
listObjectsRequest.setBucketName(bucketName);
listObjectsRequest.setPrefix(getConfiguration().getPrefix());
listObjectsRequest.setMaxKeys(maxMessagesPerPoll);
ObjectListing listObjects = getAmazonS3Client().listObjects(listObjectsRequest);
if (LOG.isTraceEnabled()) {
LOG.trace(
"Found {} objects in bucket [{}]...",
listObjects.getObjectSummaries().size(),
bucketName);
}
exchanges = createExchanges(listObjects.getObjectSummaries());
}
return processBatch(CastUtils.cast(exchanges));
}
// 列出某个bucket的所有objects
public static void listObjectInBucket(Bucket bucket) {
List<Bucket> buckets = conn.listBuckets();
ObjectListing objects = conn.listObjects(bucket.getName());
do {
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
System.out.println(
objectSummary.getKey()
+ "\t"
+ objectSummary.getSize()
+ "\t"
+ StringUtils.fromDate(objectSummary.getLastModified()));
}
objects = conn.listNextBatchOfObjects(objects);
} while (objects.isTruncated());
}
/**
* Tests the S3 access.
*
* @throws Exception if test fails
*/
@Test
public void s3Test() throws Exception {
// Create a real object and mock its initClient method
final S3NutDao dao = spy(new S3NutDao("/path", false, null, -1, "wuic", "login", "pwd", false));
// Build client mock
final AmazonS3Client client = mock(AmazonS3Client.class);
when(dao.initClient()).thenReturn(client);
// List returned by client
final ObjectListing list = mock(ObjectListing.class);
final S3ObjectSummary summary = mock(S3ObjectSummary.class);
when(summary.getKey()).thenReturn("[cloud].css");
final S3ObjectSummary summarBis = mock(S3ObjectSummary.class);
when(summarBis.getKey()).thenReturn("cloud.css");
when(client.listObjects(any(ListObjectsRequest.class))).thenReturn(list);
when(list.getObjectSummaries()).thenReturn(Arrays.asList(summary, summarBis));
// Bytes returned by mocked S3
final byte[] array = ".cloud { text-align : justify;}".getBytes();
final S3Object object = mock(S3Object.class);
when(object.getObjectContent())
.thenReturn(new S3ObjectInputStream(new ByteArrayInputStream(array), null));
when(client.getObject(anyString(), anyString())).thenReturn(object);
// TODO : problem here : we specify '[cloud.css]' but getNuts() returns 'cloud.css' because
// regex are always activated !
final NutsHeap nutsHeap = new NutsHeap(Arrays.asList("[cloud].css"), dao, "heap");
Assert.assertEquals(nutsHeap.getNuts().size(), 1);
final Engine compressor = new CssYuiCompressorEngine(true, "UTF-8", -1);
final Engine cacheEngine = new EhCacheEngine(false, null);
final Engine aggregator = new CGTextAggregatorEngine(true);
cacheEngine.setNext(compressor);
compressor.setNext(aggregator);
final List<Nut> group =
cacheEngine.parse(new EngineRequest("", "", nutsHeap, new HashMap<NutType, Engine>()));
Assert.assertFalse(group.isEmpty());
InputStream is;
for (Nut res : group) {
is = res.openStream();
Assert.assertTrue(IOUtils.readString(new InputStreamReader(is)).length() > 0);
is.close();
}
}
// 列出有prefix前缀的所有objects
public static void listObjectInBucketWithPrefix(Bucket bucket, String prefix) {
ListObjectsRequest listObjectsRequest =
new ListObjectsRequest().withBucketName(bucket.getName()).withPrefix(prefix);
ObjectListing objects = conn.listObjects(listObjectsRequest);
do {
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
System.out.println(
objectSummary.getKey()
+ "\t"
+ objectSummary.getSize()
+ "\t"
+ StringUtils.fromDate(objectSummary.getLastModified()));
}
objects = conn.listNextBatchOfObjects(objects);
} while (objects.isTruncated());
}
public ArrayList<String> getObjectList() {
s3 =
new AmazonS3Client(
new BasicAWSCredentials(
"AKIAIL5P46R7BWHUK4JA", "YvBz0m95efCfDGRe/uOEwWz/bd09RRZ5ROYF4QY3"));
objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName));
ArrayList<String> nodeList = new ArrayList<String>();
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
nodeList.add(objectSummary.getKey());
}
return nodeList;
}
public static void main(final String[] args) {
DesktopSetup.deploy();
final RelativePath target = JUtils.newRelativePath();
final AmazonS3 s3 = new AmazonS3Client(new ProfileCredentialsProvider());
final ListObjectsRequest request = new ListObjectsRequest().withBucketName("jfix.by");
final String prefix = "/";
request.withPrefix("wp-content");
request.setDelimiter("");
final ObjectListing objectListing = s3.listObjects(request);
final List<String> prefixes = Collections.newList(objectListing.getCommonPrefixes());
prefixes.print("prefixes");
final List<S3ObjectSummary> summs = Collections.newList(objectListing.getObjectSummaries());
}
public void processListing(ObjectListing listing, JsonArray succ, JsonArray fail) {
for (S3ObjectSummary obj : listing.getObjectSummaries()) {
try {
Key k = PersistS3.loadKey(obj);
JsonObject o = new JsonObject();
o.addProperty(KEY, k.toString());
o.addProperty(FILE, obj.getKey());
o.addProperty(VALUE_SIZE, obj.getSize());
succ.add(o);
} catch (IOException e) {
JsonObject o = new JsonObject();
o.addProperty(FILE, obj.getKey());
o.addProperty(ERROR, e.getMessage());
fail.add(o);
}
}
}
@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
String key = user + "/" + "notebook" + "/" + noteId;
final ListObjectsRequest listObjectsRequest =
new ListObjectsRequest().withBucketName(bucketName).withPrefix(key);
try {
ObjectListing objects = s3client.listObjects(listObjectsRequest);
do {
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
s3client.deleteObject(bucketName, objectSummary.getKey());
}
objects = s3client.listNextBatchOfObjects(objects);
} while (objects.isTruncated());
} catch (AmazonClientException ace) {
throw new IOException("Unable to remove note in S3: " + ace, ace);
}
}
@Override
public void recursiveDelete(String keyPrefix, FilenameFilter except) throws IOException {
if (keyPrefix.charAt(keyPrefix.length() - 1) == '/') {
// Need to delete the dir too, and to list it, can't specify its trailing slash
keyPrefix = keyPrefix.substring(0, keyPrefix.length() - 1);
}
boolean truncated = true;
String marker = null;
String bucket = Namespaces.get().getBucket();
while (truncated) {
ListObjectsRequest listRequest =
new ListObjectsRequest(bucket, keyPrefix, marker, null, null);
ObjectListing listing;
try {
listing = s3Client.listObjects(listRequest);
Collection<S3ObjectSummary> summaries = listing.getObjectSummaries();
if (summaries.isEmpty()) {
return;
}
List<DeleteObjectsRequest.KeyVersion> keysToDelete =
Lists.newArrayListWithCapacity(summaries.size());
for (S3ObjectSummary summary : summaries) {
String key = summary.getKey();
if (except == null || !except.accept(null, key)) {
keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key));
}
}
DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket);
deleteObjectsRequest.setKeys(keysToDelete);
s3Client.deleteObjects(deleteObjectsRequest);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
truncated = listing.isTruncated();
marker = listing.getNextMarker();
}
}
private void scanBucket(Set<TocInfo> toc, Queue<TocInfo> tocQueue) throws Exception {
ListObjectsRequest listRequest = new ListObjectsRequest();
listRequest.setBucketName(s3BucketName);
// listRequest.setGeneralProgressListener(this);
listRequest.setMaxKeys(1000);
String nextMarker = null;
ObjectListing objectListing = null;
while (true) {
objectListing = s3Client.listObjects(listRequest);
List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
for (S3ObjectSummary objSummary : objectSummaries) {
String key = objSummary.getKey();
TocInfo tocInfo = new TocInfo(key, objSummary.getSize());
// is it a "dir/" ?
if (key.lastIndexOf("/") == (key.length() - 1)) {
tocInfo.isDirectory = true;
} else {
tocInfo.isDirectory = false;
}
toc.add(tocInfo);
tocQueue.add(tocInfo);
tocInfosGenerated++; // increment for logging
}
// for pagination
nextMarker = objectListing.getNextMarker();
if (nextMarker == null) {
break;
} else {
listRequest.setMarker(nextMarker);
logger.debug("scanBucket() nextMarker we will request listing for => " + nextMarker);
}
}
}
@Override
public InputStream next() {
if (currObject < currObjects.size()) {
InputStream ret = s3.objectForKey(bucket, currObjects.get(currObject).getKey());
currObject++;
return ret;
} else if (currList.isTruncated()) {
currList = s3.nextList(currList);
currObjects = currList.getObjectSummaries();
currObject = 0;
InputStream ret = s3.objectForKey(bucket, currObjects.get(currObject).getKey());
currObject++;
return ret;
}
return null;
}
@Override
public List<String> list(String prefix, boolean files) throws IOException {
boolean truncated = true;
String marker = null;
List<String> result = Lists.newArrayList();
while (truncated) {
if (marker != null) {
log.info("Next page after {}", marker);
}
ListObjectsRequest listRequest =
new ListObjectsRequest(Namespaces.get().getBucket(), prefix, marker, "/", null);
ObjectListing listing;
try {
listing = s3Client.listObjects(listRequest);
} catch (AmazonClientException ace) {
throw new IOException(ace);
}
truncated = listing.isTruncated();
marker = listing.getNextMarker();
if (!truncated) {
if (files) {
for (S3ObjectSummary summary : listing.getObjectSummaries()) {
String key = summary.getKey();
if (!key.endsWith("_SUCCESS")) {
result.add(key);
}
}
} else {
for (String key : listing.getCommonPrefixes()) {
if (!key.endsWith("_SUCCESS")) {
result.add(key);
}
}
}
}
}
Collections.sort(result);
return result;
}
public static void main(String[] args) {
if (args.length < 2) {
System.out.println("USAGE: localPath bucketname <bucketPrefix>");
System.exit(1);
}
String localPath = args[0];
String bucketName = args[1];
String bucketPrefix = "";
// add on extra slash
if (!localPath.endsWith("/")) localPath += "/";
if (args.length == 3) bucketPrefix = args[2];
// check local dir, make if it does not exist
File localDir = new File(localPath);
if (!localDir.exists()) localDir.mkdirs();
if (!localDir.isDirectory()) {
System.out.println("Local Dir is not a dir: " + localPath);
System.exit(1);
}
long totalBytes = 0;
long start = System.currentTimeMillis();
AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());
ObjectListing listObjects = s3.listObjects(bucketName, bucketPrefix);
do {
for (S3ObjectSummary objectSummary : listObjects.getObjectSummaries()) {
S3Object object = s3.getObject(bucketName, objectSummary.getKey());
S3ObjectInputStream inputStream = object.getObjectContent();
if ("gzip".equals(object.getObjectMetadata().getContentEncoding())) {
InputStream in = null;
try {
totalBytes += object.getObjectMetadata().getContentLength();
in = new GZIPInputStream(inputStream);
// write this sucker out
String path = localPath + object.getKey();
// have to take the gz off since this is not downloading compressed!
if (path.endsWith(".gz")) path = path.substring(0, path.length() - 3);
System.out.print("Writing file: " + path);
File check = new File(path);
File parentFile = check.getParentFile();
if (!parentFile.exists()) parentFile.mkdirs();
FileOutputStream out = new FileOutputStream(path);
IOUtils.copy(in, out);
System.out.println(" written.");
} catch (IOException e) {
System.out.println("crap");
e.printStackTrace();
throw new IllegalStateException("files are too hard", e);
} finally {
IOUtils.closeQuietly(in);
}
} else {
System.out.println(
"unhandled content encoding: " + object.getObjectMetadata().getContentEncoding());
}
}
listObjects = s3.listNextBatchOfObjects(listObjects);
} while (listObjects.isTruncated());
long now = System.currentTimeMillis();
System.out.println(
(totalBytes / 1000.0 / 1000.0)
+ " mb downloaded in "
+ ((now - start) / 1000)
+ " seconds.");
}
/** Deletes content of the repository files in the bucket */
public void cleanRepositoryFiles(String basePath) {
Settings settings = internalCluster().getInstance(Settings.class);
Settings[] buckets = {
settings.getByPrefix("repositories.s3."),
settings.getByPrefix("repositories.s3.private-bucket."),
settings.getByPrefix("repositories.s3.remote-bucket."),
settings.getByPrefix("repositories.s3.external-bucket.")
};
for (Settings bucket : buckets) {
String endpoint = bucket.get("endpoint", settings.get("repositories.s3.endpoint"));
String protocol = bucket.get("protocol", settings.get("repositories.s3.protocol"));
String region = bucket.get("region", settings.get("repositories.s3.region"));
String accessKey = bucket.get("access_key", settings.get("cloud.aws.access_key"));
String secretKey = bucket.get("secret_key", settings.get("cloud.aws.secret_key"));
String bucketName = bucket.get("bucket");
// We check that settings has been set in elasticsearch.yml integration test file
// as described in README
assertThat(
"Your settings in elasticsearch.yml are incorrects. Check README file.",
bucketName,
notNullValue());
AmazonS3 client =
internalCluster()
.getInstance(AwsS3Service.class)
.client(endpoint, protocol, region, accessKey, secretKey);
try {
ObjectListing prevListing = null;
// From
// http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
// we can do at most 1K objects per delete
// We don't know the bucket name until first object listing
DeleteObjectsRequest multiObjectDeleteRequest = null;
ArrayList<DeleteObjectsRequest.KeyVersion> keys =
new ArrayList<DeleteObjectsRequest.KeyVersion>();
while (true) {
ObjectListing list;
if (prevListing != null) {
list = client.listNextBatchOfObjects(prevListing);
} else {
list = client.listObjects(bucketName, basePath);
multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
}
for (S3ObjectSummary summary : list.getObjectSummaries()) {
keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
// Every 500 objects batch the delete request
if (keys.size() > 500) {
multiObjectDeleteRequest.setKeys(keys);
client.deleteObjects(multiObjectDeleteRequest);
multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
keys.clear();
}
}
if (list.isTruncated()) {
prevListing = list;
} else {
break;
}
}
if (!keys.isEmpty()) {
multiObjectDeleteRequest.setKeys(keys);
client.deleteObjects(multiObjectDeleteRequest);
}
} catch (Throwable ex) {
logger.warn("Failed to delete S3 repository [{}] in [{}]", ex, bucketName, region);
}
}
}
public static void main(String[] args) throws IOException {
/*
* This credentials provider implementation loads your AWS credentials
* from a properties file at the root of your classpath.
*
* Important: Be sure to fill in your AWS access credentials in the
* AwsCredentials.properties file before you try to run this
* sample.
* http://aws.amazon.com/security-credentials
*/
AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());
String bucketName = "my-first-s3-bucket-" + UUID.randomUUID();
String key = "MyObjectKey";
System.out.println("===========================================");
System.out.println("Getting Started with Amazon S3");
System.out.println("===========================================\n");
try {
/*
* Create a new S3 bucket - Amazon S3 bucket names are globally unique,
* so once a bucket name has been taken by any user, you can't create
* another bucket with that same name.
*
* You can optionally specify a location for your bucket if you want to
* keep your data closer to your applications or users.
*/
System.out.println("Creating bucket " + bucketName + "\n");
s3.createBucket(bucketName);
/*
* List the buckets in your account
*/
System.out.println("Listing buckets");
for (Bucket bucket : s3.listBuckets()) {
System.out.println(" - " + bucket.getName());
}
System.out.println();
/*
* Upload an object to your bucket - You can easily upload a file to
* S3, or upload directly an InputStream if you know the length of
* the data in the stream. You can also specify your own metadata
* when uploading to S3, which allows you set a variety of options
* like content-type and content-encoding, plus additional metadata
* specific to your applications.
*/
System.out.println("Uploading a new object to S3 from a file\n");
s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile()));
/*
* Download an object - When you download an object, you get all of
* the object's metadata and a stream from which to read the contents.
* It's important to read the contents of the stream as quickly as
* possibly since the data is streamed directly from Amazon S3 and your
* network connection will remain open until you read all the data or
* close the input stream.
*
* GetObjectRequest also supports several other options, including
* conditional downloading of objects based on modification times,
* ETags, and selectively downloading a range of an object.
*/
System.out.println("Downloading an object");
S3Object object = s3.getObject(new GetObjectRequest(bucketName, key));
System.out.println("Content-Type: " + object.getObjectMetadata().getContentType());
displayTextInputStream(object.getObjectContent());
/*
* List objects in your bucket by prefix - There are many options for
* listing the objects in your bucket. Keep in mind that buckets with
* many objects might truncate their results when listing their objects,
* so be sure to check if the returned object listing is truncated, and
* use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve
* additional results.
*/
System.out.println("Listing objects");
ObjectListing objectListing =
s3.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My"));
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
System.out.println(
" - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")");
}
System.out.println();
/*
* Delete an object - Unless versioning has been turned on for your bucket,
* there is no way to undelete an object, so use caution when deleting objects.
*/
System.out.println("Deleting an object\n");
s3.deleteObject(bucketName, key);
/*
* Delete a bucket - A bucket must be completely empty before it can be
* deleted, so remember to delete any objects from your buckets before
* you try to delete them.
*/
System.out.println("Deleting bucket " + bucketName + "\n");
s3.deleteBucket(bucketName);
} catch (AmazonServiceException ase) {
System.out.println(
"Caught an AmazonServiceException, which means your request made it "
+ "to Amazon S3, but was rejected with an error response for some reason.");
System.out.println("Error Message: " + ase.getMessage());
System.out.println("HTTP Status Code: " + ase.getStatusCode());
System.out.println("AWS Error Code: " + ase.getErrorCode());
System.out.println("Error Type: " + ase.getErrorType());
System.out.println("Request ID: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println(
"Caught an AmazonClientException, which means the client encountered "
+ "a serious internal problem while trying to communicate with S3, "
+ "such as not being able to access the network.");
System.out.println("Error Message: " + ace.getMessage());
}
}
private Iterator<LocatedFileStatus> statusFromListing(ObjectListing listing) {
return Iterators.concat(
statusFromPrefixes(listing.getCommonPrefixes()),
statusFromObjects(listing.getObjectSummaries()));
}