public boolean isAmazonS3ErrorCode(String errorCode) {
if (getCause() instanceof AmazonS3Exception) {
AmazonS3Exception s3Exception = (AmazonS3Exception) getCause();
return errorCode.equalsIgnoreCase(s3Exception.getErrorCode());
}
return false;
}
public UrlList getUrlList(String url) {
UrlList urlList = null;
try {
String key = Hash.hashKey(url);
GetObjectRequest req = new GetObjectRequest(bucketName, key);
S3Object s3Object = s3client.getObject(req);
InputStream objectData = s3Object.getObjectContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(objectData));
StringBuilder s3Content = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
s3Content.append(line + "\r\n");
}
reader.close();
objectData.close();
ObjectMapper mapper = new ObjectMapper();
mapper.setVisibility(PropertyAccessor.ALL, Visibility.NONE);
mapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY);
urlList = mapper.readValue(s3Content.toString(), UrlList.class);
} catch (AmazonS3Exception ase) {
System.out.println("S3UrlListDA : document does not exist");
ase.printStackTrace();
} catch (IOException e) {
System.out.println("S3UrlListDA : IOException while fetching document from S3");
e.printStackTrace();
} catch (NoSuchAlgorithmException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return urlList;
}
private ObjectMetadata getS3ObjectMetadata(final Path path) throws IOException {
try {
return retry()
.maxAttempts(maxClientRetries)
.exponentialBackoff(new Duration(1, TimeUnit.SECONDS), maxBackoffTime, maxRetryTime, 2.0)
.stopOn(InterruptedException.class, UnrecoverableS3OperationException.class)
.run(
"getS3ObjectMetadata",
() -> {
try {
return s3.getObjectMetadata(uri.getHost(), keyFromPath(path));
} catch (AmazonS3Exception e) {
if (e.getStatusCode() == SC_NOT_FOUND) {
return null;
} else if (e.getStatusCode() == SC_FORBIDDEN) {
throw new UnrecoverableS3OperationException(e);
}
throw Throwables.propagate(e);
}
});
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw Throwables.propagate(e);
} catch (Exception e) {
Throwables.propagateIfInstanceOf(e, IOException.class);
throw Throwables.propagate(e);
}
}
@Override
public void deleteObject(String container, String object, Config config) {
super.deleteObject(container, object, config);
try {
client.deleteObject(container, object);
} catch (AmazonS3Exception awse) {
if (awse.getStatusCode() != HttpStatus.SC_NOT_FOUND) {
throw new StorageException(awse);
}
} catch (Exception e) {
throw new StorageException(e);
}
}
@Override
public void deleteContainer(String container, Config config) {
super.deleteContainer(container, config);
try {
if (client.doesBucketExist(container)) {
client.deleteBucket(container);
}
} catch (AmazonS3Exception awse) {
if (awse.getStatusCode() != HttpStatus.SC_NOT_FOUND) {
throw new StorageException(awse);
}
} catch (Exception e) {
throw new StorageException(e);
}
}
private void validateConnection(Stage.Context context, List<Stage.ConfigIssue> issues) {
// Access Key ID - username [unique in aws]
// secret access key - password
AWSCredentials credentials = new BasicAWSCredentials(accessKeyId, secretAccessKey);
s3Client = new AmazonS3Client(credentials, new ClientConfiguration());
s3Client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
if (endPoint != null && !endPoint.isEmpty()) {
s3Client.setEndpoint(endPoint);
} else {
s3Client.setRegion(Region.getRegion(region));
}
try {
// check if the credentials are right by trying to list buckets
s3Client.listBuckets();
} catch (AmazonS3Exception e) {
issues.add(
context.createConfigIssue(
Groups.S3.name(), "accessKeyId", Errors.S3_SPOOLDIR_20, e.toString()));
}
}
private boolean isNotFoundError(AmazonS3Exception e) {
return (e.getStatusCode() == 404) || (e.getStatusCode() == 403);
}