private S3Object getS3Object(final Path path, final long start) throws IOException {
try {
return retry()
.maxAttempts(maxClientRetry)
.exponentialBackoff(
new Duration(1, TimeUnit.SECONDS), maxBackoffTime, maxRetryTime, 2.0)
.stopOn(InterruptedException.class, UnrecoverableS3OperationException.class)
.run(
"getS3Object",
() -> {
try {
return s3.getObject(
new GetObjectRequest(host, keyFromPath(path))
.withRange(start, Long.MAX_VALUE));
} catch (AmazonServiceException e) {
if (e.getStatusCode() == SC_FORBIDDEN) {
throw new UnrecoverableS3OperationException(e);
}
throw Throwables.propagate(e);
}
});
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw Throwables.propagate(e);
} catch (Exception e) {
Throwables.propagateIfInstanceOf(e, IOException.class);
throw Throwables.propagate(e);
}
}
private static void createKey(String keyName, AmazonEC2 ec2) {
try {
List<KeyPairInfo> keyPairList = ec2.describeKeyPairs().getKeyPairs();
for (KeyPairInfo keyPair : keyPairList) {
if (keyName.equalsIgnoreCase(keyPair.getKeyName())) {
System.out.println("Using key " + keyName);
return;
}
}
System.out.println("Creating key " + keyName + "in local directory");
CreateKeyPairRequest newKeyRequest = new CreateKeyPairRequest();
newKeyRequest.setKeyName(keyName);
CreateKeyPairResult keyresult = ec2.createKeyPair(newKeyRequest);
KeyPair keyPair = new KeyPair();
keyPair = keyresult.getKeyPair();
String privateKey = keyPair.getKeyMaterial();
writeKeytoFile(keyName, privateKey);
} catch (AmazonServiceException ase) {
System.out.println("Caught Exception: " + ase.getMessage());
System.out.println("Reponse Status Code: " + ase.getStatusCode());
System.out.println("Error Code: " + ase.getErrorCode());
System.out.println("Request ID: " + ase.getRequestId());
}
}
public static void createBucket(AmazonS3Client s3, String bucketName, String zone) {
try {
List<Bucket> bucketList = s3.listBuckets();
for (Bucket bucket : bucketList) {
// System.out.println(bucket.getName());
if (bucketName.equalsIgnoreCase(bucket.getName())) {
System.out.println("Using bucket " + bucketName);
return;
}
}
System.out.println("Created s3 bucket " + bucketName);
s3.createBucket(bucketName);
return;
} catch (AmazonServiceException ase) {
System.out.println("Caught Exception: " + ase.getMessage());
System.out.println("Reponse Status Code: " + ase.getStatusCode());
System.out.println("Error Code: " + ase.getErrorCode());
System.out.println("Request ID: " + ase.getRequestId());
}
}
private void updateItemVersionedInternal(
String tableName,
Key key,
Map<String, AttributeValueUpdate> attributes,
String expectedVersion,
PersistentEntity persistentEntity,
int attempt)
throws DataAccessException {
UpdateItemRequest request =
new UpdateItemRequest(tableName, key, attributes)
.withExpected(getOptimisticVersionCondition(expectedVersion));
try {
ddb.updateItem(request);
} catch (AmazonServiceException e) {
if (DynamoDBUtil.AWS_ERR_CODE_CONDITIONAL_CHECK_FAILED.equals(e.getErrorCode())) {
throw new OptimisticLockingException(persistentEntity, key);
} else if (DynamoDBUtil.AWS_ERR_CODE_RESOURCE_NOT_FOUND.equals(e.getErrorCode())) {
throw new IllegalArgumentException("no such table: " + tableName, e);
} else if (DynamoDBUtil.AWS_STATUS_CODE_SERVICE_UNAVAILABLE == e.getStatusCode()) {
// retry after a small pause
DynamoDBUtil.sleepBeforeRetry(attempt);
attempt++;
updateItemVersionedInternal(
tableName, key, attributes, expectedVersion, persistentEntity, attempt);
} else {
throw new DataStoreOperationException(
"problem with table: " + tableName + ", key: " + key + ", attributes: " + attributes,
e);
}
}
}
public LinkedList<String> listNames(String prefix, String bucketName, String[] canonicalIDs)
throws StorageCloudException {
LinkedList<String> find = new LinkedList<String>();
try {
ObjectListing objectListing = null;
if (bucketName == null)
objectListing =
conn.listObjects(
new ListObjectsRequest().withBucketName(defaultBucketName).withPrefix(prefix));
else {
bucketName = bucketName.concat(location);
objectListing =
conn.listObjects(
new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix));
}
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
find.add(objectSummary.getKey());
}
} catch (AmazonServiceException e1) {
throw new ServiceSiteException("AWSS3Exception::" + e1.getMessage());
} catch (AmazonClientException e2) {
throw new ClientServiceException("AWSS3Exception::" + e2.getMessage());
}
return find;
}
private Map<String, AttributeValue> getConsistentInternal(
String tableName, Key key, int attempt) {
GetItemRequest request = new GetItemRequest(tableName, key);
request.setConsistentRead(true);
try {
GetItemResult result = ddb.getItem(request);
Map<String, AttributeValue> attributes = result.getItem();
if (attributes == null || attributes.isEmpty()) {
return null;
}
return attributes;
} catch (AmazonServiceException e) {
if (DynamoDBUtil.AWS_ERR_CODE_RESOURCE_NOT_FOUND.equals(e.getErrorCode())) {
throw new IllegalArgumentException("no such table: " + tableName, e);
} else if (DynamoDBUtil.AWS_STATUS_CODE_SERVICE_UNAVAILABLE == e.getStatusCode()) {
// retry after a small pause
DynamoDBUtil.sleepBeforeRetry(attempt);
attempt++;
return getConsistentInternal(tableName, key, attempt);
} else {
throw new DataStoreOperationException(
"problem with table: " + tableName + ", key: " + key, e);
}
}
}
public String[] setAcl(String bucketNameToShare, String[] canonicalId, String permission)
throws StorageCloudException {
boolean withRead = false;
if (bucketNameToShare != null) {
bucketNameToShare = bucketNameToShare.concat(location);
if (!conn.doesBucketExist(bucketNameToShare)) {
conn.createBucket(bucketNameToShare, region);
}
} else {
return null;
}
// set acl
AccessControlList acl = conn.getBucketAcl(bucketNameToShare);
for (int i = 0; i < canonicalId.length; i++) {
if (permission.equals("rw")) {
CanonicalGrantee grantee = new CanonicalGrantee(canonicalId[i]);
acl.grantPermission(grantee, Permission.Read);
acl.grantPermission(grantee, Permission.Write);
withRead = true;
} else if (permission.equals("r")) {
acl.grantPermission(new CanonicalGrantee(canonicalId[i]), Permission.Read);
withRead = true;
} else if (permission.equals("w")) {
acl.grantPermission(new CanonicalGrantee(canonicalId[i]), Permission.Write);
}
}
try {
if (withRead) {
ObjectListing objectListing = conn.listObjects(bucketNameToShare);
AccessControlList aclKeys = null;
for (S3ObjectSummary elem : objectListing.getObjectSummaries()) {
aclKeys = conn.getObjectAcl(bucketNameToShare, elem.getKey());
for (int i = 0; i < canonicalId.length; i++) {
aclKeys.grantPermission(new CanonicalGrantee(canonicalId[i]), Permission.Read);
}
conn.setObjectAcl(bucketNameToShare, elem.getKey(), aclKeys);
}
}
// confirm if acl well
conn.setBucketAcl(bucketNameToShare, acl);
AccessControlList newAcl = conn.getBucketAcl(bucketNameToShare);
Set<Grant> grants = newAcl.getGrants();
boolean flag = false;
for (Grant grant : grants) {
if (grant.getGrantee().getIdentifier().equals(canonicalId[0])) {
flag = true;
}
}
if (!flag) {
throw new ServiceSiteException("AWSS3Exception:: ACL");
}
} catch (AmazonServiceException e1) {
throw new ServiceSiteException("AWSS3Exception::" + e1.getMessage());
} catch (AmazonClientException e2) {
throw new ClientServiceException("AWSS3Exception::" + e2.getMessage());
}
return canonicalId;
}
@Override
public int read(
String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) {
logger.debug("readkey: " + key + " from table: " + table);
GetItemRequest req = new GetItemRequest(table, createPrimaryKey(key));
req.setAttributesToGet(fields);
req.setConsistentRead(consistentRead);
GetItemResult res = null;
try {
res = dynamoDB.getItem(req);
} catch (AmazonServiceException ex) {
logger.error(ex.getMessage());
return SERVER_ERROR;
} catch (AmazonClientException ex) {
logger.error(ex.getMessage());
return CLIENT_ERROR;
}
if (null != res.getItem()) {
result.putAll(extractResult(res.getItem()));
logger.debug("Result: " + res.toString());
}
return OK;
}
private static void waitForTableToBecomeAvailable(AmazonDynamoDB dynamo, String tableName) {
System.out.println("Waiting for " + tableName + " to become ACTIVE...");
long startTime = System.currentTimeMillis();
long endTime = startTime + (10 * 60 * 1000);
while (System.currentTimeMillis() < endTime) {
try {
Thread.sleep(1000 * 20);
} catch (Exception e) {
}
try {
DescribeTableRequest request = new DescribeTableRequest().withTableName(tableName);
TableDescription table = dynamo.describeTable(request).getTable();
if (table == null) continue;
String tableStatus = table.getTableStatus();
System.out.println(" - current state: " + tableStatus);
if (tableStatus.equals(TableStatus.ACTIVE.toString())) return;
} catch (AmazonServiceException ase) {
if (ase.getErrorCode().equalsIgnoreCase("ResourceNotFoundException") == false) throw ase;
}
}
throw new RuntimeException("Table " + tableName + " never went active");
}
@Override
public AmazonServiceException handle(com.amazonaws.http.HttpResponse response)
throws Exception {
AmazonServiceException ase = new AmazonServiceException("Fake service exception.");
ase.setStatusCode(response.getStatusCode());
ase.setErrorCode(response.getStatusText());
return ase;
}
private boolean isServiceSignUpException(Exception e) {
if (e instanceof AmazonServiceException) {
AmazonServiceException ase = (AmazonServiceException) e;
return "OptInRequired".equalsIgnoreCase(ase.getErrorCode());
}
return false;
}
private static boolean doesTableExist(AmazonDynamoDB dynamo, String tableName) {
try {
TableDescription table =
dynamo.describeTable(new DescribeTableRequest().withTableName(tableName)).getTable();
return "ACTIVE".equals(table.getTableStatus());
} catch (AmazonServiceException ase) {
if (ase.getErrorCode().equals("ResourceNotFoundException")) return false;
throw ase;
}
}
public static void sendMessage(AmazonSQS sqs, String sqsUrl, String info) {
try {
sqs.sendMessage(new SendMessageRequest(sqsUrl, info));
logger.info("Message sent to queue: " + info);
} catch (AmazonServiceException e) {
logger.severe(e.getMessage());
} catch (AmazonClientException e) {
logger.severe(e.getMessage());
}
}
public void create(String store) throws IBlobStore.Error {
try {
s3client.createBucket(store);
} catch (AmazonServiceException e) {
if (e.getStatusCode() == 403) throw new IBlobStore.AuthError("" + e);
throw new IBlobStore.IOError("" + e);
} catch (AmazonClientException e) {
throw new IBlobStore.IOError("" + e);
}
}
private void logException(AmazonServiceException ase) {
logger.error(
"AmazonServiceException: error={}, statuscode={}, "
+ "awserrcode={}, errtype={}, reqid={}",
ase.getMessage(),
ase.getStatusCode(),
ase.getErrorCode(),
ase.getErrorType(),
ase.getRequestId());
}
/**
* Retrieves an instruction file from S3. If no instruction file is found, returns null.
*
* @param getObjectRequest A GET request for an object in S3. The parameters from this request
* will be used to retrieve the corresponding instruction file.
* @return An instruction file, or null if no instruction file was found.
*/
private S3Object getInstructionFile(GetObjectRequest getObjectRequest) {
try {
GetObjectRequest instructionFileRequest =
EncryptionUtils.createInstructionGetRequest(getObjectRequest);
return super.getObject(instructionFileRequest);
} catch (AmazonServiceException e) {
// If no instruction file is found, log a debug message, and return null.
log.debug("Unable to retrieve instruction file : " + e.getMessage());
return null;
}
}
/** {@inheritDoc} */
@Override
public void terminateInstance(String instanceId) {
try {
ec2Client().terminateInstances(new TerminateInstancesRequest(Arrays.asList(instanceId)));
} catch (AmazonServiceException e) {
if (e.getErrorCode().equals("InvalidInstanceID.NotFound")) {
throw new NotFoundException("AWS instance " + instanceId + " not found", e);
}
throw e;
}
}
public static void deleteTaskMessage(Message msg, String sqsUrl, AmazonSQS sqs) {
String handle = msg.getReceiptHandle();
try {
sqs.deleteMessage(new DeleteMessageRequest(sqsUrl, handle));
logger.info("Message deleted: " + msg.getBody());
} catch (AmazonServiceException e) {
logger.severe(e.getMessage());
} catch (AmazonClientException e) {
logger.severe(e.getMessage());
}
}
public boolean storeExists(String store) throws IBlobStore.Error {
try {
s3client.listObjects(store);
return true;
} catch (final AmazonServiceException e) {
if (e.getStatusCode() == 404) return false;
if (e.getStatusCode() == 403) throw new IBlobStore.AuthError("" + e);
throw new IBlobStore.IOError("" + e);
} catch (AmazonClientException e) {
Log.v(TAG, "Error: " + e);
throw new IBlobStore.IOError("" + e);
}
}
@Override
public void delete(final TranscodingJob transcodingJob) {
final ImmutableMap.Builder<String, AttributeValue> keyBuilder = ImmutableMap.builder();
keyBuilder.put("transcode-job-id", new AttributeValue(transcodingJob.getId()));
try {
this.dynamoDB.deleteItem(TABLE_NAME, keyBuilder.build());
} catch (final AmazonServiceException e) {
logger.error(
"AmazonServiceException while storing {}. RID: {}", transcodingJob, e.getRequestId(), e);
} catch (final AmazonClientException e) {
logger.error("AmazonClientException while storing {}.", transcodingJob, e);
}
}
// Appends a S3 http:// and bucket address to file path.
public static String generateS3FileAddress(AmazonS3 s3, String path) {
String address;
try {
address = s3.getBucketLocation(bucket);
} catch (AmazonServiceException e) {
logger.severe(e.getMessage());
return null;
} catch (AmazonClientException e) {
logger.severe(e.getMessage());
return null;
}
return "https://s3-" + address + ".amazonaws.com/" + bucket + "/" + path;
}
@Action(
value = "/manage/submitEC2Key",
results = {
@Result(name = "input", location = "/manage/view_ec2_keys.jsp"),
@Result(name = "success", location = "/manage/viewEC2Keys.action", type = "redirect")
})
public String submitEC2Key() {
String retVal = SUCCESS;
try {
// get AWS credentials from DB
AWSCred awsCred = AWSCredDB.getAWSCred();
// set AWS credentials for service
BasicAWSCredentials awsCredentials =
new BasicAWSCredentials(awsCred.getAccessKey(), awsCred.getSecretKey());
// create service
AmazonEC2 service = new AmazonEC2Client(awsCredentials);
service.setEndpoint(ec2Key.getEc2Region());
// create key pair request
CreateKeyPairRequest createKeyPairRequest = new CreateKeyPairRequest();
createKeyPairRequest.withKeyName(ec2Key.getKeyNm());
// call service
CreateKeyPairResult createKeyPairResult = service.createKeyPair(createKeyPairRequest);
// get key pair result
KeyPair keyPair = createKeyPairResult.getKeyPair();
// set private key
String privateKey = keyPair.getKeyMaterial();
ec2Key.setPrivateKey(privateKey);
// add to db
Long keyId = EC2KeyDB.saveEC2Key(ec2Key);
// store private key
SSHUtil.storePrivateKey(keyId.toString(), ec2Key.getPrivateKey().trim());
} catch (AmazonServiceException ex) {
addActionError(ex.getMessage());
retVal = INPUT;
}
return retVal;
}
@Override
public void store(final TranscodingJob transcodingJob) {
final ImmutableMap.Builder<String, AttributeValue> itemBuilder = ImmutableMap.builder();
itemBuilder.put("transcode-job-id", new AttributeValue(transcodingJob.getId()));
itemBuilder.put("inputMovie", new AttributeValue(transcodingJob.getInputPath().toString()));
itemBuilder.put("outputMovie", new AttributeValue(transcodingJob.getOutputPath().toString()));
try {
this.dynamoDB.putItem(TABLE_NAME, itemBuilder.build());
} catch (final AmazonServiceException e) {
logger.error(
"AmazonServiceException while storing {}. RID: {}", transcodingJob, e.getRequestId(), e);
} catch (final AmazonClientException e) {
logger.error("AmazonClientException while storing {}.", transcodingJob, e);
}
}
private int parseClockSkewOffset(
org.apache.http.HttpResponse response, AmazonServiceException exception) {
Date deviceDate = new Date();
Date serverDate = null;
String serverDateStr = null;
Header[] responseDateHeader = response.getHeaders("Date");
try {
if (responseDateHeader.length == 0) {
// SQS doesn't return Date header
serverDateStr = getServerDateFromException(exception.getMessage());
serverDate = DateUtils.parseCompressedISO8601Date(serverDateStr);
} else {
serverDateStr = responseDateHeader[0].getValue();
serverDate = DateUtils.parseRFC822Date(serverDateStr);
}
} catch (RuntimeException e) {
log.warn("Unable to parse clock skew offset from response: " + serverDateStr, e);
return 0;
}
long diff = deviceDate.getTime() - serverDate.getTime();
return (int) (diff / 1000);
}
/**
* Returns true if a failed request should be retried.
*
* @param method The current HTTP method being executed.
* @param exception The exception from the failed request.
* @param retries The number of times the current request has been attempted.
* @return True if the failed request should be retried.
*/
private boolean shouldRetry(HttpRequestBase method, Exception exception, int retries) {
if (retries >= config.getMaxErrorRetry()) return false;
if (method instanceof HttpEntityEnclosingRequest) {
HttpEntity entity = ((HttpEntityEnclosingRequest) method).getEntity();
if (entity != null && !entity.isRepeatable()) {
if (log.isDebugEnabled()) {
log.debug("Entity not repeatable");
}
return false;
}
}
if (exception instanceof IOException) {
if (log.isDebugEnabled()) {
log.debug("Retrying on " + exception.getClass().getName() + ": " + exception.getMessage());
}
return true;
}
if (exception instanceof AmazonServiceException) {
AmazonServiceException ase = (AmazonServiceException) exception;
/*
* For 500 internal server errors and 503 service
* unavailable errors, we want to retry, but we need to use
* an exponential back-off strategy so that we don't overload
* a server with a flood of retries. If we've surpassed our
* retry limit we handle the error response as a non-retryable
* error and go ahead and throw it back to the user as an exception.
*/
if (ase.getStatusCode() == HttpStatus.SC_INTERNAL_SERVER_ERROR
|| ase.getStatusCode() == HttpStatus.SC_SERVICE_UNAVAILABLE) {
return true;
}
/*
* Throttling is reported as a 400 error from newer services. To try
* and smooth out an occasional throttling error, we'll pause and
* retry, hoping that the pause is long enough for the request to
* get through the next time.
*/
if (isThrottlingException(ase)) return true;
}
return false;
}
protected Void doInBackground(Void... voids) {
try {
SimpleDB.createItem(domainName.getSelectedItem().toString(), itemName.getText().toString());
} catch (AmazonServiceException e) {
if ("InvalidClientTokenId".equals(e.getErrorCode())) {
putRefreshError();
} else {
setStackAndPost(e);
}
} catch (Throwable e) {
setStackAndPost(e);
}
return null;
}
// Makes a putObjectRequest to make file public, and sends request.
private static boolean putObject(AmazonS3 s3, PutObjectRequest req) {
// Set file as public.
req.withCannedAcl(CannedAccessControlList.PublicRead);
// Send upload request.
try {
s3.putObject(req);
} catch (AmazonServiceException e) {
logger.severe(e.getMessage());
return false;
} catch (AmazonClientException e) {
logger.severe(e.getMessage());
return false;
}
return true;
}
public static List<Message> getMessages(ReceiveMessageRequest req, AmazonSQS sqs) {
logger.fine("Getting SQS messages.");
List<Message> msgs;
try {
msgs = sqs.receiveMessage(req).getMessages();
} catch (AmazonServiceException e) {
logger.severe(e.getMessage());
return null;
} catch (AmazonClientException e) {
logger.severe(e.getMessage());
return null;
}
return msgs;
}
@Action(
value = "/manage/importEC2Key",
results = {
@Result(name = "input", location = "/manage/view_ec2_keys.jsp"),
@Result(name = "success", location = "/manage/viewEC2Keys.action", type = "redirect")
})
public String importEC2Key() {
String retVal = SUCCESS;
try {
// get AWS credentials from DB
AWSCred awsCred = AWSCredDB.getAWSCred();
// set AWS credentials for service
BasicAWSCredentials awsCredentials =
new BasicAWSCredentials(awsCred.getAccessKey(), awsCred.getSecretKey());
// create service
AmazonEC2 service = new AmazonEC2Client(awsCredentials);
service.setEndpoint(ec2Key.getEc2Region());
// describe key pair request
DescribeKeyPairsRequest describeKeyPairsRequest = new DescribeKeyPairsRequest();
describeKeyPairsRequest.setKeyNames(Arrays.asList(ec2Key.getKeyNm()));
// call service
DescribeKeyPairsResult describeKeyPairsResult =
service.describeKeyPairs(describeKeyPairsRequest);
if (describeKeyPairsResult != null && describeKeyPairsResult.getKeyPairs().size() > 0) {
// add to db
Long keyId = EC2KeyDB.saveEC2Key(ec2Key);
SSHUtil.storePrivateKey(keyId.toString(), ec2Key.getPrivateKey().trim());
} else {
addActionError("Imported key does not exist on AWS");
retVal = INPUT;
}
} catch (AmazonServiceException ex) {
addActionError(ex.getMessage());
retVal = INPUT;
}
return retVal;
}
@Override
public int delete(String table, String key) {
logger.debug("deletekey: " + key + " from table: " + table);
DeleteItemRequest req = new DeleteItemRequest(table, createPrimaryKey(key));
DeleteItemResult res = null;
try {
res = dynamoDB.deleteItem(req);
} catch (AmazonServiceException ex) {
logger.error(ex.getMessage());
return SERVER_ERROR;
} catch (AmazonClientException ex) {
logger.error(ex.getMessage());
return CLIENT_ERROR;
}
return OK;
}