/**
* Downloads an S3Object, as returned from {@link
* AmazonS3Client#getObject(com.amazonaws.services.s3.model.GetObjectRequest)}, to the specified
* file.
*
* @param s3Object The S3Object containing a reference to an InputStream containing the object's
* data.
* @param destinationFile The file to store the object's data in.
* @param performIntegrityCheck Boolean valuable to indicate whether do the integrity check or not
*/
public static void downloadObjectToFile(
S3Object s3Object, File destinationFile, boolean performIntegrityCheck) {
// attempt to create the parent if it doesn't exist
File parentDirectory = destinationFile.getParentFile();
if (parentDirectory != null && !parentDirectory.exists()) {
parentDirectory.mkdirs();
}
OutputStream outputStream = null;
try {
outputStream = new BufferedOutputStream(new FileOutputStream(destinationFile));
byte[] buffer = new byte[1024 * 10];
int bytesRead;
while ((bytesRead = s3Object.getObjectContent().read(buffer)) > -1) {
outputStream.write(buffer, 0, bytesRead);
}
} catch (IOException e) {
try {
s3Object.getObjectContent().abort();
} catch (IOException abortException) {
log.warn("Couldn't abort stream", e);
}
throw new AmazonClientException(
"Unable to store object contents to disk: " + e.getMessage(), e);
} finally {
try {
outputStream.close();
} catch (Exception e) {
}
try {
s3Object.getObjectContent().close();
} catch (Exception e) {
}
}
byte[] clientSideHash = null;
byte[] serverSideHash = null;
try {
// Multipart Uploads don't have an MD5 calculated on the service side
if (ServiceUtils.isMultipartUploadETag(s3Object.getObjectMetadata().getETag()) == false) {
clientSideHash = Md5Utils.computeMD5Hash(new FileInputStream(destinationFile));
serverSideHash = BinaryUtils.fromHex(s3Object.getObjectMetadata().getETag());
}
} catch (Exception e) {
log.warn("Unable to calculate MD5 hash to validate download: " + e.getMessage(), e);
}
if (performIntegrityCheck
&& clientSideHash != null
&& serverSideHash != null
&& !Arrays.equals(clientSideHash, serverSideHash)) {
throw new AmazonClientException(
"Unable to verify integrity of data download. "
+ "Client calculated content hash didn't match hash calculated by Amazon S3. "
+ "The data stored in '"
+ destinationFile.getAbsolutePath()
+ "' may be corrupt.");
}
}
/** @see com.amazonaws.http.HttpResponseHandler#handle(com.amazonaws.http.HttpResponse) */
public AmazonWebServiceResponse<S3Object> handle(HttpResponse response) throws Exception {
/*
* TODO: It'd be nice to set the bucket name and key here, but the
* information isn't easy to pull out of the response/request
* currently.
*/
S3Object object = new S3Object();
AmazonWebServiceResponse<S3Object> awsResponse = parseResponseMetadata(response);
if (response.getHeaders().get(Headers.REDIRECT_LOCATION) != null) {
object.setRedirectLocation(response.getHeaders().get(Headers.REDIRECT_LOCATION));
}
ObjectMetadata metadata = object.getObjectMetadata();
populateObjectMetadata(response, metadata);
boolean hasServerSideCalculatedChecksum =
!ServiceUtils.isMultipartUploadETag(metadata.getETag());
boolean responseContainsEntireObject = response.getHeaders().get("Content-Range") == null;
if (hasServerSideCalculatedChecksum && responseContainsEntireObject) {
byte[] expectedChecksum = BinaryUtils.fromHex(metadata.getETag());
object.setObjectContent(
new S3ObjectInputStream(
new ChecksumValidatingInputStream(
response.getContent(),
expectedChecksum,
object.getBucketName() + "/" + object.getKey()),
response.getHttpRequest()));
} else {
object.setObjectContent(
new S3ObjectInputStream(response.getContent(), response.getHttpRequest()));
}
awsResponse.setResult(object);
return awsResponse;
}
public Request<PublishRequest> marshall(PublishRequest publishRequest) {
if (publishRequest == null) {
throw new AmazonClientException("Invalid argument passed to marshall(...)");
}
Request<PublishRequest> request =
new DefaultRequest<PublishRequest>(publishRequest, "AWSIotData");
request.setHttpMethod(HttpMethodName.POST);
String uriResourcePath = "/topics/{topic}";
uriResourcePath =
uriResourcePath.replace(
"{topic}",
(publishRequest.getTopic() == null)
? ""
: StringUtils.fromString(publishRequest.getTopic()));
request.setResourcePath(uriResourcePath);
String qos =
(publishRequest.getQos() == null) ? null : StringUtils.fromInteger(publishRequest.getQos());
if (qos != null) {
request.addParameter("qos", qos);
}
request.setContent(BinaryUtils.toStream(publishRequest.getPayload()));
if (!request.getHeaders().containsKey("Content-Type")) {
request.addHeader("Content-Type", DEFAULT_CONTENT_TYPE);
}
return request;
}
/* (non-Javadoc)
* @see com.amazonaws.auth.Signer#sign(com.amazonaws.Request, com.amazonaws.auth.AWSCredentials)
*/
public void sign(Request<?> request, AWSCredentials credentials) throws AmazonClientException {
// annonymous credentials, don't sign
if (credentials instanceof AnonymousAWSCredentials) {
return;
}
AWSCredentials sanitizedCredentials = sanitizeCredentials(credentials);
if (sanitizedCredentials instanceof AWSSessionCredentials) {
addSessionCredentials(request, (AWSSessionCredentials) sanitizedCredentials);
}
SimpleDateFormat dateStampFormat = new SimpleDateFormat("yyyyMMdd");
dateStampFormat.setTimeZone(new SimpleTimeZone(0, "UTC"));
SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'");
dateTimeFormat.setTimeZone(new SimpleTimeZone(0, "UTC"));
String regionName = extractRegionName(request.getEndpoint());
String serviceName = extractServiceName(request.getEndpoint());
// AWS4 requires that we sign the Host header so we
// have to have it in the request by the time we sign.
String hostHeader = request.getEndpoint().getHost();
if (HttpUtils.isUsingNonDefaultPort(request.getEndpoint())) {
hostHeader += ":" + request.getEndpoint().getPort();
}
request.addHeader("Host", hostHeader);
Date date = new Date();
if (overriddenDate != null) date = overriddenDate;
String dateTime = dateTimeFormat.format(date);
String dateStamp = dateStampFormat.format(date);
InputStream payloadStream = getBinaryRequestPayloadStream(request);
payloadStream.mark(-1);
String contentSha256 = BinaryUtils.toHex(hash(payloadStream));
try {
payloadStream.reset();
} catch (IOException e) {
throw new AmazonClientException("Unable to reset stream after calculating AWS4 signature", e);
}
request.addHeader("X-Amz-Date", dateTime);
request.addHeader("x-amz-content-sha256", contentSha256);
String canonicalRequest =
request.getHttpMethod().toString()
+ "\n"
+ super.getCanonicalizedResourcePath(request.getResourcePath())
+ "\n"
+ getCanonicalizedQueryString(request)
+ "\n"
+ getCanonicalizedHeaderString(request)
+ "\n"
+ getSignedHeadersString(request)
+ "\n"
+ contentSha256;
log.debug("AWS4 Canonical Request: '\"" + canonicalRequest + "\"");
String scope = dateStamp + "/" + regionName + "/" + serviceName + "/" + TERMINATOR;
String signingCredentials = sanitizedCredentials.getAWSAccessKeyId() + "/" + scope;
String stringToSign =
ALGORITHM
+ "\n"
+ dateTime
+ "\n"
+ scope
+ "\n"
+ BinaryUtils.toHex(hash(canonicalRequest));
log.debug("AWS4 String to Sign: '\"" + stringToSign + "\"");
// AWS4 uses a series of derived keys, formed by hashing different pieces of data
byte[] kSecret = ("AWS4" + sanitizedCredentials.getAWSSecretKey()).getBytes();
byte[] kDate = sign(dateStamp, kSecret, SigningAlgorithm.HmacSHA256);
byte[] kRegion = sign(regionName, kDate, SigningAlgorithm.HmacSHA256);
byte[] kService = sign(serviceName, kRegion, SigningAlgorithm.HmacSHA256);
byte[] kSigning = sign(TERMINATOR, kService, SigningAlgorithm.HmacSHA256);
byte[] signature = sign(stringToSign.getBytes(), kSigning, SigningAlgorithm.HmacSHA256);
String credentialsAuthorizationHeader = "Credential=" + signingCredentials;
String signedHeadersAuthorizationHeader = "SignedHeaders=" + getSignedHeadersString(request);
String signatureAuthorizationHeader = "Signature=" + BinaryUtils.toHex(signature);
String authorizationHeader =
ALGORITHM
+ " "
+ credentialsAuthorizationHeader
+ ", "
+ signedHeadersAuthorizationHeader
+ ", "
+ signatureAuthorizationHeader;
request.addHeader("Authorization", authorizationHeader);
}
