/**
* Converts the specified request object into a URL, containing all the specified parameters, the
* specified request endpoint, etc.
*
* @param request The request to convert into a URL.
* @param removeLeadingSlashInResourcePath Whether the leading slash in resource-path should be
* removed before appending to the endpoint.
* @return A new URL representing the specified request.
* @throws AmazonClientException If the request cannot be converted to a well formed URL.
*/
public static URL convertRequestToUrl(
Request<?> request, boolean removeLeadingSlashInResourcePath) {
String resourcePath = HttpUtils.urlEncode(request.getResourcePath(), true);
// Removed the padding "/" that was already added into the request's resource path.
if (removeLeadingSlashInResourcePath && resourcePath.startsWith("/")) {
resourcePath = resourcePath.substring(1);
}
// Some http client libraries (e.g. Apache HttpClient) cannot handle
// consecutive "/"s between URL authority and path components.
// So we escape "////..." into "/%2F%2F%2F...", in the same way as how
// we treat consecutive "/"s in AmazonS3Client#presignRequest(...)
String urlPath = "/" + resourcePath;
urlPath = urlPath.replaceAll("(?<=/)/", "%2F");
String urlString = request.getEndpoint() + urlPath;
boolean firstParam = true;
for (String param : request.getParameters().keySet()) {
if (firstParam) {
urlString += "?";
firstParam = false;
} else {
urlString += "&";
}
String value = request.getParameters().get(param);
urlString += param + "=" + HttpUtils.urlEncode(value, false);
}
try {
return new URL(urlString);
} catch (MalformedURLException e) {
throw new AmazonClientException(
"Unable to convert request to well formed URL: " + e.getMessage(), e);
}
}
/* (non-Javadoc)
* @see com.amazonaws.auth.Signer#sign(com.amazonaws.Request, com.amazonaws.auth.AWSCredentials)
*/
public void sign(Request<?> request, AWSCredentials credentials) throws AmazonClientException {
// annonymous credentials, don't sign
if (credentials instanceof AnonymousAWSCredentials) {
return;
}
AWSCredentials sanitizedCredentials = sanitizeCredentials(credentials);
if (sanitizedCredentials instanceof AWSSessionCredentials) {
addSessionCredentials(request, (AWSSessionCredentials) sanitizedCredentials);
}
SimpleDateFormat dateStampFormat = new SimpleDateFormat("yyyyMMdd");
dateStampFormat.setTimeZone(new SimpleTimeZone(0, "UTC"));
SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'");
dateTimeFormat.setTimeZone(new SimpleTimeZone(0, "UTC"));
String regionName = extractRegionName(request.getEndpoint());
String serviceName = extractServiceName(request.getEndpoint());
// AWS4 requires that we sign the Host header so we
// have to have it in the request by the time we sign.
String hostHeader = request.getEndpoint().getHost();
if (HttpUtils.isUsingNonDefaultPort(request.getEndpoint())) {
hostHeader += ":" + request.getEndpoint().getPort();
}
request.addHeader("Host", hostHeader);
Date date = new Date();
if (overriddenDate != null) date = overriddenDate;
String dateTime = dateTimeFormat.format(date);
String dateStamp = dateStampFormat.format(date);
InputStream payloadStream = getBinaryRequestPayloadStream(request);
payloadStream.mark(-1);
String contentSha256 = BinaryUtils.toHex(hash(payloadStream));
try {
payloadStream.reset();
} catch (IOException e) {
throw new AmazonClientException("Unable to reset stream after calculating AWS4 signature", e);
}
request.addHeader("X-Amz-Date", dateTime);
request.addHeader("x-amz-content-sha256", contentSha256);
String canonicalRequest =
request.getHttpMethod().toString()
+ "\n"
+ super.getCanonicalizedResourcePath(request.getResourcePath())
+ "\n"
+ getCanonicalizedQueryString(request)
+ "\n"
+ getCanonicalizedHeaderString(request)
+ "\n"
+ getSignedHeadersString(request)
+ "\n"
+ contentSha256;
log.debug("AWS4 Canonical Request: '\"" + canonicalRequest + "\"");
String scope = dateStamp + "/" + regionName + "/" + serviceName + "/" + TERMINATOR;
String signingCredentials = sanitizedCredentials.getAWSAccessKeyId() + "/" + scope;
String stringToSign =
ALGORITHM
+ "\n"
+ dateTime
+ "\n"
+ scope
+ "\n"
+ BinaryUtils.toHex(hash(canonicalRequest));
log.debug("AWS4 String to Sign: '\"" + stringToSign + "\"");
// AWS4 uses a series of derived keys, formed by hashing different pieces of data
byte[] kSecret = ("AWS4" + sanitizedCredentials.getAWSSecretKey()).getBytes();
byte[] kDate = sign(dateStamp, kSecret, SigningAlgorithm.HmacSHA256);
byte[] kRegion = sign(regionName, kDate, SigningAlgorithm.HmacSHA256);
byte[] kService = sign(serviceName, kRegion, SigningAlgorithm.HmacSHA256);
byte[] kSigning = sign(TERMINATOR, kService, SigningAlgorithm.HmacSHA256);
byte[] signature = sign(stringToSign.getBytes(), kSigning, SigningAlgorithm.HmacSHA256);
String credentialsAuthorizationHeader = "Credential=" + signingCredentials;
String signedHeadersAuthorizationHeader = "SignedHeaders=" + getSignedHeadersString(request);
String signatureAuthorizationHeader = "Signature=" + BinaryUtils.toHex(signature);
String authorizationHeader =
ALGORITHM
+ " "
+ credentialsAuthorizationHeader
+ ", "
+ signedHeadersAuthorizationHeader
+ ", "
+ signatureAuthorizationHeader;
request.addHeader("Authorization", authorizationHeader);
}
@Override
public void afterResponse(com.amazonaws.Request<?> request, com.amazonaws.Response<?> response) {
try {
if (response == null) {
return;
}
Map<String, String> headers = response.getHttpResponse().getHeaders();
String responseTimeStr = headers.get(REQUESTTIME_HEADER);
long responseTime = startTime - System.currentTimeMillis();
if (responseTimeStr != null && responseTimeStr.trim().length() > 0) {
try {
responseTime = Long.parseLong(responseTimeStr.trim());
} catch (NumberFormatException e) {
}
}
double responseDouble = -1;
try {
responseDouble = Double.valueOf(Long.toString(responseTime));
} catch (NumberFormatException numberFormatException) {
Log.e(TAG, "Couldn't convert response time to double format", numberFormatException);
}
String requestAttemptsStr = headers.get(REQUESTATTEMPTS_HEADER);
double requestAttempts = 0;
if (requestAttemptsStr != null && requestAttemptsStr.trim().length() > 0) {
try {
requestAttempts = Double.parseDouble(requestAttemptsStr.trim());
} catch (NumberFormatException e) {
}
}
String serverInfo = headers.get(SERVERINFO_HEADER);
if (!StringUtil.isNullOrEmpty(responseTimeStr)) {
if (eventClient == null) {
return;
}
AnalyticsEvent recordEvent =
eventClient
.createEvent("_httpRequestTiming")
.withAttribute("url", request.getEndpoint().toURL().toString())
.withAttribute(
"responseCode", Integer.toString(response.getHttpResponse().getStatusCode()))
.withAttribute("timeZone", getTimeZone())
.withMetric("attempts", requestAttempts)
.withMetric("totalTime", responseDouble)
.withMetric("requestSize", (double) contentLength);
String connectionType = "UNKNOWN";
if (connectivity != null) {
if (connectivity.hasWifi()) {
connectionType = "WIFI";
} else if (connectivity.hasWAN()) {
connectionType = "WAN";
}
}
recordEvent.withAttribute("network", connectionType);
if (serverInfo != null) {
recordEvent.withAttribute("serverInfo", serverInfo);
}
eventClient.recordEvent(recordEvent);
}
} catch (Exception e) {
Log.e(TAG, "Unable to record _RequestTime event", e);
}
}
/**
* Internal method to execute the HTTP method given.
*
* @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler)
* @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler,
* ExecutionContext)
*/
private <T> Response<T> executeHelper(
Request<?> request,
HttpResponseHandler<AmazonWebServiceResponse<T>> responseHandler,
HttpResponseHandler<AmazonServiceException> errorResponseHandler,
ExecutionContext executionContext)
throws AmazonClientException, AmazonServiceException {
/*
* Depending on which response handler we end up choosing to handle the
* HTTP response, it might require us to leave the underlying HTTP
* connection open, depending on whether or not it reads the complete
* HTTP response stream from the HTTP connection, or if delays reading
* any of the content until after a response is returned to the caller.
*/
boolean leaveHttpConnectionOpen = false;
AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics();
/* add the service endpoint to the logs. You can infer service name from service endpoint */
awsRequestMetrics.addProperty(Field.ServiceName, request.getServiceName());
awsRequestMetrics.addProperty(Field.ServiceEndpoint, request.getEndpoint());
// Apply whatever request options we know how to handle, such as user-agent.
setUserAgent(request);
int requestCount = 0;
URI redirectedURI = null;
HttpEntity entity = null;
AmazonClientException retriedException = null;
// Make a copy of the original request params and headers so that we can
// permute it in this loop and start over with the original every time.
Map<String, String> originalParameters = new LinkedHashMap<String, String>();
originalParameters.putAll(request.getParameters());
Map<String, String> originalHeaders = new HashMap<String, String>();
originalHeaders.putAll(request.getHeaders());
final AWSCredentials credentials = executionContext.getCredentials();
Signer signer = null;
while (true) {
++requestCount;
awsRequestMetrics.setCounter(Field.RequestCount, requestCount);
if (requestCount > 1) { // retry
request.setParameters(originalParameters);
request.setHeaders(originalHeaders);
}
HttpRequestBase httpRequest = null;
org.apache.http.HttpResponse apacheResponse = null;
try {
// Sign the request if a signer was provided
if (signer == null) signer = executionContext.getSignerByURI(request.getEndpoint());
if (signer != null && credentials != null) {
awsRequestMetrics.startEvent(Field.RequestSigningTime);
try {
signer.sign(request, credentials);
} finally {
awsRequestMetrics.endEvent(Field.RequestSigningTime);
}
}
if (requestLog.isDebugEnabled()) {
requestLog.debug("Sending Request: " + request.toString());
}
httpRequest = httpRequestFactory.createHttpRequest(request, config, executionContext);
if (httpRequest instanceof HttpEntityEnclosingRequest) {
entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity();
}
if (redirectedURI != null) {
httpRequest.setURI(redirectedURI);
}
if (requestCount > 1) { // retry
awsRequestMetrics.startEvent(Field.RetryPauseTime);
try {
pauseBeforeNextRetry(
request.getOriginalRequest(),
retriedException,
requestCount,
config.getRetryPolicy());
} finally {
awsRequestMetrics.endEvent(Field.RetryPauseTime);
}
}
if (entity != null) {
InputStream content = entity.getContent();
if (requestCount > 1) { // retry
if (content.markSupported()) {
content.reset();
content.mark(-1);
}
} else {
if (content.markSupported()) {
content.mark(-1);
}
}
}
captureConnectionPoolMetrics(httpClient.getConnectionManager(), awsRequestMetrics);
HttpContext httpContext = new BasicHttpContext();
httpContext.setAttribute(AWSRequestMetrics.class.getSimpleName(), awsRequestMetrics);
retriedException = null;
awsRequestMetrics.startEvent(Field.HttpRequestTime);
try {
apacheResponse = httpClient.execute(httpRequest, httpContext);
} finally {
awsRequestMetrics.endEvent(Field.HttpRequestTime);
}
if (isRequestSuccessful(apacheResponse)) {
awsRequestMetrics.addProperty(
Field.StatusCode, apacheResponse.getStatusLine().getStatusCode());
/*
* If we get back any 2xx status code, then we know we should
* treat the service call as successful.
*/
leaveHttpConnectionOpen = responseHandler.needsConnectionLeftOpen();
HttpResponse httpResponse = createResponse(httpRequest, request, apacheResponse);
T response =
handleResponse(
request,
responseHandler,
httpRequest,
httpResponse,
apacheResponse,
executionContext);
return new Response<T>(response, httpResponse);
} else if (isTemporaryRedirect(apacheResponse)) {
/*
* S3 sends 307 Temporary Redirects if you try to delete an
* EU bucket from the US endpoint. If we get a 307, we'll
* point the HTTP method to the redirected location, and let
* the next retry deliver the request to the right location.
*/
Header[] locationHeaders = apacheResponse.getHeaders("location");
String redirectedLocation = locationHeaders[0].getValue();
log.debug("Redirecting to: " + redirectedLocation);
redirectedURI = URI.create(redirectedLocation);
httpRequest.setURI(redirectedURI);
awsRequestMetrics.addProperty(
Field.StatusCode, apacheResponse.getStatusLine().getStatusCode());
awsRequestMetrics.addProperty(Field.RedirectLocation, redirectedLocation);
awsRequestMetrics.addProperty(Field.AWSRequestID, null);
} else {
leaveHttpConnectionOpen = errorResponseHandler.needsConnectionLeftOpen();
AmazonServiceException ase =
handleErrorResponse(request, errorResponseHandler, httpRequest, apacheResponse);
awsRequestMetrics.addProperty(Field.AWSRequestID, ase.getRequestId());
awsRequestMetrics.addProperty(Field.AWSErrorCode, ase.getErrorCode());
awsRequestMetrics.addProperty(Field.StatusCode, ase.getStatusCode());
if (!shouldRetry(
request.getOriginalRequest(),
httpRequest,
ase,
requestCount,
config.getRetryPolicy())) {
throw ase;
}
// Cache the retryable exception
retriedException = ase;
/*
* Checking for clock skew error again because we don't want to set the
* global time offset for every service exception.
*/
if (RetryUtils.isClockSkewError(ase)) {
int timeOffset = parseClockSkewOffset(apacheResponse, ase);
SDKGlobalConfiguration.setGlobalTimeOffset(timeOffset);
}
resetRequestAfterError(request, ase);
}
} catch (IOException ioe) {
if (log.isInfoEnabled()) {
log.info("Unable to execute HTTP request: " + ioe.getMessage(), ioe);
}
awsRequestMetrics.incrementCounter(Field.Exception);
awsRequestMetrics.addProperty(Field.Exception, ioe);
awsRequestMetrics.addProperty(Field.AWSRequestID, null);
AmazonClientException ace =
new AmazonClientException("Unable to execute HTTP request: " + ioe.getMessage(), ioe);
if (!shouldRetry(
request.getOriginalRequest(),
httpRequest,
ace,
requestCount,
config.getRetryPolicy())) {
throw ace;
}
// Cache the retryable exception
retriedException = ace;
resetRequestAfterError(request, ioe);
} catch (RuntimeException e) {
throw handleUnexpectedFailure(e, awsRequestMetrics);
} catch (Error e) {
throw handleUnexpectedFailure(e, awsRequestMetrics);
} finally {
/*
* Some response handlers need to manually manage the HTTP
* connection and will take care of releasing the connection on
* their own, but if this response handler doesn't need the
* connection left open, we go ahead and release the it to free
* up resources.
*/
if (!leaveHttpConnectionOpen) {
try {
if (apacheResponse != null
&& apacheResponse.getEntity() != null
&& apacheResponse.getEntity().getContent() != null) {
apacheResponse.getEntity().getContent().close();
}
} catch (IOException e) {
log.warn("Cannot close the response content.", e);
}
}
}
} /* end while (true) */
}
/**
* Internal method to execute the HTTP method given.
*
* @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler)
* @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler,
* ExecutionContext)
*/
private <T extends Object> T executeHelper(
Request<?> request,
HttpResponseHandler<AmazonWebServiceResponse<T>> responseHandler,
HttpResponseHandler<AmazonServiceException> errorResponseHandler,
ExecutionContext executionContext)
throws AmazonClientException, AmazonServiceException {
/*
* Depending on which response handler we end up choosing to handle the
* HTTP response, it might require us to leave the underlying HTTP
* connection open, depending on whether or not it reads the complete
* HTTP response stream from the HTTP connection, or if delays reading
* any of the content until after a response is returned to the caller.
*/
boolean leaveHttpConnectionOpen = false;
AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics();
/* add the service endpoint to the logs. You can infer service name from service endpoint */
awsRequestMetrics.addProperty(Field.ServiceName.name(), request.getServiceName());
awsRequestMetrics.addProperty(Field.ServiceEndpoint.name(), request.getEndpoint());
// Apply whatever request options we know how to handle, such as user-agent.
applyRequestData(request);
int retryCount = 0;
URI redirectedURI = null;
HttpEntity entity = null;
AmazonServiceException exception = null;
// Make a copy of the original request params and headers so that we can
// permute it in this loop and start over with the original every time.
Map<String, String> originalParameters = new HashMap<String, String>();
originalParameters.putAll(request.getParameters());
Map<String, String> originalHeaders = new HashMap<String, String>();
originalHeaders.putAll(request.getHeaders());
while (true) {
awsRequestMetrics.setCounter(Field.AttemptCount.name(), retryCount + 1);
if (retryCount > 0) {
request.setParameters(originalParameters);
request.setHeaders(originalHeaders);
}
HttpRequestBase httpRequest = null;
org.apache.http.HttpResponse response = null;
try {
// Sign the request if a signer was provided
if (executionContext.getSigner() != null && executionContext.getCredentials() != null) {
awsRequestMetrics.startEvent(Field.RequestSigningTime.name());
executionContext.getSigner().sign(request, executionContext.getCredentials());
awsRequestMetrics.endEvent(Field.RequestSigningTime.name());
}
if (requestLog.isDebugEnabled()) {
requestLog.debug("Sending Request: " + request.toString());
}
httpRequest =
httpRequestFactory.createHttpRequest(request, config, entity, executionContext);
if (httpRequest instanceof HttpEntityEnclosingRequest) {
entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity();
}
if (redirectedURI != null) {
httpRequest.setURI(redirectedURI);
}
if (retryCount > 0) {
awsRequestMetrics.startEvent(Field.RetryPauseTime.name());
pauseExponentially(retryCount, exception, executionContext.getCustomBackoffStrategy());
awsRequestMetrics.endEvent(Field.RetryPauseTime.name());
}
if (entity != null) {
InputStream content = entity.getContent();
if (retryCount > 0) {
if (content.markSupported()) {
content.reset();
content.mark(-1);
}
} else {
if (content.markSupported()) {
content.mark(-1);
}
}
}
exception = null;
awsRequestMetrics.startEvent(Field.HttpRequestTime.name());
response = httpClient.execute(httpRequest);
awsRequestMetrics.endEvent(Field.HttpRequestTime.name());
if (isRequestSuccessful(response)) {
awsRequestMetrics.addProperty(
Field.StatusCode.name(), response.getStatusLine().getStatusCode());
/*
* If we get back any 2xx status code, then we know we should
* treat the service call as successful.
*/
leaveHttpConnectionOpen = responseHandler.needsConnectionLeftOpen();
return handleResponse(request, responseHandler, httpRequest, response, executionContext);
} else if (isTemporaryRedirect(response)) {
/*
* S3 sends 307 Temporary Redirects if you try to delete an
* EU bucket from the US endpoint. If we get a 307, we'll
* point the HTTP method to the redirected location, and let
* the next retry deliver the request to the right location.
*/
Header[] locationHeaders = response.getHeaders("location");
String redirectedLocation = locationHeaders[0].getValue();
log.debug("Redirecting to: " + redirectedLocation);
redirectedURI = URI.create(redirectedLocation);
httpRequest.setURI(redirectedURI);
awsRequestMetrics.addProperty(
Field.StatusCode.name(), response.getStatusLine().getStatusCode());
awsRequestMetrics.addProperty(Field.RedirectLocation.name(), redirectedLocation);
awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null);
} else {
leaveHttpConnectionOpen = errorResponseHandler.needsConnectionLeftOpen();
exception = handleErrorResponse(request, errorResponseHandler, httpRequest, response);
awsRequestMetrics.addProperty(Field.AWSRequestID.name(), exception.getRequestId());
awsRequestMetrics.addProperty(Field.AWSErrorCode.name(), exception.getErrorCode());
awsRequestMetrics.addProperty(Field.StatusCode.name(), exception.getStatusCode());
if (!shouldRetry(httpRequest, exception, retryCount)) {
throw exception;
}
resetRequestAfterError(request, exception);
}
} catch (IOException ioe) {
log.info("Unable to execute HTTP request: " + ioe.getMessage(), ioe);
awsRequestMetrics.addProperty(Field.Exception.name(), ioe.toString());
awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null);
if (!shouldRetry(httpRequest, ioe, retryCount)) {
throw new AmazonClientException(
"Unable to execute HTTP request: " + ioe.getMessage(), ioe);
}
resetRequestAfterError(request, ioe);
} finally {
retryCount++;
/*
* Some response handlers need to manually manage the HTTP
* connection and will take care of releasing the connection on
* their own, but if this response handler doesn't need the
* connection left open, we go ahead and release the it to free
* up resources.
*/
if (!leaveHttpConnectionOpen) {
try {
response.getEntity().getContent().close();
} catch (Throwable t) {
}
}
}
} /* end while (true) */
}