/** * Converts the specified request object into a URL, containing all the specified parameters, the * specified request endpoint, etc. * * @param request The request to convert into a URL. * @param removeLeadingSlashInResourcePath Whether the leading slash in resource-path should be * removed before appending to the endpoint. * @return A new URL representing the specified request. * @throws AmazonClientException If the request cannot be converted to a well formed URL. */ public static URL convertRequestToUrl( Request<?> request, boolean removeLeadingSlashInResourcePath) { String resourcePath = HttpUtils.urlEncode(request.getResourcePath(), true); // Removed the padding "/" that was already added into the request's resource path. if (removeLeadingSlashInResourcePath && resourcePath.startsWith("/")) { resourcePath = resourcePath.substring(1); } // Some http client libraries (e.g. Apache HttpClient) cannot handle // consecutive "/"s between URL authority and path components. // So we escape "////..." into "/%2F%2F%2F...", in the same way as how // we treat consecutive "/"s in AmazonS3Client#presignRequest(...) String urlPath = "/" + resourcePath; urlPath = urlPath.replaceAll("(?<=/)/", "%2F"); String urlString = request.getEndpoint() + urlPath; boolean firstParam = true; for (String param : request.getParameters().keySet()) { if (firstParam) { urlString += "?"; firstParam = false; } else { urlString += "&"; } String value = request.getParameters().get(param); urlString += param + "=" + HttpUtils.urlEncode(value, false); } try { return new URL(urlString); } catch (MalformedURLException e) { throw new AmazonClientException( "Unable to convert request to well formed URL: " + e.getMessage(), e); } }
/* (non-Javadoc) * @see com.amazonaws.auth.Signer#sign(com.amazonaws.Request, com.amazonaws.auth.AWSCredentials) */ public void sign(Request<?> request, AWSCredentials credentials) throws AmazonClientException { // annonymous credentials, don't sign if (credentials instanceof AnonymousAWSCredentials) { return; } AWSCredentials sanitizedCredentials = sanitizeCredentials(credentials); if (sanitizedCredentials instanceof AWSSessionCredentials) { addSessionCredentials(request, (AWSSessionCredentials) sanitizedCredentials); } SimpleDateFormat dateStampFormat = new SimpleDateFormat("yyyyMMdd"); dateStampFormat.setTimeZone(new SimpleTimeZone(0, "UTC")); SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'"); dateTimeFormat.setTimeZone(new SimpleTimeZone(0, "UTC")); String regionName = extractRegionName(request.getEndpoint()); String serviceName = extractServiceName(request.getEndpoint()); // AWS4 requires that we sign the Host header so we // have to have it in the request by the time we sign. String hostHeader = request.getEndpoint().getHost(); if (HttpUtils.isUsingNonDefaultPort(request.getEndpoint())) { hostHeader += ":" + request.getEndpoint().getPort(); } request.addHeader("Host", hostHeader); Date date = new Date(); if (overriddenDate != null) date = overriddenDate; String dateTime = dateTimeFormat.format(date); String dateStamp = dateStampFormat.format(date); InputStream payloadStream = getBinaryRequestPayloadStream(request); payloadStream.mark(-1); String contentSha256 = BinaryUtils.toHex(hash(payloadStream)); try { payloadStream.reset(); } catch (IOException e) { throw new AmazonClientException("Unable to reset stream after calculating AWS4 signature", e); } request.addHeader("X-Amz-Date", dateTime); request.addHeader("x-amz-content-sha256", contentSha256); String canonicalRequest = request.getHttpMethod().toString() + "\n" + super.getCanonicalizedResourcePath(request.getResourcePath()) + "\n" + getCanonicalizedQueryString(request) + "\n" + getCanonicalizedHeaderString(request) + "\n" + getSignedHeadersString(request) + "\n" + contentSha256; log.debug("AWS4 Canonical Request: '\"" + canonicalRequest + "\""); String scope = dateStamp + "/" + regionName + "/" + serviceName + "/" + TERMINATOR; String signingCredentials = sanitizedCredentials.getAWSAccessKeyId() + "/" + scope; String stringToSign = ALGORITHM + "\n" + dateTime + "\n" + scope + "\n" + BinaryUtils.toHex(hash(canonicalRequest)); log.debug("AWS4 String to Sign: '\"" + stringToSign + "\""); // AWS4 uses a series of derived keys, formed by hashing different pieces of data byte[] kSecret = ("AWS4" + sanitizedCredentials.getAWSSecretKey()).getBytes(); byte[] kDate = sign(dateStamp, kSecret, SigningAlgorithm.HmacSHA256); byte[] kRegion = sign(regionName, kDate, SigningAlgorithm.HmacSHA256); byte[] kService = sign(serviceName, kRegion, SigningAlgorithm.HmacSHA256); byte[] kSigning = sign(TERMINATOR, kService, SigningAlgorithm.HmacSHA256); byte[] signature = sign(stringToSign.getBytes(), kSigning, SigningAlgorithm.HmacSHA256); String credentialsAuthorizationHeader = "Credential=" + signingCredentials; String signedHeadersAuthorizationHeader = "SignedHeaders=" + getSignedHeadersString(request); String signatureAuthorizationHeader = "Signature=" + BinaryUtils.toHex(signature); String authorizationHeader = ALGORITHM + " " + credentialsAuthorizationHeader + ", " + signedHeadersAuthorizationHeader + ", " + signatureAuthorizationHeader; request.addHeader("Authorization", authorizationHeader); }
@Override public void afterResponse(com.amazonaws.Request<?> request, com.amazonaws.Response<?> response) { try { if (response == null) { return; } Map<String, String> headers = response.getHttpResponse().getHeaders(); String responseTimeStr = headers.get(REQUESTTIME_HEADER); long responseTime = startTime - System.currentTimeMillis(); if (responseTimeStr != null && responseTimeStr.trim().length() > 0) { try { responseTime = Long.parseLong(responseTimeStr.trim()); } catch (NumberFormatException e) { } } double responseDouble = -1; try { responseDouble = Double.valueOf(Long.toString(responseTime)); } catch (NumberFormatException numberFormatException) { Log.e(TAG, "Couldn't convert response time to double format", numberFormatException); } String requestAttemptsStr = headers.get(REQUESTATTEMPTS_HEADER); double requestAttempts = 0; if (requestAttemptsStr != null && requestAttemptsStr.trim().length() > 0) { try { requestAttempts = Double.parseDouble(requestAttemptsStr.trim()); } catch (NumberFormatException e) { } } String serverInfo = headers.get(SERVERINFO_HEADER); if (!StringUtil.isNullOrEmpty(responseTimeStr)) { if (eventClient == null) { return; } AnalyticsEvent recordEvent = eventClient .createEvent("_httpRequestTiming") .withAttribute("url", request.getEndpoint().toURL().toString()) .withAttribute( "responseCode", Integer.toString(response.getHttpResponse().getStatusCode())) .withAttribute("timeZone", getTimeZone()) .withMetric("attempts", requestAttempts) .withMetric("totalTime", responseDouble) .withMetric("requestSize", (double) contentLength); String connectionType = "UNKNOWN"; if (connectivity != null) { if (connectivity.hasWifi()) { connectionType = "WIFI"; } else if (connectivity.hasWAN()) { connectionType = "WAN"; } } recordEvent.withAttribute("network", connectionType); if (serverInfo != null) { recordEvent.withAttribute("serverInfo", serverInfo); } eventClient.recordEvent(recordEvent); } } catch (Exception e) { Log.e(TAG, "Unable to record _RequestTime event", e); } }
/** * Internal method to execute the HTTP method given. * * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler) * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler, * ExecutionContext) */ private <T> Response<T> executeHelper( Request<?> request, HttpResponseHandler<AmazonWebServiceResponse<T>> responseHandler, HttpResponseHandler<AmazonServiceException> errorResponseHandler, ExecutionContext executionContext) throws AmazonClientException, AmazonServiceException { /* * Depending on which response handler we end up choosing to handle the * HTTP response, it might require us to leave the underlying HTTP * connection open, depending on whether or not it reads the complete * HTTP response stream from the HTTP connection, or if delays reading * any of the content until after a response is returned to the caller. */ boolean leaveHttpConnectionOpen = false; AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics(); /* add the service endpoint to the logs. You can infer service name from service endpoint */ awsRequestMetrics.addProperty(Field.ServiceName, request.getServiceName()); awsRequestMetrics.addProperty(Field.ServiceEndpoint, request.getEndpoint()); // Apply whatever request options we know how to handle, such as user-agent. setUserAgent(request); int requestCount = 0; URI redirectedURI = null; HttpEntity entity = null; AmazonClientException retriedException = null; // Make a copy of the original request params and headers so that we can // permute it in this loop and start over with the original every time. Map<String, String> originalParameters = new LinkedHashMap<String, String>(); originalParameters.putAll(request.getParameters()); Map<String, String> originalHeaders = new HashMap<String, String>(); originalHeaders.putAll(request.getHeaders()); final AWSCredentials credentials = executionContext.getCredentials(); Signer signer = null; while (true) { ++requestCount; awsRequestMetrics.setCounter(Field.RequestCount, requestCount); if (requestCount > 1) { // retry request.setParameters(originalParameters); request.setHeaders(originalHeaders); } HttpRequestBase httpRequest = null; org.apache.http.HttpResponse apacheResponse = null; try { // Sign the request if a signer was provided if (signer == null) signer = executionContext.getSignerByURI(request.getEndpoint()); if (signer != null && credentials != null) { awsRequestMetrics.startEvent(Field.RequestSigningTime); try { signer.sign(request, credentials); } finally { awsRequestMetrics.endEvent(Field.RequestSigningTime); } } if (requestLog.isDebugEnabled()) { requestLog.debug("Sending Request: " + request.toString()); } httpRequest = httpRequestFactory.createHttpRequest(request, config, executionContext); if (httpRequest instanceof HttpEntityEnclosingRequest) { entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity(); } if (redirectedURI != null) { httpRequest.setURI(redirectedURI); } if (requestCount > 1) { // retry awsRequestMetrics.startEvent(Field.RetryPauseTime); try { pauseBeforeNextRetry( request.getOriginalRequest(), retriedException, requestCount, config.getRetryPolicy()); } finally { awsRequestMetrics.endEvent(Field.RetryPauseTime); } } if (entity != null) { InputStream content = entity.getContent(); if (requestCount > 1) { // retry if (content.markSupported()) { content.reset(); content.mark(-1); } } else { if (content.markSupported()) { content.mark(-1); } } } captureConnectionPoolMetrics(httpClient.getConnectionManager(), awsRequestMetrics); HttpContext httpContext = new BasicHttpContext(); httpContext.setAttribute(AWSRequestMetrics.class.getSimpleName(), awsRequestMetrics); retriedException = null; awsRequestMetrics.startEvent(Field.HttpRequestTime); try { apacheResponse = httpClient.execute(httpRequest, httpContext); } finally { awsRequestMetrics.endEvent(Field.HttpRequestTime); } if (isRequestSuccessful(apacheResponse)) { awsRequestMetrics.addProperty( Field.StatusCode, apacheResponse.getStatusLine().getStatusCode()); /* * If we get back any 2xx status code, then we know we should * treat the service call as successful. */ leaveHttpConnectionOpen = responseHandler.needsConnectionLeftOpen(); HttpResponse httpResponse = createResponse(httpRequest, request, apacheResponse); T response = handleResponse( request, responseHandler, httpRequest, httpResponse, apacheResponse, executionContext); return new Response<T>(response, httpResponse); } else if (isTemporaryRedirect(apacheResponse)) { /* * S3 sends 307 Temporary Redirects if you try to delete an * EU bucket from the US endpoint. If we get a 307, we'll * point the HTTP method to the redirected location, and let * the next retry deliver the request to the right location. */ Header[] locationHeaders = apacheResponse.getHeaders("location"); String redirectedLocation = locationHeaders[0].getValue(); log.debug("Redirecting to: " + redirectedLocation); redirectedURI = URI.create(redirectedLocation); httpRequest.setURI(redirectedURI); awsRequestMetrics.addProperty( Field.StatusCode, apacheResponse.getStatusLine().getStatusCode()); awsRequestMetrics.addProperty(Field.RedirectLocation, redirectedLocation); awsRequestMetrics.addProperty(Field.AWSRequestID, null); } else { leaveHttpConnectionOpen = errorResponseHandler.needsConnectionLeftOpen(); AmazonServiceException ase = handleErrorResponse(request, errorResponseHandler, httpRequest, apacheResponse); awsRequestMetrics.addProperty(Field.AWSRequestID, ase.getRequestId()); awsRequestMetrics.addProperty(Field.AWSErrorCode, ase.getErrorCode()); awsRequestMetrics.addProperty(Field.StatusCode, ase.getStatusCode()); if (!shouldRetry( request.getOriginalRequest(), httpRequest, ase, requestCount, config.getRetryPolicy())) { throw ase; } // Cache the retryable exception retriedException = ase; /* * Checking for clock skew error again because we don't want to set the * global time offset for every service exception. */ if (RetryUtils.isClockSkewError(ase)) { int timeOffset = parseClockSkewOffset(apacheResponse, ase); SDKGlobalConfiguration.setGlobalTimeOffset(timeOffset); } resetRequestAfterError(request, ase); } } catch (IOException ioe) { if (log.isInfoEnabled()) { log.info("Unable to execute HTTP request: " + ioe.getMessage(), ioe); } awsRequestMetrics.incrementCounter(Field.Exception); awsRequestMetrics.addProperty(Field.Exception, ioe); awsRequestMetrics.addProperty(Field.AWSRequestID, null); AmazonClientException ace = new AmazonClientException("Unable to execute HTTP request: " + ioe.getMessage(), ioe); if (!shouldRetry( request.getOriginalRequest(), httpRequest, ace, requestCount, config.getRetryPolicy())) { throw ace; } // Cache the retryable exception retriedException = ace; resetRequestAfterError(request, ioe); } catch (RuntimeException e) { throw handleUnexpectedFailure(e, awsRequestMetrics); } catch (Error e) { throw handleUnexpectedFailure(e, awsRequestMetrics); } finally { /* * Some response handlers need to manually manage the HTTP * connection and will take care of releasing the connection on * their own, but if this response handler doesn't need the * connection left open, we go ahead and release the it to free * up resources. */ if (!leaveHttpConnectionOpen) { try { if (apacheResponse != null && apacheResponse.getEntity() != null && apacheResponse.getEntity().getContent() != null) { apacheResponse.getEntity().getContent().close(); } } catch (IOException e) { log.warn("Cannot close the response content.", e); } } } } /* end while (true) */ }
/** * Internal method to execute the HTTP method given. * * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler) * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler, * ExecutionContext) */ private <T extends Object> T executeHelper( Request<?> request, HttpResponseHandler<AmazonWebServiceResponse<T>> responseHandler, HttpResponseHandler<AmazonServiceException> errorResponseHandler, ExecutionContext executionContext) throws AmazonClientException, AmazonServiceException { /* * Depending on which response handler we end up choosing to handle the * HTTP response, it might require us to leave the underlying HTTP * connection open, depending on whether or not it reads the complete * HTTP response stream from the HTTP connection, or if delays reading * any of the content until after a response is returned to the caller. */ boolean leaveHttpConnectionOpen = false; AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics(); /* add the service endpoint to the logs. You can infer service name from service endpoint */ awsRequestMetrics.addProperty(Field.ServiceName.name(), request.getServiceName()); awsRequestMetrics.addProperty(Field.ServiceEndpoint.name(), request.getEndpoint()); // Apply whatever request options we know how to handle, such as user-agent. applyRequestData(request); int retryCount = 0; URI redirectedURI = null; HttpEntity entity = null; AmazonServiceException exception = null; // Make a copy of the original request params and headers so that we can // permute it in this loop and start over with the original every time. Map<String, String> originalParameters = new HashMap<String, String>(); originalParameters.putAll(request.getParameters()); Map<String, String> originalHeaders = new HashMap<String, String>(); originalHeaders.putAll(request.getHeaders()); while (true) { awsRequestMetrics.setCounter(Field.AttemptCount.name(), retryCount + 1); if (retryCount > 0) { request.setParameters(originalParameters); request.setHeaders(originalHeaders); } HttpRequestBase httpRequest = null; org.apache.http.HttpResponse response = null; try { // Sign the request if a signer was provided if (executionContext.getSigner() != null && executionContext.getCredentials() != null) { awsRequestMetrics.startEvent(Field.RequestSigningTime.name()); executionContext.getSigner().sign(request, executionContext.getCredentials()); awsRequestMetrics.endEvent(Field.RequestSigningTime.name()); } if (requestLog.isDebugEnabled()) { requestLog.debug("Sending Request: " + request.toString()); } httpRequest = httpRequestFactory.createHttpRequest(request, config, entity, executionContext); if (httpRequest instanceof HttpEntityEnclosingRequest) { entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity(); } if (redirectedURI != null) { httpRequest.setURI(redirectedURI); } if (retryCount > 0) { awsRequestMetrics.startEvent(Field.RetryPauseTime.name()); pauseExponentially(retryCount, exception, executionContext.getCustomBackoffStrategy()); awsRequestMetrics.endEvent(Field.RetryPauseTime.name()); } if (entity != null) { InputStream content = entity.getContent(); if (retryCount > 0) { if (content.markSupported()) { content.reset(); content.mark(-1); } } else { if (content.markSupported()) { content.mark(-1); } } } exception = null; awsRequestMetrics.startEvent(Field.HttpRequestTime.name()); response = httpClient.execute(httpRequest); awsRequestMetrics.endEvent(Field.HttpRequestTime.name()); if (isRequestSuccessful(response)) { awsRequestMetrics.addProperty( Field.StatusCode.name(), response.getStatusLine().getStatusCode()); /* * If we get back any 2xx status code, then we know we should * treat the service call as successful. */ leaveHttpConnectionOpen = responseHandler.needsConnectionLeftOpen(); return handleResponse(request, responseHandler, httpRequest, response, executionContext); } else if (isTemporaryRedirect(response)) { /* * S3 sends 307 Temporary Redirects if you try to delete an * EU bucket from the US endpoint. If we get a 307, we'll * point the HTTP method to the redirected location, and let * the next retry deliver the request to the right location. */ Header[] locationHeaders = response.getHeaders("location"); String redirectedLocation = locationHeaders[0].getValue(); log.debug("Redirecting to: " + redirectedLocation); redirectedURI = URI.create(redirectedLocation); httpRequest.setURI(redirectedURI); awsRequestMetrics.addProperty( Field.StatusCode.name(), response.getStatusLine().getStatusCode()); awsRequestMetrics.addProperty(Field.RedirectLocation.name(), redirectedLocation); awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null); } else { leaveHttpConnectionOpen = errorResponseHandler.needsConnectionLeftOpen(); exception = handleErrorResponse(request, errorResponseHandler, httpRequest, response); awsRequestMetrics.addProperty(Field.AWSRequestID.name(), exception.getRequestId()); awsRequestMetrics.addProperty(Field.AWSErrorCode.name(), exception.getErrorCode()); awsRequestMetrics.addProperty(Field.StatusCode.name(), exception.getStatusCode()); if (!shouldRetry(httpRequest, exception, retryCount)) { throw exception; } resetRequestAfterError(request, exception); } } catch (IOException ioe) { log.info("Unable to execute HTTP request: " + ioe.getMessage(), ioe); awsRequestMetrics.addProperty(Field.Exception.name(), ioe.toString()); awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null); if (!shouldRetry(httpRequest, ioe, retryCount)) { throw new AmazonClientException( "Unable to execute HTTP request: " + ioe.getMessage(), ioe); } resetRequestAfterError(request, ioe); } finally { retryCount++; /* * Some response handlers need to manually manage the HTTP * connection and will take care of releasing the connection on * their own, but if this response handler doesn't need the * connection left open, we go ahead and release the it to free * up resources. */ if (!leaveHttpConnectionOpen) { try { response.getEntity().getContent().close(); } catch (Throwable t) { } } } } /* end while (true) */ }