Пример #1
0
  /**
   * Converts the specified request object into a URL, containing all the specified parameters, the
   * specified request endpoint, etc.
   *
   * @param request The request to convert into a URL.
   * @param removeLeadingSlashInResourcePath Whether the leading slash in resource-path should be
   *     removed before appending to the endpoint.
   * @return A new URL representing the specified request.
   * @throws AmazonClientException If the request cannot be converted to a well formed URL.
   */
  public static URL convertRequestToUrl(
      Request<?> request, boolean removeLeadingSlashInResourcePath) {
    String resourcePath = HttpUtils.urlEncode(request.getResourcePath(), true);

    // Removed the padding "/" that was already added into the request's resource path.
    if (removeLeadingSlashInResourcePath && resourcePath.startsWith("/")) {
      resourcePath = resourcePath.substring(1);
    }

    // Some http client libraries (e.g. Apache HttpClient) cannot handle
    // consecutive "/"s between URL authority and path components.
    // So we escape "////..." into "/%2F%2F%2F...", in the same way as how
    // we treat consecutive "/"s in AmazonS3Client#presignRequest(...)
    String urlPath = "/" + resourcePath;
    urlPath = urlPath.replaceAll("(?<=/)/", "%2F");
    String urlString = request.getEndpoint() + urlPath;

    boolean firstParam = true;
    for (String param : request.getParameters().keySet()) {
      if (firstParam) {
        urlString += "?";
        firstParam = false;
      } else {
        urlString += "&";
      }

      String value = request.getParameters().get(param);
      urlString += param + "=" + HttpUtils.urlEncode(value, false);
    }

    try {
      return new URL(urlString);
    } catch (MalformedURLException e) {
      throw new AmazonClientException(
          "Unable to convert request to well formed URL: " + e.getMessage(), e);
    }
  }
Пример #2
0
  /* (non-Javadoc)
   * @see com.amazonaws.auth.Signer#sign(com.amazonaws.Request, com.amazonaws.auth.AWSCredentials)
   */
  public void sign(Request<?> request, AWSCredentials credentials) throws AmazonClientException {
    // annonymous credentials, don't sign
    if (credentials instanceof AnonymousAWSCredentials) {
      return;
    }

    AWSCredentials sanitizedCredentials = sanitizeCredentials(credentials);
    if (sanitizedCredentials instanceof AWSSessionCredentials) {
      addSessionCredentials(request, (AWSSessionCredentials) sanitizedCredentials);
    }

    SimpleDateFormat dateStampFormat = new SimpleDateFormat("yyyyMMdd");
    dateStampFormat.setTimeZone(new SimpleTimeZone(0, "UTC"));

    SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'");
    dateTimeFormat.setTimeZone(new SimpleTimeZone(0, "UTC"));

    String regionName = extractRegionName(request.getEndpoint());
    String serviceName = extractServiceName(request.getEndpoint());

    // AWS4 requires that we sign the Host header so we
    // have to have it in the request by the time we sign.
    String hostHeader = request.getEndpoint().getHost();
    if (HttpUtils.isUsingNonDefaultPort(request.getEndpoint())) {
      hostHeader += ":" + request.getEndpoint().getPort();
    }
    request.addHeader("Host", hostHeader);

    Date date = new Date();
    if (overriddenDate != null) date = overriddenDate;

    String dateTime = dateTimeFormat.format(date);
    String dateStamp = dateStampFormat.format(date);

    InputStream payloadStream = getBinaryRequestPayloadStream(request);
    payloadStream.mark(-1);
    String contentSha256 = BinaryUtils.toHex(hash(payloadStream));
    try {
      payloadStream.reset();
    } catch (IOException e) {
      throw new AmazonClientException("Unable to reset stream after calculating AWS4 signature", e);
    }

    request.addHeader("X-Amz-Date", dateTime);
    request.addHeader("x-amz-content-sha256", contentSha256);

    String canonicalRequest =
        request.getHttpMethod().toString()
            + "\n"
            + super.getCanonicalizedResourcePath(request.getResourcePath())
            + "\n"
            + getCanonicalizedQueryString(request)
            + "\n"
            + getCanonicalizedHeaderString(request)
            + "\n"
            + getSignedHeadersString(request)
            + "\n"
            + contentSha256;

    log.debug("AWS4 Canonical Request: '\"" + canonicalRequest + "\"");

    String scope = dateStamp + "/" + regionName + "/" + serviceName + "/" + TERMINATOR;
    String signingCredentials = sanitizedCredentials.getAWSAccessKeyId() + "/" + scope;
    String stringToSign =
        ALGORITHM
            + "\n"
            + dateTime
            + "\n"
            + scope
            + "\n"
            + BinaryUtils.toHex(hash(canonicalRequest));
    log.debug("AWS4 String to Sign: '\"" + stringToSign + "\"");

    // AWS4 uses a series of derived keys, formed by hashing different pieces of data
    byte[] kSecret = ("AWS4" + sanitizedCredentials.getAWSSecretKey()).getBytes();
    byte[] kDate = sign(dateStamp, kSecret, SigningAlgorithm.HmacSHA256);
    byte[] kRegion = sign(regionName, kDate, SigningAlgorithm.HmacSHA256);
    byte[] kService = sign(serviceName, kRegion, SigningAlgorithm.HmacSHA256);
    byte[] kSigning = sign(TERMINATOR, kService, SigningAlgorithm.HmacSHA256);

    byte[] signature = sign(stringToSign.getBytes(), kSigning, SigningAlgorithm.HmacSHA256);

    String credentialsAuthorizationHeader = "Credential=" + signingCredentials;
    String signedHeadersAuthorizationHeader = "SignedHeaders=" + getSignedHeadersString(request);
    String signatureAuthorizationHeader = "Signature=" + BinaryUtils.toHex(signature);

    String authorizationHeader =
        ALGORITHM
            + " "
            + credentialsAuthorizationHeader
            + ", "
            + signedHeadersAuthorizationHeader
            + ", "
            + signatureAuthorizationHeader;

    request.addHeader("Authorization", authorizationHeader);
  }
  @Override
  public void afterResponse(com.amazonaws.Request<?> request, com.amazonaws.Response<?> response) {
    try {
      if (response == null) {
        return;
      }

      Map<String, String> headers = response.getHttpResponse().getHeaders();
      String responseTimeStr = headers.get(REQUESTTIME_HEADER);
      long responseTime = startTime - System.currentTimeMillis();
      if (responseTimeStr != null && responseTimeStr.trim().length() > 0) {
        try {
          responseTime = Long.parseLong(responseTimeStr.trim());
        } catch (NumberFormatException e) {
        }
      }

      double responseDouble = -1;
      try {
        responseDouble = Double.valueOf(Long.toString(responseTime));
      } catch (NumberFormatException numberFormatException) {
        Log.e(TAG, "Couldn't convert response time to double format", numberFormatException);
      }
      String requestAttemptsStr = headers.get(REQUESTATTEMPTS_HEADER);
      double requestAttempts = 0;
      if (requestAttemptsStr != null && requestAttemptsStr.trim().length() > 0) {
        try {
          requestAttempts = Double.parseDouble(requestAttemptsStr.trim());
        } catch (NumberFormatException e) {
        }
      }

      String serverInfo = headers.get(SERVERINFO_HEADER);
      if (!StringUtil.isNullOrEmpty(responseTimeStr)) {
        if (eventClient == null) {
          return;
        }
        AnalyticsEvent recordEvent =
            eventClient
                .createEvent("_httpRequestTiming")
                .withAttribute("url", request.getEndpoint().toURL().toString())
                .withAttribute(
                    "responseCode", Integer.toString(response.getHttpResponse().getStatusCode()))
                .withAttribute("timeZone", getTimeZone())
                .withMetric("attempts", requestAttempts)
                .withMetric("totalTime", responseDouble)
                .withMetric("requestSize", (double) contentLength);

        String connectionType = "UNKNOWN";
        if (connectivity != null) {
          if (connectivity.hasWifi()) {
            connectionType = "WIFI";
          } else if (connectivity.hasWAN()) {
            connectionType = "WAN";
          }
        }

        recordEvent.withAttribute("network", connectionType);

        if (serverInfo != null) {
          recordEvent.withAttribute("serverInfo", serverInfo);
        }

        eventClient.recordEvent(recordEvent);
      }
    } catch (Exception e) {
      Log.e(TAG, "Unable to record _RequestTime event", e);
    }
  }
Пример #4
0
  /**
   * Internal method to execute the HTTP method given.
   *
   * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler)
   * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler,
   *     ExecutionContext)
   */
  private <T> Response<T> executeHelper(
      Request<?> request,
      HttpResponseHandler<AmazonWebServiceResponse<T>> responseHandler,
      HttpResponseHandler<AmazonServiceException> errorResponseHandler,
      ExecutionContext executionContext)
      throws AmazonClientException, AmazonServiceException {
    /*
     * Depending on which response handler we end up choosing to handle the
     * HTTP response, it might require us to leave the underlying HTTP
     * connection open, depending on whether or not it reads the complete
     * HTTP response stream from the HTTP connection, or if delays reading
     * any of the content until after a response is returned to the caller.
     */
    boolean leaveHttpConnectionOpen = false;
    AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics();
    /* add the service endpoint to the logs. You can infer service name from service endpoint */
    awsRequestMetrics.addProperty(Field.ServiceName, request.getServiceName());
    awsRequestMetrics.addProperty(Field.ServiceEndpoint, request.getEndpoint());
    // Apply whatever request options we know how to handle, such as user-agent.
    setUserAgent(request);
    int requestCount = 0;
    URI redirectedURI = null;
    HttpEntity entity = null;
    AmazonClientException retriedException = null;

    // Make a copy of the original request params and headers so that we can
    // permute it in this loop and start over with the original every time.
    Map<String, String> originalParameters = new LinkedHashMap<String, String>();
    originalParameters.putAll(request.getParameters());
    Map<String, String> originalHeaders = new HashMap<String, String>();
    originalHeaders.putAll(request.getHeaders());
    final AWSCredentials credentials = executionContext.getCredentials();
    Signer signer = null;

    while (true) {
      ++requestCount;
      awsRequestMetrics.setCounter(Field.RequestCount, requestCount);
      if (requestCount > 1) { // retry
        request.setParameters(originalParameters);
        request.setHeaders(originalHeaders);
      }
      HttpRequestBase httpRequest = null;
      org.apache.http.HttpResponse apacheResponse = null;

      try {
        // Sign the request if a signer was provided
        if (signer == null) signer = executionContext.getSignerByURI(request.getEndpoint());
        if (signer != null && credentials != null) {
          awsRequestMetrics.startEvent(Field.RequestSigningTime);
          try {
            signer.sign(request, credentials);
          } finally {
            awsRequestMetrics.endEvent(Field.RequestSigningTime);
          }
        }

        if (requestLog.isDebugEnabled()) {
          requestLog.debug("Sending Request: " + request.toString());
        }

        httpRequest = httpRequestFactory.createHttpRequest(request, config, executionContext);

        if (httpRequest instanceof HttpEntityEnclosingRequest) {
          entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity();
        }

        if (redirectedURI != null) {
          httpRequest.setURI(redirectedURI);
        }

        if (requestCount > 1) { // retry
          awsRequestMetrics.startEvent(Field.RetryPauseTime);
          try {
            pauseBeforeNextRetry(
                request.getOriginalRequest(),
                retriedException,
                requestCount,
                config.getRetryPolicy());
          } finally {
            awsRequestMetrics.endEvent(Field.RetryPauseTime);
          }
        }

        if (entity != null) {
          InputStream content = entity.getContent();
          if (requestCount > 1) { // retry
            if (content.markSupported()) {
              content.reset();
              content.mark(-1);
            }
          } else {
            if (content.markSupported()) {
              content.mark(-1);
            }
          }
        }

        captureConnectionPoolMetrics(httpClient.getConnectionManager(), awsRequestMetrics);
        HttpContext httpContext = new BasicHttpContext();
        httpContext.setAttribute(AWSRequestMetrics.class.getSimpleName(), awsRequestMetrics);
        retriedException = null;
        awsRequestMetrics.startEvent(Field.HttpRequestTime);
        try {
          apacheResponse = httpClient.execute(httpRequest, httpContext);
        } finally {
          awsRequestMetrics.endEvent(Field.HttpRequestTime);
        }

        if (isRequestSuccessful(apacheResponse)) {
          awsRequestMetrics.addProperty(
              Field.StatusCode, apacheResponse.getStatusLine().getStatusCode());
          /*
           * If we get back any 2xx status code, then we know we should
           * treat the service call as successful.
           */
          leaveHttpConnectionOpen = responseHandler.needsConnectionLeftOpen();
          HttpResponse httpResponse = createResponse(httpRequest, request, apacheResponse);
          T response =
              handleResponse(
                  request,
                  responseHandler,
                  httpRequest,
                  httpResponse,
                  apacheResponse,
                  executionContext);
          return new Response<T>(response, httpResponse);
        } else if (isTemporaryRedirect(apacheResponse)) {
          /*
           * S3 sends 307 Temporary Redirects if you try to delete an
           * EU bucket from the US endpoint. If we get a 307, we'll
           * point the HTTP method to the redirected location, and let
           * the next retry deliver the request to the right location.
           */
          Header[] locationHeaders = apacheResponse.getHeaders("location");
          String redirectedLocation = locationHeaders[0].getValue();
          log.debug("Redirecting to: " + redirectedLocation);
          redirectedURI = URI.create(redirectedLocation);
          httpRequest.setURI(redirectedURI);
          awsRequestMetrics.addProperty(
              Field.StatusCode, apacheResponse.getStatusLine().getStatusCode());
          awsRequestMetrics.addProperty(Field.RedirectLocation, redirectedLocation);
          awsRequestMetrics.addProperty(Field.AWSRequestID, null);

        } else {
          leaveHttpConnectionOpen = errorResponseHandler.needsConnectionLeftOpen();
          AmazonServiceException ase =
              handleErrorResponse(request, errorResponseHandler, httpRequest, apacheResponse);
          awsRequestMetrics.addProperty(Field.AWSRequestID, ase.getRequestId());
          awsRequestMetrics.addProperty(Field.AWSErrorCode, ase.getErrorCode());
          awsRequestMetrics.addProperty(Field.StatusCode, ase.getStatusCode());

          if (!shouldRetry(
              request.getOriginalRequest(),
              httpRequest,
              ase,
              requestCount,
              config.getRetryPolicy())) {
            throw ase;
          }

          // Cache the retryable exception
          retriedException = ase;
          /*
           * Checking for clock skew error again because we don't want to set the
           * global time offset for every service exception.
           */
          if (RetryUtils.isClockSkewError(ase)) {
            int timeOffset = parseClockSkewOffset(apacheResponse, ase);
            SDKGlobalConfiguration.setGlobalTimeOffset(timeOffset);
          }
          resetRequestAfterError(request, ase);
        }
      } catch (IOException ioe) {
        if (log.isInfoEnabled()) {
          log.info("Unable to execute HTTP request: " + ioe.getMessage(), ioe);
        }
        awsRequestMetrics.incrementCounter(Field.Exception);
        awsRequestMetrics.addProperty(Field.Exception, ioe);
        awsRequestMetrics.addProperty(Field.AWSRequestID, null);

        AmazonClientException ace =
            new AmazonClientException("Unable to execute HTTP request: " + ioe.getMessage(), ioe);
        if (!shouldRetry(
            request.getOriginalRequest(),
            httpRequest,
            ace,
            requestCount,
            config.getRetryPolicy())) {
          throw ace;
        }

        // Cache the retryable exception
        retriedException = ace;
        resetRequestAfterError(request, ioe);
      } catch (RuntimeException e) {
        throw handleUnexpectedFailure(e, awsRequestMetrics);
      } catch (Error e) {
        throw handleUnexpectedFailure(e, awsRequestMetrics);
      } finally {
        /*
         * Some response handlers need to manually manage the HTTP
         * connection and will take care of releasing the connection on
         * their own, but if this response handler doesn't need the
         * connection left open, we go ahead and release the it to free
         * up resources.
         */
        if (!leaveHttpConnectionOpen) {
          try {
            if (apacheResponse != null
                && apacheResponse.getEntity() != null
                && apacheResponse.getEntity().getContent() != null) {
              apacheResponse.getEntity().getContent().close();
            }
          } catch (IOException e) {
            log.warn("Cannot close the response content.", e);
          }
        }
      }
    } /* end while (true) */
  }
  /**
   * Internal method to execute the HTTP method given.
   *
   * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler)
   * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler,
   *     ExecutionContext)
   */
  private <T extends Object> T executeHelper(
      Request<?> request,
      HttpResponseHandler<AmazonWebServiceResponse<T>> responseHandler,
      HttpResponseHandler<AmazonServiceException> errorResponseHandler,
      ExecutionContext executionContext)
      throws AmazonClientException, AmazonServiceException {

    /*
     * Depending on which response handler we end up choosing to handle the
     * HTTP response, it might require us to leave the underlying HTTP
     * connection open, depending on whether or not it reads the complete
     * HTTP response stream from the HTTP connection, or if delays reading
     * any of the content until after a response is returned to the caller.
     */
    boolean leaveHttpConnectionOpen = false;

    AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics();
    /* add the service endpoint to the logs. You can infer service name from service endpoint */
    awsRequestMetrics.addProperty(Field.ServiceName.name(), request.getServiceName());
    awsRequestMetrics.addProperty(Field.ServiceEndpoint.name(), request.getEndpoint());

    // Apply whatever request options we know how to handle, such as user-agent.
    applyRequestData(request);

    int retryCount = 0;
    URI redirectedURI = null;
    HttpEntity entity = null;
    AmazonServiceException exception = null;

    // Make a copy of the original request params and headers so that we can
    // permute it in this loop and start over with the original every time.
    Map<String, String> originalParameters = new HashMap<String, String>();
    originalParameters.putAll(request.getParameters());
    Map<String, String> originalHeaders = new HashMap<String, String>();
    originalHeaders.putAll(request.getHeaders());

    while (true) {
      awsRequestMetrics.setCounter(Field.AttemptCount.name(), retryCount + 1);
      if (retryCount > 0) {
        request.setParameters(originalParameters);
        request.setHeaders(originalHeaders);
      }

      HttpRequestBase httpRequest = null;
      org.apache.http.HttpResponse response = null;

      try {
        // Sign the request if a signer was provided
        if (executionContext.getSigner() != null && executionContext.getCredentials() != null) {
          awsRequestMetrics.startEvent(Field.RequestSigningTime.name());
          executionContext.getSigner().sign(request, executionContext.getCredentials());
          awsRequestMetrics.endEvent(Field.RequestSigningTime.name());
        }

        if (requestLog.isDebugEnabled()) {
          requestLog.debug("Sending Request: " + request.toString());
        }

        httpRequest =
            httpRequestFactory.createHttpRequest(request, config, entity, executionContext);

        if (httpRequest instanceof HttpEntityEnclosingRequest) {
          entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity();
        }

        if (redirectedURI != null) {
          httpRequest.setURI(redirectedURI);
        }

        if (retryCount > 0) {
          awsRequestMetrics.startEvent(Field.RetryPauseTime.name());
          pauseExponentially(retryCount, exception, executionContext.getCustomBackoffStrategy());
          awsRequestMetrics.endEvent(Field.RetryPauseTime.name());
        }

        if (entity != null) {
          InputStream content = entity.getContent();
          if (retryCount > 0) {
            if (content.markSupported()) {
              content.reset();
              content.mark(-1);
            }
          } else {
            if (content.markSupported()) {
              content.mark(-1);
            }
          }
        }

        exception = null;

        awsRequestMetrics.startEvent(Field.HttpRequestTime.name());
        response = httpClient.execute(httpRequest);
        awsRequestMetrics.endEvent(Field.HttpRequestTime.name());

        if (isRequestSuccessful(response)) {

          awsRequestMetrics.addProperty(
              Field.StatusCode.name(), response.getStatusLine().getStatusCode());

          /*
           * If we get back any 2xx status code, then we know we should
           * treat the service call as successful.
           */
          leaveHttpConnectionOpen = responseHandler.needsConnectionLeftOpen();
          return handleResponse(request, responseHandler, httpRequest, response, executionContext);
        } else if (isTemporaryRedirect(response)) {
          /*
           * S3 sends 307 Temporary Redirects if you try to delete an
           * EU bucket from the US endpoint. If we get a 307, we'll
           * point the HTTP method to the redirected location, and let
           * the next retry deliver the request to the right location.
           */
          Header[] locationHeaders = response.getHeaders("location");
          String redirectedLocation = locationHeaders[0].getValue();
          log.debug("Redirecting to: " + redirectedLocation);
          redirectedURI = URI.create(redirectedLocation);
          httpRequest.setURI(redirectedURI);
          awsRequestMetrics.addProperty(
              Field.StatusCode.name(), response.getStatusLine().getStatusCode());
          awsRequestMetrics.addProperty(Field.RedirectLocation.name(), redirectedLocation);
          awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null);

        } else {
          leaveHttpConnectionOpen = errorResponseHandler.needsConnectionLeftOpen();
          exception = handleErrorResponse(request, errorResponseHandler, httpRequest, response);
          awsRequestMetrics.addProperty(Field.AWSRequestID.name(), exception.getRequestId());
          awsRequestMetrics.addProperty(Field.AWSErrorCode.name(), exception.getErrorCode());
          awsRequestMetrics.addProperty(Field.StatusCode.name(), exception.getStatusCode());

          if (!shouldRetry(httpRequest, exception, retryCount)) {
            throw exception;
          }
          resetRequestAfterError(request, exception);
        }
      } catch (IOException ioe) {
        log.info("Unable to execute HTTP request: " + ioe.getMessage(), ioe);
        awsRequestMetrics.addProperty(Field.Exception.name(), ioe.toString());
        awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null);

        if (!shouldRetry(httpRequest, ioe, retryCount)) {
          throw new AmazonClientException(
              "Unable to execute HTTP request: " + ioe.getMessage(), ioe);
        }
        resetRequestAfterError(request, ioe);
      } finally {
        retryCount++;

        /*
         * Some response handlers need to manually manage the HTTP
         * connection and will take care of releasing the connection on
         * their own, but if this response handler doesn't need the
         * connection left open, we go ahead and release the it to free
         * up resources.
         */
        if (!leaveHttpConnectionOpen) {
          try {
            response.getEntity().getContent().close();
          } catch (Throwable t) {
          }
        }
      }
    } /* end while (true) */
  }