/** @see com.amazonaws.http.HttpResponseHandler#handle(com.amazonaws.http.HttpResponse) */
  public AmazonWebServiceResponse<S3Object> handle(HttpResponse response) throws Exception {
    /*
     * TODO: It'd be nice to set the bucket name and key here, but the
     *       information isn't easy to pull out of the response/request
     *       currently.
     */
    S3Object object = new S3Object();
    AmazonWebServiceResponse<S3Object> awsResponse = parseResponseMetadata(response);
    if (response.getHeaders().get(Headers.REDIRECT_LOCATION) != null) {
      object.setRedirectLocation(response.getHeaders().get(Headers.REDIRECT_LOCATION));
    }
    ObjectMetadata metadata = object.getObjectMetadata();
    populateObjectMetadata(response, metadata);
    boolean hasServerSideCalculatedChecksum =
        !ServiceUtils.isMultipartUploadETag(metadata.getETag());
    boolean responseContainsEntireObject = response.getHeaders().get("Content-Range") == null;

    if (hasServerSideCalculatedChecksum && responseContainsEntireObject) {
      byte[] expectedChecksum = BinaryUtils.fromHex(metadata.getETag());
      object.setObjectContent(
          new S3ObjectInputStream(
              new ChecksumValidatingInputStream(
                  response.getContent(),
                  expectedChecksum,
                  object.getBucketName() + "/" + object.getKey()),
              response.getHttpRequest()));
    } else {
      object.setObjectContent(
          new S3ObjectInputStream(response.getContent(), response.getHttpRequest()));
    }

    awsResponse.setResult(object);
    return awsResponse;
  }
Exemplo n.º 2
0
  /**
   * Downloads an S3Object, as returned from {@link
   * AmazonS3Client#getObject(com.amazonaws.services.s3.model.GetObjectRequest)}, to the specified
   * file.
   *
   * @param s3Object The S3Object containing a reference to an InputStream containing the object's
   *     data.
   * @param destinationFile The file to store the object's data in.
   * @param performIntegrityCheck Boolean valuable to indicate whether do the integrity check or not
   */
  public static void downloadObjectToFile(
      S3Object s3Object, File destinationFile, boolean performIntegrityCheck) {

    // attempt to create the parent if it doesn't exist
    File parentDirectory = destinationFile.getParentFile();
    if (parentDirectory != null && !parentDirectory.exists()) {
      parentDirectory.mkdirs();
    }

    OutputStream outputStream = null;
    try {
      outputStream = new BufferedOutputStream(new FileOutputStream(destinationFile));
      byte[] buffer = new byte[1024 * 10];
      int bytesRead;
      while ((bytesRead = s3Object.getObjectContent().read(buffer)) > -1) {
        outputStream.write(buffer, 0, bytesRead);
      }
    } catch (IOException e) {
      try {
        s3Object.getObjectContent().abort();
      } catch (IOException abortException) {
        log.warn("Couldn't abort stream", e);
      }
      throw new AmazonClientException(
          "Unable to store object contents to disk: " + e.getMessage(), e);
    } finally {
      try {
        outputStream.close();
      } catch (Exception e) {
      }
      try {
        s3Object.getObjectContent().close();
      } catch (Exception e) {
      }
    }

    byte[] clientSideHash = null;
    byte[] serverSideHash = null;
    try {
      // Multipart Uploads don't have an MD5 calculated on the service side
      if (ServiceUtils.isMultipartUploadETag(s3Object.getObjectMetadata().getETag()) == false) {
        clientSideHash = Md5Utils.computeMD5Hash(new FileInputStream(destinationFile));
        serverSideHash = BinaryUtils.fromHex(s3Object.getObjectMetadata().getETag());
      }
    } catch (Exception e) {
      log.warn("Unable to calculate MD5 hash to validate download: " + e.getMessage(), e);
    }

    if (performIntegrityCheck
        && clientSideHash != null
        && serverSideHash != null
        && !Arrays.equals(clientSideHash, serverSideHash)) {
      throw new AmazonClientException(
          "Unable to verify integrity of data download.  "
              + "Client calculated content hash didn't match hash calculated by Amazon S3.  "
              + "The data stored in '"
              + destinationFile.getAbsolutePath()
              + "' may be corrupt.");
    }
  }
Exemplo n.º 3
0
  /**
   * Gets an object stored in S3 and downloads it into the specified file. This method includes the
   * one-time retry mechanism after integrity check failure on the downloaded file. It will also
   * return immediately after getting null valued S3Object (when getObject request does not meet the
   * specified constraints).
   *
   * @param file The file to store the object's data in.
   * @param safeS3DownloadTask The implementation of SafeS3DownloadTask interface which allows user
   *     to get access to all the visible variables at the calling site of this method.
   */
  public static S3Object retryableDownloadS3ObjectToFile(
      File file, RetryableS3DownloadTask retryableS3DownloadTask) {
    boolean hasRetried = false;
    boolean needRetry;
    S3Object s3Object;
    do {
      needRetry = false;
      s3Object = retryableS3DownloadTask.getS3ObjectStream();
      if (s3Object == null) return null;

      try {
        ServiceUtils.downloadObjectToFile(
            s3Object, file, retryableS3DownloadTask.needIntegrityCheck());
      } catch (AmazonClientException ace) {
        // Determine whether an immediate retry is needed according to the captured
        // AmazonClientException.
        // (There are three cases when downloadObjectToFile() throws AmazonClientException:
        // 		1) SocketException or SSLProtocolException when writing to disk (e.g. when user aborts
        // the download)
        //		2) Other IOException when writing to disk
        //		3) MD5 hashes don't match
        // The current code will retry the download only when case 2) or 3) happens.
        if (ace.getCause() instanceof SocketException
            || ace.getCause() instanceof SSLProtocolException) {
          throw ace;
        } else {
          needRetry = true;
          if (hasRetried) throw ace;
          else {
            log.info(
                "Retry the download of object "
                    + s3Object.getKey()
                    + " (bucket "
                    + s3Object.getBucketName()
                    + ")",
                ace);
            hasRetried = true;
          }
        }
      } finally {
        try {
          s3Object.getObjectContent().abort();
        } catch (IOException e) {
        }
      }
    } while (needRetry);
    return s3Object;
  }