Example #1
0
 private S3Object getS3Object(final Path path, final long start) throws IOException {
   try {
     return retry()
         .maxAttempts(maxClientRetry)
         .exponentialBackoff(
             new Duration(1, TimeUnit.SECONDS), maxBackoffTime, maxRetryTime, 2.0)
         .stopOn(InterruptedException.class, UnrecoverableS3OperationException.class)
         .run(
             "getS3Object",
             () -> {
               try {
                 return s3.getObject(
                     new GetObjectRequest(host, keyFromPath(path))
                         .withRange(start, Long.MAX_VALUE));
               } catch (AmazonServiceException e) {
                 if (e.getStatusCode() == SC_FORBIDDEN) {
                   throw new UnrecoverableS3OperationException(e);
                 }
                 throw Throwables.propagate(e);
               }
             });
   } catch (InterruptedException e) {
     Thread.currentThread().interrupt();
     throw Throwables.propagate(e);
   } catch (Exception e) {
     Throwables.propagateIfInstanceOf(e, IOException.class);
     throw Throwables.propagate(e);
   }
 }
Example #2
0
  private static void createKey(String keyName, AmazonEC2 ec2) {
    try {
      List<KeyPairInfo> keyPairList = ec2.describeKeyPairs().getKeyPairs();
      for (KeyPairInfo keyPair : keyPairList) {
        if (keyName.equalsIgnoreCase(keyPair.getKeyName())) {
          System.out.println("Using key " + keyName);
          return;
        }
      }
      System.out.println("Creating key " + keyName + "in local directory");
      CreateKeyPairRequest newKeyRequest = new CreateKeyPairRequest();
      newKeyRequest.setKeyName(keyName);
      CreateKeyPairResult keyresult = ec2.createKeyPair(newKeyRequest);
      KeyPair keyPair = new KeyPair();
      keyPair = keyresult.getKeyPair();
      String privateKey = keyPair.getKeyMaterial();
      writeKeytoFile(keyName, privateKey);

    } catch (AmazonServiceException ase) {
      System.out.println("Caught Exception: " + ase.getMessage());
      System.out.println("Reponse Status Code: " + ase.getStatusCode());
      System.out.println("Error Code: " + ase.getErrorCode());
      System.out.println("Request ID: " + ase.getRequestId());
    }
  }
Example #3
0
  public static void createBucket(AmazonS3Client s3, String bucketName, String zone) {
    try {

      List<Bucket> bucketList = s3.listBuckets();

      for (Bucket bucket : bucketList) {
        // System.out.println(bucket.getName());
        if (bucketName.equalsIgnoreCase(bucket.getName())) {
          System.out.println("Using bucket " + bucketName);
          return;
        }
      }

      System.out.println("Created s3 bucket " + bucketName);
      s3.createBucket(bucketName);

      return;

    } catch (AmazonServiceException ase) {
      System.out.println("Caught Exception: " + ase.getMessage());
      System.out.println("Reponse Status Code: " + ase.getStatusCode());
      System.out.println("Error Code: " + ase.getErrorCode());
      System.out.println("Request ID: " + ase.getRequestId());
    }
  }
 private void updateItemVersionedInternal(
     String tableName,
     Key key,
     Map<String, AttributeValueUpdate> attributes,
     String expectedVersion,
     PersistentEntity persistentEntity,
     int attempt)
     throws DataAccessException {
   UpdateItemRequest request =
       new UpdateItemRequest(tableName, key, attributes)
           .withExpected(getOptimisticVersionCondition(expectedVersion));
   try {
     ddb.updateItem(request);
   } catch (AmazonServiceException e) {
     if (DynamoDBUtil.AWS_ERR_CODE_CONDITIONAL_CHECK_FAILED.equals(e.getErrorCode())) {
       throw new OptimisticLockingException(persistentEntity, key);
     } else if (DynamoDBUtil.AWS_ERR_CODE_RESOURCE_NOT_FOUND.equals(e.getErrorCode())) {
       throw new IllegalArgumentException("no such table: " + tableName, e);
     } else if (DynamoDBUtil.AWS_STATUS_CODE_SERVICE_UNAVAILABLE == e.getStatusCode()) {
       // retry after a small pause
       DynamoDBUtil.sleepBeforeRetry(attempt);
       attempt++;
       updateItemVersionedInternal(
           tableName, key, attributes, expectedVersion, persistentEntity, attempt);
     } else {
       throw new DataStoreOperationException(
           "problem with table: " + tableName + ", key: " + key + ", attributes: " + attributes,
           e);
     }
   }
 }
Example #5
0
  public LinkedList<String> listNames(String prefix, String bucketName, String[] canonicalIDs)
      throws StorageCloudException {
    LinkedList<String> find = new LinkedList<String>();
    try {
      ObjectListing objectListing = null;
      if (bucketName == null)
        objectListing =
            conn.listObjects(
                new ListObjectsRequest().withBucketName(defaultBucketName).withPrefix(prefix));
      else {
        bucketName = bucketName.concat(location);
        objectListing =
            conn.listObjects(
                new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix));
      }
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        find.add(objectSummary.getKey());
      }
    } catch (AmazonServiceException e1) {
      throw new ServiceSiteException("AWSS3Exception::" + e1.getMessage());
    } catch (AmazonClientException e2) {
      throw new ClientServiceException("AWSS3Exception::" + e2.getMessage());
    }

    return find;
  }
 private Map<String, AttributeValue> getConsistentInternal(
     String tableName, Key key, int attempt) {
   GetItemRequest request = new GetItemRequest(tableName, key);
   request.setConsistentRead(true);
   try {
     GetItemResult result = ddb.getItem(request);
     Map<String, AttributeValue> attributes = result.getItem();
     if (attributes == null || attributes.isEmpty()) {
       return null;
     }
     return attributes;
   } catch (AmazonServiceException e) {
     if (DynamoDBUtil.AWS_ERR_CODE_RESOURCE_NOT_FOUND.equals(e.getErrorCode())) {
       throw new IllegalArgumentException("no such table: " + tableName, e);
     } else if (DynamoDBUtil.AWS_STATUS_CODE_SERVICE_UNAVAILABLE == e.getStatusCode()) {
       // retry after a small pause
       DynamoDBUtil.sleepBeforeRetry(attempt);
       attempt++;
       return getConsistentInternal(tableName, key, attempt);
     } else {
       throw new DataStoreOperationException(
           "problem with table: " + tableName + ", key: " + key, e);
     }
   }
 }
Example #7
0
  public String[] setAcl(String bucketNameToShare, String[] canonicalId, String permission)
      throws StorageCloudException {
    boolean withRead = false;
    if (bucketNameToShare != null) {
      bucketNameToShare = bucketNameToShare.concat(location);
      if (!conn.doesBucketExist(bucketNameToShare)) {
        conn.createBucket(bucketNameToShare, region);
      }
    } else {
      return null;
    }

    // set acl
    AccessControlList acl = conn.getBucketAcl(bucketNameToShare);
    for (int i = 0; i < canonicalId.length; i++) {
      if (permission.equals("rw")) {
        CanonicalGrantee grantee = new CanonicalGrantee(canonicalId[i]);
        acl.grantPermission(grantee, Permission.Read);
        acl.grantPermission(grantee, Permission.Write);
        withRead = true;
      } else if (permission.equals("r")) {
        acl.grantPermission(new CanonicalGrantee(canonicalId[i]), Permission.Read);
        withRead = true;
      } else if (permission.equals("w")) {
        acl.grantPermission(new CanonicalGrantee(canonicalId[i]), Permission.Write);
      }
    }
    try {
      if (withRead) {
        ObjectListing objectListing = conn.listObjects(bucketNameToShare);
        AccessControlList aclKeys = null;
        for (S3ObjectSummary elem : objectListing.getObjectSummaries()) {
          aclKeys = conn.getObjectAcl(bucketNameToShare, elem.getKey());
          for (int i = 0; i < canonicalId.length; i++) {
            aclKeys.grantPermission(new CanonicalGrantee(canonicalId[i]), Permission.Read);
          }
          conn.setObjectAcl(bucketNameToShare, elem.getKey(), aclKeys);
        }
      }

      // confirm if acl well
      conn.setBucketAcl(bucketNameToShare, acl);
      AccessControlList newAcl = conn.getBucketAcl(bucketNameToShare);
      Set<Grant> grants = newAcl.getGrants();
      boolean flag = false;
      for (Grant grant : grants) {
        if (grant.getGrantee().getIdentifier().equals(canonicalId[0])) {
          flag = true;
        }
      }
      if (!flag) {
        throw new ServiceSiteException("AWSS3Exception:: ACL");
      }
    } catch (AmazonServiceException e1) {
      throw new ServiceSiteException("AWSS3Exception::" + e1.getMessage());
    } catch (AmazonClientException e2) {
      throw new ClientServiceException("AWSS3Exception::" + e2.getMessage());
    }
    return canonicalId;
  }
Example #8
0
  @Override
  public int read(
      String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) {

    logger.debug("readkey: " + key + " from table: " + table);
    GetItemRequest req = new GetItemRequest(table, createPrimaryKey(key));
    req.setAttributesToGet(fields);
    req.setConsistentRead(consistentRead);
    GetItemResult res = null;

    try {
      res = dynamoDB.getItem(req);
    } catch (AmazonServiceException ex) {
      logger.error(ex.getMessage());
      return SERVER_ERROR;
    } catch (AmazonClientException ex) {
      logger.error(ex.getMessage());
      return CLIENT_ERROR;
    }

    if (null != res.getItem()) {
      result.putAll(extractResult(res.getItem()));
      logger.debug("Result: " + res.toString());
    }
    return OK;
  }
  private static void waitForTableToBecomeAvailable(AmazonDynamoDB dynamo, String tableName) {
    System.out.println("Waiting for " + tableName + " to become ACTIVE...");

    long startTime = System.currentTimeMillis();
    long endTime = startTime + (10 * 60 * 1000);
    while (System.currentTimeMillis() < endTime) {
      try {
        Thread.sleep(1000 * 20);
      } catch (Exception e) {
      }
      try {
        DescribeTableRequest request = new DescribeTableRequest().withTableName(tableName);
        TableDescription table = dynamo.describeTable(request).getTable();
        if (table == null) continue;

        String tableStatus = table.getTableStatus();
        System.out.println("  - current state: " + tableStatus);
        if (tableStatus.equals(TableStatus.ACTIVE.toString())) return;
      } catch (AmazonServiceException ase) {
        if (ase.getErrorCode().equalsIgnoreCase("ResourceNotFoundException") == false) throw ase;
      }
    }

    throw new RuntimeException("Table " + tableName + " never went active");
  }
 @Override
 public AmazonServiceException handle(com.amazonaws.http.HttpResponse response)
     throws Exception {
   AmazonServiceException ase = new AmazonServiceException("Fake service exception.");
   ase.setStatusCode(response.getStatusCode());
   ase.setErrorCode(response.getStatusText());
   return ase;
 }
  private boolean isServiceSignUpException(Exception e) {
    if (e instanceof AmazonServiceException) {
      AmazonServiceException ase = (AmazonServiceException) e;
      return "OptInRequired".equalsIgnoreCase(ase.getErrorCode());
    }

    return false;
  }
 private static boolean doesTableExist(AmazonDynamoDB dynamo, String tableName) {
   try {
     TableDescription table =
         dynamo.describeTable(new DescribeTableRequest().withTableName(tableName)).getTable();
     return "ACTIVE".equals(table.getTableStatus());
   } catch (AmazonServiceException ase) {
     if (ase.getErrorCode().equals("ResourceNotFoundException")) return false;
     throw ase;
   }
 }
Example #13
0
 public static void sendMessage(AmazonSQS sqs, String sqsUrl, String info) {
   try {
     sqs.sendMessage(new SendMessageRequest(sqsUrl, info));
     logger.info("Message sent to queue: " + info);
   } catch (AmazonServiceException e) {
     logger.severe(e.getMessage());
   } catch (AmazonClientException e) {
     logger.severe(e.getMessage());
   }
 }
Example #14
0
 public void create(String store) throws IBlobStore.Error {
   try {
     s3client.createBucket(store);
   } catch (AmazonServiceException e) {
     if (e.getStatusCode() == 403) throw new IBlobStore.AuthError("" + e);
     throw new IBlobStore.IOError("" + e);
   } catch (AmazonClientException e) {
     throw new IBlobStore.IOError("" + e);
   }
 }
 private void logException(AmazonServiceException ase) {
   logger.error(
       "AmazonServiceException: error={}, statuscode={}, "
           + "awserrcode={}, errtype={}, reqid={}",
       ase.getMessage(),
       ase.getStatusCode(),
       ase.getErrorCode(),
       ase.getErrorType(),
       ase.getRequestId());
 }
 /**
  * Retrieves an instruction file from S3. If no instruction file is found, returns null.
  *
  * @param getObjectRequest A GET request for an object in S3. The parameters from this request
  *     will be used to retrieve the corresponding instruction file.
  * @return An instruction file, or null if no instruction file was found.
  */
 private S3Object getInstructionFile(GetObjectRequest getObjectRequest) {
   try {
     GetObjectRequest instructionFileRequest =
         EncryptionUtils.createInstructionGetRequest(getObjectRequest);
     return super.getObject(instructionFileRequest);
   } catch (AmazonServiceException e) {
     // If no instruction file is found, log a debug message, and return null.
     log.debug("Unable to retrieve instruction file : " + e.getMessage());
     return null;
   }
 }
Example #17
0
 /** {@inheritDoc} */
 @Override
 public void terminateInstance(String instanceId) {
   try {
     ec2Client().terminateInstances(new TerminateInstancesRequest(Arrays.asList(instanceId)));
   } catch (AmazonServiceException e) {
     if (e.getErrorCode().equals("InvalidInstanceID.NotFound")) {
       throw new NotFoundException("AWS instance " + instanceId + " not found", e);
     }
     throw e;
   }
 }
Example #18
0
  public static void deleteTaskMessage(Message msg, String sqsUrl, AmazonSQS sqs) {
    String handle = msg.getReceiptHandle();

    try {
      sqs.deleteMessage(new DeleteMessageRequest(sqsUrl, handle));
      logger.info("Message deleted: " + msg.getBody());
    } catch (AmazonServiceException e) {
      logger.severe(e.getMessage());
    } catch (AmazonClientException e) {
      logger.severe(e.getMessage());
    }
  }
Example #19
0
 public boolean storeExists(String store) throws IBlobStore.Error {
   try {
     s3client.listObjects(store);
     return true;
   } catch (final AmazonServiceException e) {
     if (e.getStatusCode() == 404) return false;
     if (e.getStatusCode() == 403) throw new IBlobStore.AuthError("" + e);
     throw new IBlobStore.IOError("" + e);
   } catch (AmazonClientException e) {
     Log.v(TAG, "Error: " + e);
     throw new IBlobStore.IOError("" + e);
   }
 }
  @Override
  public void delete(final TranscodingJob transcodingJob) {
    final ImmutableMap.Builder<String, AttributeValue> keyBuilder = ImmutableMap.builder();
    keyBuilder.put("transcode-job-id", new AttributeValue(transcodingJob.getId()));

    try {
      this.dynamoDB.deleteItem(TABLE_NAME, keyBuilder.build());
    } catch (final AmazonServiceException e) {
      logger.error(
          "AmazonServiceException while storing {}.  RID: {}", transcodingJob, e.getRequestId(), e);
    } catch (final AmazonClientException e) {
      logger.error("AmazonClientException while storing {}.", transcodingJob, e);
    }
  }
Example #21
0
  // Appends a S3 http:// and bucket address to file path.
  public static String generateS3FileAddress(AmazonS3 s3, String path) {
    String address;

    try {
      address = s3.getBucketLocation(bucket);
    } catch (AmazonServiceException e) {
      logger.severe(e.getMessage());
      return null;
    } catch (AmazonClientException e) {
      logger.severe(e.getMessage());
      return null;
    }

    return "https://s3-" + address + ".amazonaws.com/" + bucket + "/" + path;
  }
Example #22
0
  @Action(
      value = "/manage/submitEC2Key",
      results = {
        @Result(name = "input", location = "/manage/view_ec2_keys.jsp"),
        @Result(name = "success", location = "/manage/viewEC2Keys.action", type = "redirect")
      })
  public String submitEC2Key() {

    String retVal = SUCCESS;

    try {

      // get AWS credentials from DB
      AWSCred awsCred = AWSCredDB.getAWSCred();

      // set  AWS credentials for service
      BasicAWSCredentials awsCredentials =
          new BasicAWSCredentials(awsCred.getAccessKey(), awsCred.getSecretKey());

      // create service
      AmazonEC2 service = new AmazonEC2Client(awsCredentials);
      service.setEndpoint(ec2Key.getEc2Region());

      // create key pair request
      CreateKeyPairRequest createKeyPairRequest = new CreateKeyPairRequest();
      createKeyPairRequest.withKeyName(ec2Key.getKeyNm());

      // call service
      CreateKeyPairResult createKeyPairResult = service.createKeyPair(createKeyPairRequest);
      // get key pair result
      KeyPair keyPair = createKeyPairResult.getKeyPair();

      // set private key
      String privateKey = keyPair.getKeyMaterial();
      ec2Key.setPrivateKey(privateKey);

      // add to db
      Long keyId = EC2KeyDB.saveEC2Key(ec2Key);

      // store private key
      SSHUtil.storePrivateKey(keyId.toString(), ec2Key.getPrivateKey().trim());
    } catch (AmazonServiceException ex) {
      addActionError(ex.getMessage());
      retVal = INPUT;
    }

    return retVal;
  }
  @Override
  public void store(final TranscodingJob transcodingJob) {
    final ImmutableMap.Builder<String, AttributeValue> itemBuilder = ImmutableMap.builder();
    itemBuilder.put("transcode-job-id", new AttributeValue(transcodingJob.getId()));
    itemBuilder.put("inputMovie", new AttributeValue(transcodingJob.getInputPath().toString()));
    itemBuilder.put("outputMovie", new AttributeValue(transcodingJob.getOutputPath().toString()));

    try {
      this.dynamoDB.putItem(TABLE_NAME, itemBuilder.build());
    } catch (final AmazonServiceException e) {
      logger.error(
          "AmazonServiceException while storing {}.  RID: {}", transcodingJob, e.getRequestId(), e);
    } catch (final AmazonClientException e) {
      logger.error("AmazonClientException while storing {}.", transcodingJob, e);
    }
  }
Example #24
0
  private int parseClockSkewOffset(
      org.apache.http.HttpResponse response, AmazonServiceException exception) {
    Date deviceDate = new Date();
    Date serverDate = null;
    String serverDateStr = null;
    Header[] responseDateHeader = response.getHeaders("Date");

    try {

      if (responseDateHeader.length == 0) {
        // SQS doesn't return Date header
        serverDateStr = getServerDateFromException(exception.getMessage());
        serverDate = DateUtils.parseCompressedISO8601Date(serverDateStr);
      } else {
        serverDateStr = responseDateHeader[0].getValue();
        serverDate = DateUtils.parseRFC822Date(serverDateStr);
      }
    } catch (RuntimeException e) {
      log.warn("Unable to parse clock skew offset from response: " + serverDateStr, e);
      return 0;
    }

    long diff = deviceDate.getTime() - serverDate.getTime();
    return (int) (diff / 1000);
  }
  /**
   * Returns true if a failed request should be retried.
   *
   * @param method The current HTTP method being executed.
   * @param exception The exception from the failed request.
   * @param retries The number of times the current request has been attempted.
   * @return True if the failed request should be retried.
   */
  private boolean shouldRetry(HttpRequestBase method, Exception exception, int retries) {
    if (retries >= config.getMaxErrorRetry()) return false;

    if (method instanceof HttpEntityEnclosingRequest) {
      HttpEntity entity = ((HttpEntityEnclosingRequest) method).getEntity();
      if (entity != null && !entity.isRepeatable()) {
        if (log.isDebugEnabled()) {
          log.debug("Entity not repeatable");
        }
        return false;
      }
    }

    if (exception instanceof IOException) {
      if (log.isDebugEnabled()) {
        log.debug("Retrying on " + exception.getClass().getName() + ": " + exception.getMessage());
      }
      return true;
    }

    if (exception instanceof AmazonServiceException) {
      AmazonServiceException ase = (AmazonServiceException) exception;

      /*
       * For 500 internal server errors and 503 service
       * unavailable errors, we want to retry, but we need to use
       * an exponential back-off strategy so that we don't overload
       * a server with a flood of retries. If we've surpassed our
       * retry limit we handle the error response as a non-retryable
       * error and go ahead and throw it back to the user as an exception.
       */
      if (ase.getStatusCode() == HttpStatus.SC_INTERNAL_SERVER_ERROR
          || ase.getStatusCode() == HttpStatus.SC_SERVICE_UNAVAILABLE) {
        return true;
      }

      /*
       * Throttling is reported as a 400 error from newer services. To try
       * and smooth out an occasional throttling error, we'll pause and
       * retry, hoping that the pause is long enough for the request to
       * get through the next time.
       */
      if (isThrottlingException(ase)) return true;
    }

    return false;
  }
    protected Void doInBackground(Void... voids) {

      try {
        SimpleDB.createItem(domainName.getSelectedItem().toString(), itemName.getText().toString());

      } catch (AmazonServiceException e) {
        if ("InvalidClientTokenId".equals(e.getErrorCode())) {
          putRefreshError();
        } else {
          setStackAndPost(e);
        }
      } catch (Throwable e) {
        setStackAndPost(e);
      }

      return null;
    }
Example #27
0
  // Makes a putObjectRequest to make file public, and sends request.
  private static boolean putObject(AmazonS3 s3, PutObjectRequest req) {
    // Set file as public.
    req.withCannedAcl(CannedAccessControlList.PublicRead);

    // Send upload request.
    try {
      s3.putObject(req);
    } catch (AmazonServiceException e) {
      logger.severe(e.getMessage());
      return false;
    } catch (AmazonClientException e) {
      logger.severe(e.getMessage());
      return false;
    }

    return true;
  }
Example #28
0
  public static List<Message> getMessages(ReceiveMessageRequest req, AmazonSQS sqs) {
    logger.fine("Getting SQS messages.");

    List<Message> msgs;

    try {
      msgs = sqs.receiveMessage(req).getMessages();
    } catch (AmazonServiceException e) {
      logger.severe(e.getMessage());
      return null;
    } catch (AmazonClientException e) {
      logger.severe(e.getMessage());
      return null;
    }

    return msgs;
  }
Example #29
0
  @Action(
      value = "/manage/importEC2Key",
      results = {
        @Result(name = "input", location = "/manage/view_ec2_keys.jsp"),
        @Result(name = "success", location = "/manage/viewEC2Keys.action", type = "redirect")
      })
  public String importEC2Key() {

    String retVal = SUCCESS;

    try {
      // get AWS credentials from DB
      AWSCred awsCred = AWSCredDB.getAWSCred();

      // set  AWS credentials for service
      BasicAWSCredentials awsCredentials =
          new BasicAWSCredentials(awsCred.getAccessKey(), awsCred.getSecretKey());

      // create service
      AmazonEC2 service = new AmazonEC2Client(awsCredentials);
      service.setEndpoint(ec2Key.getEc2Region());

      // describe key pair request
      DescribeKeyPairsRequest describeKeyPairsRequest = new DescribeKeyPairsRequest();
      describeKeyPairsRequest.setKeyNames(Arrays.asList(ec2Key.getKeyNm()));

      // call service
      DescribeKeyPairsResult describeKeyPairsResult =
          service.describeKeyPairs(describeKeyPairsRequest);

      if (describeKeyPairsResult != null && describeKeyPairsResult.getKeyPairs().size() > 0) {
        // add to db
        Long keyId = EC2KeyDB.saveEC2Key(ec2Key);
        SSHUtil.storePrivateKey(keyId.toString(), ec2Key.getPrivateKey().trim());
      } else {
        addActionError("Imported key does not exist on AWS");
        retVal = INPUT;
      }

    } catch (AmazonServiceException ex) {
      addActionError(ex.getMessage());
      retVal = INPUT;
    }

    return retVal;
  }
Example #30
0
  @Override
  public int delete(String table, String key) {
    logger.debug("deletekey: " + key + " from table: " + table);
    DeleteItemRequest req = new DeleteItemRequest(table, createPrimaryKey(key));
    DeleteItemResult res = null;

    try {
      res = dynamoDB.deleteItem(req);
    } catch (AmazonServiceException ex) {
      logger.error(ex.getMessage());
      return SERVER_ERROR;
    } catch (AmazonClientException ex) {
      logger.error(ex.getMessage());
      return CLIENT_ERROR;
    }
    return OK;
  }