private void updateItemVersionedInternal(
     String tableName,
     Key key,
     Map<String, AttributeValueUpdate> attributes,
     String expectedVersion,
     PersistentEntity persistentEntity,
     int attempt)
     throws DataAccessException {
   UpdateItemRequest request =
       new UpdateItemRequest(tableName, key, attributes)
           .withExpected(getOptimisticVersionCondition(expectedVersion));
   try {
     ddb.updateItem(request);
   } catch (AmazonServiceException e) {
     if (DynamoDBUtil.AWS_ERR_CODE_CONDITIONAL_CHECK_FAILED.equals(e.getErrorCode())) {
       throw new OptimisticLockingException(persistentEntity, key);
     } else if (DynamoDBUtil.AWS_ERR_CODE_RESOURCE_NOT_FOUND.equals(e.getErrorCode())) {
       throw new IllegalArgumentException("no such table: " + tableName, e);
     } else if (DynamoDBUtil.AWS_STATUS_CODE_SERVICE_UNAVAILABLE == e.getStatusCode()) {
       // retry after a small pause
       DynamoDBUtil.sleepBeforeRetry(attempt);
       attempt++;
       updateItemVersionedInternal(
           tableName, key, attributes, expectedVersion, persistentEntity, attempt);
     } else {
       throw new DataStoreOperationException(
           "problem with table: " + tableName + ", key: " + key + ", attributes: " + attributes,
           e);
     }
   }
 }
Esempio n. 2
0
  private static void createKey(String keyName, AmazonEC2 ec2) {
    try {
      List<KeyPairInfo> keyPairList = ec2.describeKeyPairs().getKeyPairs();
      for (KeyPairInfo keyPair : keyPairList) {
        if (keyName.equalsIgnoreCase(keyPair.getKeyName())) {
          System.out.println("Using key " + keyName);
          return;
        }
      }
      System.out.println("Creating key " + keyName + "in local directory");
      CreateKeyPairRequest newKeyRequest = new CreateKeyPairRequest();
      newKeyRequest.setKeyName(keyName);
      CreateKeyPairResult keyresult = ec2.createKeyPair(newKeyRequest);
      KeyPair keyPair = new KeyPair();
      keyPair = keyresult.getKeyPair();
      String privateKey = keyPair.getKeyMaterial();
      writeKeytoFile(keyName, privateKey);

    } catch (AmazonServiceException ase) {
      System.out.println("Caught Exception: " + ase.getMessage());
      System.out.println("Reponse Status Code: " + ase.getStatusCode());
      System.out.println("Error Code: " + ase.getErrorCode());
      System.out.println("Request ID: " + ase.getRequestId());
    }
  }
  private static void waitForTableToBecomeAvailable(AmazonDynamoDB dynamo, String tableName) {
    System.out.println("Waiting for " + tableName + " to become ACTIVE...");

    long startTime = System.currentTimeMillis();
    long endTime = startTime + (10 * 60 * 1000);
    while (System.currentTimeMillis() < endTime) {
      try {
        Thread.sleep(1000 * 20);
      } catch (Exception e) {
      }
      try {
        DescribeTableRequest request = new DescribeTableRequest().withTableName(tableName);
        TableDescription table = dynamo.describeTable(request).getTable();
        if (table == null) continue;

        String tableStatus = table.getTableStatus();
        System.out.println("  - current state: " + tableStatus);
        if (tableStatus.equals(TableStatus.ACTIVE.toString())) return;
      } catch (AmazonServiceException ase) {
        if (ase.getErrorCode().equalsIgnoreCase("ResourceNotFoundException") == false) throw ase;
      }
    }

    throw new RuntimeException("Table " + tableName + " never went active");
  }
 private Map<String, AttributeValue> getConsistentInternal(
     String tableName, Key key, int attempt) {
   GetItemRequest request = new GetItemRequest(tableName, key);
   request.setConsistentRead(true);
   try {
     GetItemResult result = ddb.getItem(request);
     Map<String, AttributeValue> attributes = result.getItem();
     if (attributes == null || attributes.isEmpty()) {
       return null;
     }
     return attributes;
   } catch (AmazonServiceException e) {
     if (DynamoDBUtil.AWS_ERR_CODE_RESOURCE_NOT_FOUND.equals(e.getErrorCode())) {
       throw new IllegalArgumentException("no such table: " + tableName, e);
     } else if (DynamoDBUtil.AWS_STATUS_CODE_SERVICE_UNAVAILABLE == e.getStatusCode()) {
       // retry after a small pause
       DynamoDBUtil.sleepBeforeRetry(attempt);
       attempt++;
       return getConsistentInternal(tableName, key, attempt);
     } else {
       throw new DataStoreOperationException(
           "problem with table: " + tableName + ", key: " + key, e);
     }
   }
 }
Esempio n. 5
0
  public static void createBucket(AmazonS3Client s3, String bucketName, String zone) {
    try {

      List<Bucket> bucketList = s3.listBuckets();

      for (Bucket bucket : bucketList) {
        // System.out.println(bucket.getName());
        if (bucketName.equalsIgnoreCase(bucket.getName())) {
          System.out.println("Using bucket " + bucketName);
          return;
        }
      }

      System.out.println("Created s3 bucket " + bucketName);
      s3.createBucket(bucketName);

      return;

    } catch (AmazonServiceException ase) {
      System.out.println("Caught Exception: " + ase.getMessage());
      System.out.println("Reponse Status Code: " + ase.getStatusCode());
      System.out.println("Error Code: " + ase.getErrorCode());
      System.out.println("Request ID: " + ase.getRequestId());
    }
  }
  private static void basicAbortMPU() throws IOException {
    System.out.println("basic abort MPU");
    String bucketName = "chttest";
    String fileName = "hello.txt";
    // String uploadID = "XHGTFV4F5XTEAC5O8N3LK12TIY3DSY7OFPXIWTHRMNTE7A3WB5M8N2U5AN"; //hi
    String uploadID = "LE5JS2K6C208JU7ZX1QD2TVRWXOWWF4VNG7LE7TFIX5SYNG4HLOGW9CLAD"; // hello

    AbortMultipartUploadRequest request =
        new AbortMultipartUploadRequest(bucketName, fileName, uploadID);

    AmazonS3 s3 =
        new AmazonS3Client(
            new PropertiesCredentials(
                putBucket.class.getResourceAsStream("AwsCredentials.properties")));
    try {
      s3.abortMultipartUpload(request);
      System.out.println();
    } catch (AmazonServiceException ase) {
      System.out.println(
          "Caught an AmazonServiceException, which means your request made it "
              + "to Amazon S3, but was rejected with an error response for some reason.");
      System.out.println("Error Message:    " + ase.getMessage());
      System.out.println("HTTP Status Code: " + ase.getStatusCode());
      System.out.println("AWS Error Code:   " + ase.getErrorCode());
      System.out.println("Error Type:       " + ase.getErrorType());
      System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println(
          "Caught an AmazonClientException, which means the client encountered "
              + "a serious internal problem while trying to communicate with S3, "
              + "such as not being able to access the network.");
      System.out.println("Error Message: " + ace.getMessage());
    }
  }
  private boolean isServiceSignUpException(Exception e) {
    if (e instanceof AmazonServiceException) {
      AmazonServiceException ase = (AmazonServiceException) e;
      return "OptInRequired".equalsIgnoreCase(ase.getErrorCode());
    }

    return false;
  }
  public boolean wipeCredentialsOnAuthError(AmazonServiceException ex) {
    if (
    // STS
    // http://docs.amazonwebservices.com/STS/latest/APIReference/CommonErrors.html
    ex.getErrorCode().equals("IncompleteSignature")
        || ex.getErrorCode().equals("InternalFailure")
        || ex.getErrorCode().equals("InvalidClientTokenId")
        || ex.getErrorCode().equals("OptInRequired")
        || ex.getErrorCode().equals("RequestExpired")
        || ex.getErrorCode().equals("ServiceUnavailable")

        // DynamoDB
        // http://docs.amazonwebservices.com/amazondynamodb/latest/developerguide/ErrorHandling.html#APIErrorTypes
        || ex.getErrorCode().equals("AccessDeniedException")
        || ex.getErrorCode().equals("IncompleteSignatureException")
        || ex.getErrorCode().equals("MissingAuthenticationTokenException")
        || ex.getErrorCode().equals("ValidationException")
        || ex.getErrorCode().equals("InternalFailure")
        || ex.getErrorCode().equals("InternalServerError")) {

      clearCredentials();

      return true;
    }

    return false;
  }
    public void storeMessageInQueue(String message) {
      // System.out.println("Storing Message...");

      // Send a message

      if (driectDbConnection) {
        DbConnector db = null;
        try {
          db = dbf.createDBConnectorInstance();
        } catch (MailAppDBException e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }

        MailAppMessage mailMsg =
            new MailAppMessage(message, "" + System.currentTimeMillis() + "_" + UUID.randomUUID());
        try {
          // printDebug("Calling DB-Instance to store message "+mailMsg.getTimeuuid()+" length:
          // "+mailMsg.getSize()+"\n"+mailMsg.getWholeMessageWITHOUT2LINES());
          db.storeMessage(mailMsg);

        } catch (MailAppDBException mae) {
          // MailStoringWorkerStarter.stopMailStoringWorker(MailStoringWorkerStarter.getWorkerList(), this);
          // stop=true;
          mae.printStackTrace();
        }

      } else {

        try {
          sqs.sendMessage(new SendMessageRequest(mailQueueUrl, message));
        } catch (AmazonServiceException ase) {
          System.out.println(
              "Caught an AmazonServiceException, which means your request made it "
                  + "to Amazon SQS, but was rejected with an error response for some reason.");
          System.out.println("Error Message:    " + ase.getMessage());
          System.out.println("HTTP Status Code: " + ase.getStatusCode());
          System.out.println("AWS Error Code:   " + ase.getErrorCode());
          System.out.println("Error Type:       " + ase.getErrorType());
          System.out.println("Request ID:       " + ase.getRequestId());
        } catch (AmazonClientException ace) {
          System.out.println(
              "Caught an AmazonClientException, which means the client encountered "
                  + "a serious internal problem while trying to communicate with SQS, such as not "
                  + "being able to access the network.");
          System.out.println("Error Message: " + ace.getMessage());
        }

        printDebug("Message stored to " + mailQueueUrl);

        // connect to db ( Cassandra ) and store message
        // OR write Message to SQS to be consumed by Storage Worker ->
        // better? more scalable, db independent

      }
    }
 private void logException(AmazonServiceException ase) {
   logger.error(
       "AmazonServiceException: error={}, statuscode={}, "
           + "awserrcode={}, errtype={}, reqid={}",
       ase.getMessage(),
       ase.getStatusCode(),
       ase.getErrorCode(),
       ase.getErrorType(),
       ase.getRequestId());
 }
Esempio n. 11
0
 @Then("^I expect the response error code to be \"(.+?)\"$")
 public void then_I_expect_response_error_code(String expected) {
   assertNotNull(exception);
   assertTrue(exception instanceof AmazonServiceException);
   String actual = exception.getErrorCode();
   assertEquals(
       "Error code doesn't match. Expected : " + expected + ". Actual :" + actual,
       expected,
       actual);
 }
 private static boolean doesTableExist(AmazonDynamoDB dynamo, String tableName) {
   try {
     TableDescription table =
         dynamo.describeTable(new DescribeTableRequest().withTableName(tableName)).getTable();
     return "ACTIVE".equals(table.getTableStatus());
   } catch (AmazonServiceException ase) {
     if (ase.getErrorCode().equals("ResourceNotFoundException")) return false;
     throw ase;
   }
 }
Esempio n. 13
0
 /** {@inheritDoc} */
 @Override
 public void terminateInstance(String instanceId) {
   try {
     ec2Client().terminateInstances(new TerminateInstancesRequest(Arrays.asList(instanceId)));
   } catch (AmazonServiceException e) {
     if (e.getErrorCode().equals("InvalidInstanceID.NotFound")) {
       throw new NotFoundException("AWS instance " + instanceId + " not found", e);
     }
     throw e;
   }
 }
Esempio n. 14
0
  public boolean uploadToS3(String fileName, String text) throws IOException {

    try {
      String folderName = this.p.getProperty("KEY_NAME");
      String bucketName = this.p.getProperty("AWS_BUCKET_NAME");
      logger.info(LogKey.MESSAGE, "Uploading a new object to S3 from a file");

      byte[] contentAsBytes = text.getBytes("UTF-8");
      ByteArrayInputStream contentsAsStream = new ByteArrayInputStream(contentAsBytes);
      ObjectMetadata md = new ObjectMetadata();
      md.setContentLength(contentAsBytes.length);

      if (this.s3client == null) {
        logger.debug(LogKey.MESSAGE, "reuse S3 client !!");
        this.s3client = this.getS3Client();
      }

      logger.info(LogKey.BUCKET_NAME, bucketName);
      logger.info(LogKey.KEY_NAME, folderName);
      logger.info(LogKey.FILE_NAME, fileName);

      this.s3client.putObject(
          new PutObjectRequest(
              bucketName, folderName + File.separator + fileName, contentsAsStream, md));

      return true;

    } catch (AmazonServiceException ase) {
      logger.error(LogKey.EXCEPTION, ase);
      logger.error(
          LogKey.EXCEPTION,
          "Caught an AmazonServiceException, which "
              + "means your request made it "
              + "to Amazon S3, but was rejected with an error response"
              + " for some reason.");
      logger.error(LogKey.EXCEPTION, "Error Message:    " + ase.getMessage());
      logger.error(LogKey.EXCEPTION, "HTTP Status Code: " + ase.getStatusCode());
      logger.error(LogKey.EXCEPTION, "AWS Error Code:   " + ase.getErrorCode());
      logger.error(LogKey.EXCEPTION, "Error Type:       " + ase.getErrorType());
      logger.error(LogKey.EXCEPTION, "Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      logger.error(
          LogKey.EXCEPTION,
          "Caught an AmazonClientException, which "
              + "means the client encountered "
              + "an internal error while trying to "
              + "communicate with S3, "
              + "such as not being able to access the network.");
      logger.error(LogKey.EXCEPTION, "Error Message: " + ace.getMessage());
    }
    return false;
  }
    protected Void doInBackground(Void... voids) {

      try {
        SimpleDB.createItem(domainName.getSelectedItem().toString(), itemName.getText().toString());

      } catch (AmazonServiceException e) {
        if ("InvalidClientTokenId".equals(e.getErrorCode())) {
          putRefreshError();
        } else {
          setStackAndPost(e);
        }
      } catch (Throwable e) {
        setStackAndPost(e);
      }

      return null;
    }
  /** Verifies the RetryCondition has collected the expected context information. */
  public static void verifyExpectedContextData(
      ContextDataCollection contextDataCollection,
      AmazonWebServiceRequest failedRequest,
      AmazonClientException expectedException,
      int expectedRetries) {

    Assert.assertEquals(expectedRetries, contextDataCollection.failedRequests.size());
    Assert.assertEquals(expectedRetries, contextDataCollection.exceptions.size());
    Assert.assertEquals(expectedRetries, contextDataCollection.retriesAttemptedValues.size());

    if (expectedRetries > 0) {
      // It should keep getting the same original request instance
      for (AmazonWebServiceRequest seenRequest : contextDataCollection.failedRequests) {
        Assert.assertTrue(seenRequest == failedRequest);
      }

      // Verify the exceptions
      if (expectedException instanceof AmazonServiceException) {
        // It should get service exceptions with the expected error and
        // status code
        AmazonServiceException ase = (AmazonServiceException) expectedException;
        for (AmazonClientException seenException : contextDataCollection.exceptions) {
          Assert.assertTrue(seenException instanceof AmazonServiceException);
          Assert.assertEquals(
              ase.getErrorCode(), ((AmazonServiceException) seenException).getErrorCode());
          Assert.assertEquals(
              ase.getStatusCode(), ((AmazonServiceException) seenException).getStatusCode());
        }
      } else {
        // Client exceptions should have the same expected cause (the
        // same
        // throwable instance from the mock HttpClient).
        Throwable expectedCause = expectedException.getCause();
        for (AmazonClientException seenException : contextDataCollection.exceptions) {
          Assert.assertTrue(expectedCause == seenException.getCause());
        }
      }

      // It should get "retriesAttempted" values starting from 0
      int expectedRetriesAttempted = 0;
      for (int seenRetriesAttempted : contextDataCollection.retriesAttemptedValues) {
        Assert.assertEquals(expectedRetriesAttempted++, seenRetriesAttempted);
      }
    }
  }
 private void deleteItemInternal(String tableName, Key key, int attempt) {
   DeleteItemRequest request = new DeleteItemRequest(tableName, key);
   try {
     ddb.deleteItem(request);
   } catch (AmazonServiceException e) {
     if (DynamoDBUtil.AWS_ERR_CODE_RESOURCE_NOT_FOUND.equals(e.getErrorCode())) {
       throw new IllegalArgumentException("no such table: " + tableName, e);
     } else if (DynamoDBUtil.AWS_STATUS_CODE_SERVICE_UNAVAILABLE == e.getStatusCode()) {
       // retry after a small pause
       DynamoDBUtil.sleepBeforeRetry(attempt);
       attempt++;
       deleteItemInternal(tableName, key, attempt);
     } else {
       throw new DataStoreOperationException(
           "problem with table: " + tableName + ", key: " + key, e);
     }
   }
 }
Esempio n. 18
0
 /**
  * Update the tags stored in EC2 with the specified information. Re-try 5 times if instances isn't
  * up by catchErrorCode - e.g. InvalidSpotInstanceRequestID.NotFound or
  * InvalidInstanceRequestID.NotFound
  *
  * @param ec2
  * @param instTags
  * @param catchErrorCode
  * @param params
  * @throws InterruptedException
  */
 private void updateRemoteTags(
     AmazonEC2 ec2, Collection<Tag> instTags, String catchErrorCode, String... params)
     throws InterruptedException {
   for (int i = 0; i < 5; i++) {
     try {
       CreateTagsRequest tagRequest = new CreateTagsRequest();
       tagRequest.withResources(params).setTags(instTags);
       ec2.createTags(tagRequest);
       break;
     } catch (AmazonServiceException e) {
       if (e.getErrorCode().equals(catchErrorCode)) {
         Thread.sleep(5000);
         continue;
       }
       LOGGER.log(Level.SEVERE, e.getErrorMessage(), e);
     }
   }
 }
Esempio n. 19
0
  @Inject
  public AmazonS3Storage(Configuration configuration) {
    bucketName = configuration.getString("storage.s3.bucket");

    String accessKey = configuration.getString("storage.s3.accesskey");
    String secretKey = configuration.getString("storage.s3.secretkey");
    credentials = new BasicAWSCredentials(accessKey, secretKey);

    AmazonS3 amazonS3 = new AmazonS3Client(credentials);

    try {
      if (!(amazonS3.doesBucketExist(bucketName))) {
        amazonS3.createBucket(new CreateBucketRequest(bucketName));
      }

      String bucketLocation = amazonS3.getBucketLocation(new GetBucketLocationRequest(bucketName));
      Logger.info("Amazon S3 bucket created at " + bucketLocation);
    } catch (AmazonServiceException ase) {
      Logger.error(
          "Caught an AmazonServiceException, which "
              + "means your request made it "
              + "to Amazon S3, but was rejected with an error response "
              + "for some reason."
              + " Error Message: "
              + ase.getMessage()
              + " HTTP Status Code: "
              + ase.getStatusCode()
              + " AWS Error Code: "
              + ase.getErrorCode()
              + " Error Type: "
              + ase.getErrorType()
              + " Request ID: "
              + ase.getRequestId());
    } catch (AmazonClientException ace) {
      Logger.error(
          "Caught an AmazonClientException, which "
              + "means the client encountered "
              + "an internal error while trying to "
              + "communicate with S3, "
              + "such as not being able to access the network."
              + " Error Message: "
              + ace.getMessage());
    }
  }
Esempio n. 20
0
  public static String xs3_generate_url(String xs3_objname, String content_type) {
    AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
    ClientConfiguration xs3_clientconfig = new ClientConfiguration();
    xs3_clientconfig.setProtocol(Protocol.HTTP);

    S3ClientOptions xs3_client_options = new S3ClientOptions();
    xs3_client_options.setPathStyleAccess(true);

    xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
    xs3_client.setEndpoint(xs3_endpoint);
    xs3_client.setS3ClientOptions(xs3_client_options);

    try {
      java.util.Date expiration = new java.util.Date();

      long milliSeconds = expiration.getTime();
      milliSeconds += 1000 * 60 * 5;
      expiration.setTime(milliSeconds);

      GeneratePresignedUrlRequest xs3_genurl_req =
          new GeneratePresignedUrlRequest(xs3_bucketname, xs3_objname);
      xs3_genurl_req.setMethod(HttpMethod.PUT);
      xs3_genurl_req.setExpiration(expiration);
      xs3_genurl_req.setContentType(content_type);
      xs3_genurl_req.addRequestParameter("x-amz-acl", "public-read");

      URL url = xs3_client.generatePresignedUrl(xs3_genurl_req);

      System.out.println(url.toString());
      return url.toString();

    } catch (AmazonServiceException ase) {
      System.out.println("xs3_svr_error_message:" + ase.getMessage());
      System.out.println("xs3_svr_status_code:  " + ase.getStatusCode());
      System.out.println("xs3_svr_error_code:   " + ase.getErrorCode());
      System.out.println("xs3_svr_error_type:   " + ase.getErrorType());
      System.out.println("xs3_svr_request_id:   " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println("xs3_clt_error_message:" + ace.getMessage());
    }

    return null;
  }
  private List<Map<String, AttributeValue>> scanInternal(
      String tableName, Map<String, Condition> filter, int max, int attempt) {
    LinkedList<Map<String, AttributeValue>> items = new LinkedList<Map<String, AttributeValue>>();
    try {
      ScanRequest request = new ScanRequest(tableName).withScanFilter(filter);
      ScanResult result = ddb.scan(request);
      items.addAll(result.getItems());

      // keep repeating until we get through all matched items
      Key lastKeyEvaluated = null;
      do {
        lastKeyEvaluated = result.getLastEvaluatedKey();
        if (lastKeyEvaluated != null) {
          request =
              new ScanRequest(tableName)
                  .withScanFilter(filter)
                  .withExclusiveStartKey(lastKeyEvaluated);
          result = ddb.scan(request);
          items.addAll(result.getItems());
        }
      } while (lastKeyEvaluated != null && items.size() < max);

      // truncate if needed
      while (items.size() > max) {
        items.removeLast();
      }

      return items;
    } catch (AmazonServiceException e) {
      if (DynamoDBUtil.AWS_ERR_CODE_RESOURCE_NOT_FOUND.equals(e.getErrorCode())) {
        throw new IllegalArgumentException("no such table: " + tableName, e);
      } else if (DynamoDBUtil.AWS_STATUS_CODE_SERVICE_UNAVAILABLE == e.getStatusCode()) {
        // retry after a small pause
        DynamoDBUtil.sleepBeforeRetry(attempt);
        attempt++;
        return scanInternal(tableName, filter, max, attempt);
      } else {
        throw new DataStoreOperationException(
            "problem with table: " + tableName + ", filter: " + filter, e);
      }
    }
  }
 private void putItemInternal(
     String tableName, Map<String, AttributeValue> attributes, int attempt)
     throws DataAccessException {
   try {
     PutItemRequest request = new PutItemRequest(tableName, attributes);
     ddb.putItem(request);
   } catch (AmazonServiceException e) {
     if (DynamoDBUtil.AWS_ERR_CODE_RESOURCE_NOT_FOUND.equals(e.getErrorCode())) {
       throw new IllegalArgumentException("no such table: " + tableName, e);
     } else if (DynamoDBUtil.AWS_STATUS_CODE_SERVICE_UNAVAILABLE == e.getStatusCode()) {
       // retry after a small pause
       DynamoDBUtil.sleepBeforeRetry(attempt);
       attempt++;
       putItemInternal(tableName, attributes, attempt);
     } else {
       throw new DataStoreOperationException(
           "problem with table: " + tableName + ", attributes: " + attributes, e);
     }
   }
 }
Esempio n. 23
0
  public static String xs3_init_multi_upload(String xs3_objname, int file_size, String file_type) {
    AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
    ClientConfiguration xs3_clientconfig = new ClientConfiguration();
    xs3_clientconfig.setProtocol(Protocol.HTTP);

    S3ClientOptions xs3_client_options = new S3ClientOptions();
    xs3_client_options.setPathStyleAccess(true);

    xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
    xs3_client.setEndpoint(xs3_endpoint);
    xs3_client.setS3ClientOptions(xs3_client_options);

    try {
      InitiateMultipartUploadRequest xs3_multi_req =
          new InitiateMultipartUploadRequest(xs3_bucketname, xs3_objname);
      xs3_multi_req.setCannedACL(CannedAccessControlList.PublicRead);
      ObjectMetadata xs3_meta = new ObjectMetadata();
      xs3_meta.setContentType(file_type);
      xs3_multi_req.setObjectMetadata(xs3_meta);

      InitiateMultipartUploadResult xs3_multi_res =
          xs3_client.initiateMultipartUpload(xs3_multi_req);

      String xs3_multi_uploadid = xs3_multi_res.getUploadId();

      String json_urls = gen_part_url(xs3_multi_uploadid, file_size, xs3_objname, file_type);
      return json_urls;

    } catch (AmazonServiceException ase) {
      System.out.println("xs3_svr_error_message:" + ase.getMessage());
      System.out.println("xs3_svr_status_code:  " + ase.getStatusCode());
      System.out.println("xs3_svr_error_code:   " + ase.getErrorCode());
      System.out.println("xs3_svr_error_type:   " + ase.getErrorType());
      System.out.println("xs3_svr_request_id:   " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println("xs3_clt_error_message:" + ace.getMessage());
    }
    return null;
  }
Esempio n. 24
0
  public static void xs3_coplete_multi_upload(String xs3_objname, String uploadId) {
    AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
    ClientConfiguration xs3_clientconfig = new ClientConfiguration();
    xs3_clientconfig.setProtocol(Protocol.HTTP);

    S3ClientOptions xs3_client_options = new S3ClientOptions();
    xs3_client_options.setPathStyleAccess(true);

    xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
    xs3_client.setEndpoint(xs3_endpoint);
    xs3_client.setS3ClientOptions(xs3_client_options);

    try {
      List<PartETag> rest_parts = listPartsXml(xs3_objname, uploadId);
      if (null == rest_parts) {
        return;
      }
      for (PartETag item : rest_parts) {
        System.out.println(item.getETag() + " -> " + item.getPartNumber());
      }

      CompleteMultipartUploadRequest comp_req =
          new CompleteMultipartUploadRequest(xs3_bucketname, xs3_objname, uploadId, rest_parts);

      CompleteMultipartUploadResult comp_result = xs3_client.completeMultipartUpload(comp_req);

      System.out.println(comp_result.getETag());
      System.out.println(comp_result.getKey());
    } catch (AmazonServiceException ase) {
      System.out.println("xs3_svr_error_message:" + ase.getMessage());
      System.out.println("xs3_svr_status_code:  " + ase.getStatusCode());
      System.out.println("xs3_svr_error_code:   " + ase.getErrorCode());
      System.out.println("xs3_svr_error_type:   " + ase.getErrorType());
      System.out.println("xs3_svr_request_id:   " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println("xs3_clt_error_message:" + ace.getMessage());
    }
  }
  /**
   * @param args
   * @throws IOException
   */
  public static void main(String[] args) throws IOException {
    try {

      String myQueueUrl = "https://queue.amazonaws.com/034307772076/MyQueue";

      AmazonSQS sqs =
          new AmazonSQSClient(
              new PropertiesCredentials(
                  SimpleQueueServiceSample.class.getResourceAsStream("AwsCredentials.properties")));

      System.out.println("Sending message 1.");
      sqs.sendMessage(new SendMessageRequest(myQueueUrl, "Test message."));

      System.out.println("Receiving messages:");
      ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(myQueueUrl);
      List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages();
      for (Message message : messages) {
        System.out.print(message.getMessageId());
        System.out.println(" : " + message.getBody());
      }
    } catch (AmazonServiceException ase) {
      System.out.println(
          "Caught an AmazonServiceException, which means your request made it "
              + "to Amazon SQS, but was rejected with an error response for some reason.");
      System.out.println("Error Message:    " + ase.getMessage());
      System.out.println("HTTP Status Code: " + ase.getStatusCode());
      System.out.println("AWS Error Code:   " + ase.getErrorCode());
      System.out.println("Error Type:       " + ase.getErrorType());
      System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println(
          "Caught an AmazonClientException, which means the client encountered "
              + "a serious internal problem while trying to communicate with SQS, such as not "
              + "being able to access the network.");
      System.out.println("Error Message: " + ace.getMessage());
    }
  }
Esempio n. 26
0
  public static void main(String[] args) throws IOException {
    /*
     * This credentials provider implementation loads your AWS credentials
     * from a properties file at the root of your classpath.
     *
     * Important: Be sure to fill in your AWS access credentials in the
     *            AwsCredentials.properties file before you try to run this
     *            sample.
     * http://aws.amazon.com/security-credentials
     */
    AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());

    String bucketName = "my-first-s3-bucket-" + UUID.randomUUID();
    String key = "MyObjectKey";

    System.out.println("===========================================");
    System.out.println("Getting Started with Amazon S3");
    System.out.println("===========================================\n");

    try {
      /*
       * Create a new S3 bucket - Amazon S3 bucket names are globally unique,
       * so once a bucket name has been taken by any user, you can't create
       * another bucket with that same name.
       *
       * You can optionally specify a location for your bucket if you want to
       * keep your data closer to your applications or users.
       */
      System.out.println("Creating bucket " + bucketName + "\n");
      s3.createBucket(bucketName);

      /*
       * List the buckets in your account
       */
      System.out.println("Listing buckets");
      for (Bucket bucket : s3.listBuckets()) {
        System.out.println(" - " + bucket.getName());
      }
      System.out.println();

      /*
       * Upload an object to your bucket - You can easily upload a file to
       * S3, or upload directly an InputStream if you know the length of
       * the data in the stream. You can also specify your own metadata
       * when uploading to S3, which allows you set a variety of options
       * like content-type and content-encoding, plus additional metadata
       * specific to your applications.
       */
      System.out.println("Uploading a new object to S3 from a file\n");
      s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile()));

      /*
       * Download an object - When you download an object, you get all of
       * the object's metadata and a stream from which to read the contents.
       * It's important to read the contents of the stream as quickly as
       * possibly since the data is streamed directly from Amazon S3 and your
       * network connection will remain open until you read all the data or
       * close the input stream.
       *
       * GetObjectRequest also supports several other options, including
       * conditional downloading of objects based on modification times,
       * ETags, and selectively downloading a range of an object.
       */
      System.out.println("Downloading an object");
      S3Object object = s3.getObject(new GetObjectRequest(bucketName, key));
      System.out.println("Content-Type: " + object.getObjectMetadata().getContentType());
      displayTextInputStream(object.getObjectContent());

      /*
       * List objects in your bucket by prefix - There are many options for
       * listing the objects in your bucket.  Keep in mind that buckets with
       * many objects might truncate their results when listing their objects,
       * so be sure to check if the returned object listing is truncated, and
       * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve
       * additional results.
       */
      System.out.println("Listing objects");
      ObjectListing objectListing =
          s3.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My"));
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        System.out.println(
            " - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")");
      }
      System.out.println();

      /*
       * Delete an object - Unless versioning has been turned on for your bucket,
       * there is no way to undelete an object, so use caution when deleting objects.
       */
      System.out.println("Deleting an object\n");
      s3.deleteObject(bucketName, key);

      /*
       * Delete a bucket - A bucket must be completely empty before it can be
       * deleted, so remember to delete any objects from your buckets before
       * you try to delete them.
       */
      System.out.println("Deleting bucket " + bucketName + "\n");
      s3.deleteBucket(bucketName);
    } catch (AmazonServiceException ase) {
      System.out.println(
          "Caught an AmazonServiceException, which means your request made it "
              + "to Amazon S3, but was rejected with an error response for some reason.");
      System.out.println("Error Message:    " + ase.getMessage());
      System.out.println("HTTP Status Code: " + ase.getStatusCode());
      System.out.println("AWS Error Code:   " + ase.getErrorCode());
      System.out.println("Error Type:       " + ase.getErrorType());
      System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println(
          "Caught an AmazonClientException, which means the client encountered "
              + "a serious internal problem while trying to communicate with S3, "
              + "such as not being able to access the network.");
      System.out.println("Error Message: " + ace.getMessage());
    }
  }
Esempio n. 27
0
  private void jButton1ActionPerformed(
      java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jButton1ActionPerformed
    // TODO add your handling code here: upload file
    JFileChooser jfc = new JFileChooser();
    jfc.setAcceptAllFileFilterUsed(false);
    jfc.setFileFilter(new FileNameExtensionFilter("Text Files", "txt"));
    int returnVal = jfc.showOpenDialog(this);
    if (returnVal == JFileChooser.APPROVE_OPTION) {
      File file = jfc.getSelectedFile();
      try {
        filepath = file.getCanonicalPath();
        FileReader fr = new FileReader(filepath);

        BufferedReader br = new BufferedReader(fr);
        String line;
        int total_no_of_words = 0;

        while ((line = br.readLine()) != null) {

          String[] words_arr = line.split(" ");
          total_no_of_words += words_arr.length;
        }
        jLabel4.setText("word count :" + total_no_of_words);
        System.out.println("Total no of words = " + total_no_of_words);
        if (total_no_of_words != 0) {
          File userFile = new File(filepath);
          keyName = userFile.getName();
          String myAccessKeyID = "AKIAI7PATIX6LHJA3A3Q";
          String mySecretKey = "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";
          AWSCredentials myCredentials = new BasicAWSCredentials(myAccessKeyID, mySecretKey);
          ClientConfiguration clientConfig = new ClientConfiguration();
          clientConfig.setProtocol(Protocol.HTTP);
          AmazonS3 conn = new AmazonS3Client(myCredentials, clientConfig);
          try {

            File file1 = new File(filepath);
            file_size = file1.length();
            if (file_size < 100000) {
              conn.putObject(new PutObjectRequest(bucketName, keyName, file1));
              jLabel2.setText("Uploading complete !");
            } else {
              jLabel2.setText("Uploading failed size greater than 100kb...");
            }
          } catch (AmazonServiceException ase) {
            System.out.println(
                "Caught an AmazonServiceException, which "
                    + "means your request made it "
                    + "to Amazon S3, but was rejected with an error response"
                    + " for some reason.");
            System.out.println("Error Message:    " + ase.getMessage());
            System.out.println("HTTP Status Code: " + ase.getStatusCode());
            System.out.println("AWS Error Code:   " + ase.getErrorCode());
            System.out.println("Error Type:       " + ase.getErrorType());
            System.out.println("Request ID:       " + ase.getRequestId());
          } catch (AmazonClientException ace) {
            System.out.println(
                "Caught an AmazonClientException, which "
                    + "means the client encountered "
                    + "an internal error while trying to "
                    + "communicate with S3, "
                    + "such as not being able to access the network.");
            System.out.println("Error Message: " + ace.getMessage());
          } /*catch (InterruptedException ex) {
                Logger.getLogger(Upload.class.getName()).log(Level.SEVERE, null, ex);
            }*/
        } else {
          jLabel2.setText("Uploading failed file empty !");
        }

      } catch (IOException ex) {
        Logger.getLogger(Upload.class.getName()).log(Level.SEVERE, null, ex);
      }
    }
    if (returnVal == JFileChooser.CANCEL_OPTION) {
      System.exit(0);
    }
  } // GEN-LAST:event_jButton1ActionPerformed
Esempio n. 28
0
  public static void main(String[] args) throws Exception {
    System.out.println("===========================================");
    System.out.println("Welcome to the AWS Java SDK!");
    System.out.println("===========================================");

    init();

    try {
      /*
       * The Amazon EC2 client allows you to easily launch and configure
       * computing capacity in AWS datacenters.
       *
       * In this sample, we use the EC2 client to list the availability zones
       * in a region, and then list the instances running in those zones.
       */
      DescribeAvailabilityZonesResult availabilityZonesResult = ec2.describeAvailabilityZones();
      List<AvailabilityZone> availabilityZones = availabilityZonesResult.getAvailabilityZones();
      System.out.println("You have access to " + availabilityZones.size() + " availability zones:");
      for (AvailabilityZone zone : availabilityZones) {
        System.out.println(" - " + zone.getZoneName() + " (" + zone.getRegionName() + ")");
      }

      DescribeInstancesResult describeInstancesResult = ec2.describeInstances();
      Set<Instance> instances = new HashSet<Instance>();
      for (Reservation reservation : describeInstancesResult.getReservations()) {
        instances.addAll(reservation.getInstances());
      }

      System.out.println("You have " + instances.size() + " Amazon EC2 instance(s) running.");

      /*
       * The Amazon S3 client allows you to manage and configure buckets
       * and to upload and download data.
       *
       * In this sample, we use the S3 client to list all the buckets in
       * your account, and then iterate over the object metadata for all
       * objects in one bucket to calculate the total object count and
       * space usage for that one bucket. Note that this sample only
       * retrieves the object's metadata and doesn't actually download the
       * object's content.
       *
       * In addition to the low-level Amazon S3 client in the SDK, there
       * is also a high-level TransferManager API that provides
       * asynchronous management of uploads and downloads with an easy to
       * use API:
       *   http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/transfer/TransferManager.html
       */
      List<Bucket> buckets = s3.listBuckets();
      System.out.println("You have " + buckets.size() + " Amazon S3 bucket(s).");

      if (buckets.size() > 0) {
        Bucket bucket = buckets.get(0);

        long totalSize = 0;
        long totalItems = 0;
        /*
         * The S3Objects and S3Versions classes provide convenient APIs
         * for iterating over the contents of your buckets, without
         * having to manually deal with response pagination.
         */
        for (S3ObjectSummary objectSummary : S3Objects.inBucket(s3, bucket.getName())) {
          totalSize += objectSummary.getSize();
          totalItems++;
        }

        System.out.println(
            "The bucket '"
                + bucket.getName()
                + "' contains "
                + totalItems
                + " objects "
                + "with a total size of "
                + totalSize
                + " bytes.");
      }
    } catch (AmazonServiceException ase) {
      /*
       * AmazonServiceExceptions represent an error response from an AWS
       * services, i.e. your request made it to AWS, but the AWS service
       * either found it invalid or encountered an error trying to execute
       * it.
       */
      System.out.println("Error Message:    " + ase.getMessage());
      System.out.println("HTTP Status Code: " + ase.getStatusCode());
      System.out.println("AWS Error Code:   " + ase.getErrorCode());
      System.out.println("Error Type:       " + ase.getErrorType());
      System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      /*
       * AmazonClientExceptions represent an error that occurred inside
       * the client on the local host, either while trying to send the
       * request to AWS or interpret the response. For example, if no
       * network connection is available, the client won't be able to
       * connect to AWS to execute a request and will throw an
       * AmazonClientException.
       */
      System.out.println("Error Message: " + ace.getMessage());
    }
  }
 /**
  * Returns true if the specified exception is a throttling error.
  *
  * @param ase The exception to test.
  * @return True if the exception resulted from a throttling error message from a service,
  *     otherwise false.
  */
 private boolean isThrottlingException(AmazonServiceException ase) {
   if (ase == null) return false;
   return "Throttling".equals(ase.getErrorCode())
       || "ThrottlingException".equals(ase.getErrorCode())
       || "ProvisionedThroughputExceededException".equals(ase.getErrorCode());
 }
Esempio n. 30
0
  public static String gen_part_url(
      String uploadId, int file_size, String file_name, String file_type) {
    AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
    ClientConfiguration xs3_clientconfig = new ClientConfiguration();
    xs3_clientconfig.setProtocol(Protocol.HTTP);

    S3ClientOptions xs3_client_options = new S3ClientOptions();
    xs3_client_options.setPathStyleAccess(true);

    xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
    xs3_client.setEndpoint(xs3_endpoint);
    xs3_client.setS3ClientOptions(xs3_client_options);

    try {

      final int xs3_part_size = 1024 * 1024 * 5;

      int xs3_part_count = (int) Math.ceil((double) (file_size) / (double) xs3_part_size);
      JSONArray jsonArray = new JSONArray();
      JSONObject jsonObject_1 = new JSONObject();
      jsonObject_1.put("total_num", xs3_part_count);
      jsonObject_1.put("upload_id", uploadId);

      JSONArray jsonArray_sub = new JSONArray();
      for (int part_no = 0; part_no < xs3_part_count; part_no++) {

        long xs3_offset_bytes = xs3_part_size * part_no;

        long part_size =
            xs3_part_size < (file_size - xs3_offset_bytes)
                ? xs3_part_size
                : (file_size - xs3_offset_bytes);

        java.util.Date expiration = new java.util.Date();
        long milliSeconds = expiration.getTime();
        milliSeconds += 1000 * 60 * 5;
        expiration.setTime(milliSeconds);

        GeneratePresignedUrlRequest xs3_genurl_req =
            new GeneratePresignedUrlRequest(xs3_bucketname, file_name);
        xs3_genurl_req.setMethod(HttpMethod.PUT);
        xs3_genurl_req.setExpiration(expiration);
        xs3_genurl_req.setContentType(file_type);
        xs3_genurl_req.addRequestParameter("uploadId", uploadId);
        xs3_genurl_req.addRequestParameter("partNumber", String.valueOf(part_no + 1));

        URL url = xs3_client.generatePresignedUrl(xs3_genurl_req);
        System.out.println(url.toString());

        JSONObject jsonObject = new JSONObject();
        jsonObject.put("part_idx", part_no);
        jsonObject.put("part_url", url.toString());
        jsonObject.put("upload_len", part_size);
        jsonObject.put("part_begin", xs3_offset_bytes);
        jsonObject.put("part_end", xs3_offset_bytes + part_size);

        jsonArray_sub.add(jsonObject);
      }

      jsonObject_1.put("multi_list", jsonArray_sub);
      jsonArray.add(jsonObject_1);

      return jsonArray.toString();

    } catch (AmazonServiceException ase) {
      System.out.println("xs3_svr_error_message:" + ase.getMessage());
      System.out.println("xs3_svr_status_code:  " + ase.getStatusCode());
      System.out.println("xs3_svr_error_code:   " + ase.getErrorCode());
      System.out.println("xs3_svr_error_type:   " + ase.getErrorType());
      System.out.println("xs3_svr_request_id:   " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println("xs3_clt_error_message:" + ace.getMessage());
    }

    return null;
  }