Ejemplo n.º 1
0
  private static void createKey(String keyName, AmazonEC2 ec2) {
    try {
      List<KeyPairInfo> keyPairList = ec2.describeKeyPairs().getKeyPairs();
      for (KeyPairInfo keyPair : keyPairList) {
        if (keyName.equalsIgnoreCase(keyPair.getKeyName())) {
          System.out.println("Using key " + keyName);
          return;
        }
      }
      System.out.println("Creating key " + keyName + "in local directory");
      CreateKeyPairRequest newKeyRequest = new CreateKeyPairRequest();
      newKeyRequest.setKeyName(keyName);
      CreateKeyPairResult keyresult = ec2.createKeyPair(newKeyRequest);
      KeyPair keyPair = new KeyPair();
      keyPair = keyresult.getKeyPair();
      String privateKey = keyPair.getKeyMaterial();
      writeKeytoFile(keyName, privateKey);

    } catch (AmazonServiceException ase) {
      System.out.println("Caught Exception: " + ase.getMessage());
      System.out.println("Reponse Status Code: " + ase.getStatusCode());
      System.out.println("Error Code: " + ase.getErrorCode());
      System.out.println("Request ID: " + ase.getRequestId());
    }
  }
Ejemplo n.º 2
0
  public static void createBucket(AmazonS3Client s3, String bucketName, String zone) {
    try {

      List<Bucket> bucketList = s3.listBuckets();

      for (Bucket bucket : bucketList) {
        // System.out.println(bucket.getName());
        if (bucketName.equalsIgnoreCase(bucket.getName())) {
          System.out.println("Using bucket " + bucketName);
          return;
        }
      }

      System.out.println("Created s3 bucket " + bucketName);
      s3.createBucket(bucketName);

      return;

    } catch (AmazonServiceException ase) {
      System.out.println("Caught Exception: " + ase.getMessage());
      System.out.println("Reponse Status Code: " + ase.getStatusCode());
      System.out.println("Error Code: " + ase.getErrorCode());
      System.out.println("Request ID: " + ase.getRequestId());
    }
  }
  private static void basicAbortMPU() throws IOException {
    System.out.println("basic abort MPU");
    String bucketName = "chttest";
    String fileName = "hello.txt";
    // String uploadID = "XHGTFV4F5XTEAC5O8N3LK12TIY3DSY7OFPXIWTHRMNTE7A3WB5M8N2U5AN"; //hi
    String uploadID = "LE5JS2K6C208JU7ZX1QD2TVRWXOWWF4VNG7LE7TFIX5SYNG4HLOGW9CLAD"; // hello

    AbortMultipartUploadRequest request =
        new AbortMultipartUploadRequest(bucketName, fileName, uploadID);

    AmazonS3 s3 =
        new AmazonS3Client(
            new PropertiesCredentials(
                putBucket.class.getResourceAsStream("AwsCredentials.properties")));
    try {
      s3.abortMultipartUpload(request);
      System.out.println();
    } catch (AmazonServiceException ase) {
      System.out.println(
          "Caught an AmazonServiceException, which means your request made it "
              + "to Amazon S3, but was rejected with an error response for some reason.");
      System.out.println("Error Message:    " + ase.getMessage());
      System.out.println("HTTP Status Code: " + ase.getStatusCode());
      System.out.println("AWS Error Code:   " + ase.getErrorCode());
      System.out.println("Error Type:       " + ase.getErrorType());
      System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println(
          "Caught an AmazonClientException, which means the client encountered "
              + "a serious internal problem while trying to communicate with S3, "
              + "such as not being able to access the network.");
      System.out.println("Error Message: " + ace.getMessage());
    }
  }
    public void storeMessageInQueue(String message) {
      // System.out.println("Storing Message...");

      // Send a message

      if (driectDbConnection) {
        DbConnector db = null;
        try {
          db = dbf.createDBConnectorInstance();
        } catch (MailAppDBException e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }

        MailAppMessage mailMsg =
            new MailAppMessage(message, "" + System.currentTimeMillis() + "_" + UUID.randomUUID());
        try {
          // printDebug("Calling DB-Instance to store message "+mailMsg.getTimeuuid()+" length:
          // "+mailMsg.getSize()+"\n"+mailMsg.getWholeMessageWITHOUT2LINES());
          db.storeMessage(mailMsg);

        } catch (MailAppDBException mae) {
          // MailStoringWorkerStarter.stopMailStoringWorker(MailStoringWorkerStarter.getWorkerList(), this);
          // stop=true;
          mae.printStackTrace();
        }

      } else {

        try {
          sqs.sendMessage(new SendMessageRequest(mailQueueUrl, message));
        } catch (AmazonServiceException ase) {
          System.out.println(
              "Caught an AmazonServiceException, which means your request made it "
                  + "to Amazon SQS, but was rejected with an error response for some reason.");
          System.out.println("Error Message:    " + ase.getMessage());
          System.out.println("HTTP Status Code: " + ase.getStatusCode());
          System.out.println("AWS Error Code:   " + ase.getErrorCode());
          System.out.println("Error Type:       " + ase.getErrorType());
          System.out.println("Request ID:       " + ase.getRequestId());
        } catch (AmazonClientException ace) {
          System.out.println(
              "Caught an AmazonClientException, which means the client encountered "
                  + "a serious internal problem while trying to communicate with SQS, such as not "
                  + "being able to access the network.");
          System.out.println("Error Message: " + ace.getMessage());
        }

        printDebug("Message stored to " + mailQueueUrl);

        // connect to db ( Cassandra ) and store message
        // OR write Message to SQS to be consumed by Storage Worker ->
        // better? more scalable, db independent

      }
    }
 private void logException(AmazonServiceException ase) {
   logger.error(
       "AmazonServiceException: error={}, statuscode={}, "
           + "awserrcode={}, errtype={}, reqid={}",
       ase.getMessage(),
       ase.getStatusCode(),
       ase.getErrorCode(),
       ase.getErrorType(),
       ase.getRequestId());
 }
Ejemplo n.º 6
0
  public boolean uploadToS3(String fileName, String text) throws IOException {

    try {
      String folderName = this.p.getProperty("KEY_NAME");
      String bucketName = this.p.getProperty("AWS_BUCKET_NAME");
      logger.info(LogKey.MESSAGE, "Uploading a new object to S3 from a file");

      byte[] contentAsBytes = text.getBytes("UTF-8");
      ByteArrayInputStream contentsAsStream = new ByteArrayInputStream(contentAsBytes);
      ObjectMetadata md = new ObjectMetadata();
      md.setContentLength(contentAsBytes.length);

      if (this.s3client == null) {
        logger.debug(LogKey.MESSAGE, "reuse S3 client !!");
        this.s3client = this.getS3Client();
      }

      logger.info(LogKey.BUCKET_NAME, bucketName);
      logger.info(LogKey.KEY_NAME, folderName);
      logger.info(LogKey.FILE_NAME, fileName);

      this.s3client.putObject(
          new PutObjectRequest(
              bucketName, folderName + File.separator + fileName, contentsAsStream, md));

      return true;

    } catch (AmazonServiceException ase) {
      logger.error(LogKey.EXCEPTION, ase);
      logger.error(
          LogKey.EXCEPTION,
          "Caught an AmazonServiceException, which "
              + "means your request made it "
              + "to Amazon S3, but was rejected with an error response"
              + " for some reason.");
      logger.error(LogKey.EXCEPTION, "Error Message:    " + ase.getMessage());
      logger.error(LogKey.EXCEPTION, "HTTP Status Code: " + ase.getStatusCode());
      logger.error(LogKey.EXCEPTION, "AWS Error Code:   " + ase.getErrorCode());
      logger.error(LogKey.EXCEPTION, "Error Type:       " + ase.getErrorType());
      logger.error(LogKey.EXCEPTION, "Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      logger.error(
          LogKey.EXCEPTION,
          "Caught an AmazonClientException, which "
              + "means the client encountered "
              + "an internal error while trying to "
              + "communicate with S3, "
              + "such as not being able to access the network.");
      logger.error(LogKey.EXCEPTION, "Error Message: " + ace.getMessage());
    }
    return false;
  }
  @Override
  public void delete(final TranscodingJob transcodingJob) {
    final ImmutableMap.Builder<String, AttributeValue> keyBuilder = ImmutableMap.builder();
    keyBuilder.put("transcode-job-id", new AttributeValue(transcodingJob.getId()));

    try {
      this.dynamoDB.deleteItem(TABLE_NAME, keyBuilder.build());
    } catch (final AmazonServiceException e) {
      logger.error(
          "AmazonServiceException while storing {}.  RID: {}", transcodingJob, e.getRequestId(), e);
    } catch (final AmazonClientException e) {
      logger.error("AmazonClientException while storing {}.", transcodingJob, e);
    }
  }
  @Override
  public void store(final TranscodingJob transcodingJob) {
    final ImmutableMap.Builder<String, AttributeValue> itemBuilder = ImmutableMap.builder();
    itemBuilder.put("transcode-job-id", new AttributeValue(transcodingJob.getId()));
    itemBuilder.put("inputMovie", new AttributeValue(transcodingJob.getInputPath().toString()));
    itemBuilder.put("outputMovie", new AttributeValue(transcodingJob.getOutputPath().toString()));

    try {
      this.dynamoDB.putItem(TABLE_NAME, itemBuilder.build());
    } catch (final AmazonServiceException e) {
      logger.error(
          "AmazonServiceException while storing {}.  RID: {}", transcodingJob, e.getRequestId(), e);
    } catch (final AmazonClientException e) {
      logger.error("AmazonClientException while storing {}.", transcodingJob, e);
    }
  }
Ejemplo n.º 9
0
  @Inject
  public AmazonS3Storage(Configuration configuration) {
    bucketName = configuration.getString("storage.s3.bucket");

    String accessKey = configuration.getString("storage.s3.accesskey");
    String secretKey = configuration.getString("storage.s3.secretkey");
    credentials = new BasicAWSCredentials(accessKey, secretKey);

    AmazonS3 amazonS3 = new AmazonS3Client(credentials);

    try {
      if (!(amazonS3.doesBucketExist(bucketName))) {
        amazonS3.createBucket(new CreateBucketRequest(bucketName));
      }

      String bucketLocation = amazonS3.getBucketLocation(new GetBucketLocationRequest(bucketName));
      Logger.info("Amazon S3 bucket created at " + bucketLocation);
    } catch (AmazonServiceException ase) {
      Logger.error(
          "Caught an AmazonServiceException, which "
              + "means your request made it "
              + "to Amazon S3, but was rejected with an error response "
              + "for some reason."
              + " Error Message: "
              + ase.getMessage()
              + " HTTP Status Code: "
              + ase.getStatusCode()
              + " AWS Error Code: "
              + ase.getErrorCode()
              + " Error Type: "
              + ase.getErrorType()
              + " Request ID: "
              + ase.getRequestId());
    } catch (AmazonClientException ace) {
      Logger.error(
          "Caught an AmazonClientException, which "
              + "means the client encountered "
              + "an internal error while trying to "
              + "communicate with S3, "
              + "such as not being able to access the network."
              + " Error Message: "
              + ace.getMessage());
    }
  }
  @Override
  public TranscodingJob get(final String jobId) {
    final ImmutableMap.Builder<String, AttributeValue> keyBuilder = ImmutableMap.builder();
    keyBuilder.put("transcode-job-id", new AttributeValue(jobId));

    try {
      final GetItemResult result = this.dynamoDB.getItem(TABLE_NAME, keyBuilder.build());
      return new TranscodingJob(
          result.getItem().get("transcode-job-id").getS(),
          Paths.get(result.getItem().get("inputMovie").getS()),
          Paths.get(result.getItem().get("outputMovie").getS()));
    } catch (final AmazonServiceException e) {
      logger.error("AmazonServiceException while getting {}.  RID: {}", jobId, e.getRequestId(), e);
      throw e;
    } catch (final AmazonClientException e) {
      logger.error("AmazonClientException while getting {}.", jobId, e);
      throw e;
    }
  }
Ejemplo n.º 11
0
  public static String xs3_generate_url(String xs3_objname, String content_type) {
    AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
    ClientConfiguration xs3_clientconfig = new ClientConfiguration();
    xs3_clientconfig.setProtocol(Protocol.HTTP);

    S3ClientOptions xs3_client_options = new S3ClientOptions();
    xs3_client_options.setPathStyleAccess(true);

    xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
    xs3_client.setEndpoint(xs3_endpoint);
    xs3_client.setS3ClientOptions(xs3_client_options);

    try {
      java.util.Date expiration = new java.util.Date();

      long milliSeconds = expiration.getTime();
      milliSeconds += 1000 * 60 * 5;
      expiration.setTime(milliSeconds);

      GeneratePresignedUrlRequest xs3_genurl_req =
          new GeneratePresignedUrlRequest(xs3_bucketname, xs3_objname);
      xs3_genurl_req.setMethod(HttpMethod.PUT);
      xs3_genurl_req.setExpiration(expiration);
      xs3_genurl_req.setContentType(content_type);
      xs3_genurl_req.addRequestParameter("x-amz-acl", "public-read");

      URL url = xs3_client.generatePresignedUrl(xs3_genurl_req);

      System.out.println(url.toString());
      return url.toString();

    } catch (AmazonServiceException ase) {
      System.out.println("xs3_svr_error_message:" + ase.getMessage());
      System.out.println("xs3_svr_status_code:  " + ase.getStatusCode());
      System.out.println("xs3_svr_error_code:   " + ase.getErrorCode());
      System.out.println("xs3_svr_error_type:   " + ase.getErrorType());
      System.out.println("xs3_svr_request_id:   " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println("xs3_clt_error_message:" + ace.getMessage());
    }

    return null;
  }
Ejemplo n.º 12
0
  public static String xs3_init_multi_upload(String xs3_objname, int file_size, String file_type) {
    AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
    ClientConfiguration xs3_clientconfig = new ClientConfiguration();
    xs3_clientconfig.setProtocol(Protocol.HTTP);

    S3ClientOptions xs3_client_options = new S3ClientOptions();
    xs3_client_options.setPathStyleAccess(true);

    xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
    xs3_client.setEndpoint(xs3_endpoint);
    xs3_client.setS3ClientOptions(xs3_client_options);

    try {
      InitiateMultipartUploadRequest xs3_multi_req =
          new InitiateMultipartUploadRequest(xs3_bucketname, xs3_objname);
      xs3_multi_req.setCannedACL(CannedAccessControlList.PublicRead);
      ObjectMetadata xs3_meta = new ObjectMetadata();
      xs3_meta.setContentType(file_type);
      xs3_multi_req.setObjectMetadata(xs3_meta);

      InitiateMultipartUploadResult xs3_multi_res =
          xs3_client.initiateMultipartUpload(xs3_multi_req);

      String xs3_multi_uploadid = xs3_multi_res.getUploadId();

      String json_urls = gen_part_url(xs3_multi_uploadid, file_size, xs3_objname, file_type);
      return json_urls;

    } catch (AmazonServiceException ase) {
      System.out.println("xs3_svr_error_message:" + ase.getMessage());
      System.out.println("xs3_svr_status_code:  " + ase.getStatusCode());
      System.out.println("xs3_svr_error_code:   " + ase.getErrorCode());
      System.out.println("xs3_svr_error_type:   " + ase.getErrorType());
      System.out.println("xs3_svr_request_id:   " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println("xs3_clt_error_message:" + ace.getMessage());
    }
    return null;
  }
Ejemplo n.º 13
0
  public static void xs3_coplete_multi_upload(String xs3_objname, String uploadId) {
    AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
    ClientConfiguration xs3_clientconfig = new ClientConfiguration();
    xs3_clientconfig.setProtocol(Protocol.HTTP);

    S3ClientOptions xs3_client_options = new S3ClientOptions();
    xs3_client_options.setPathStyleAccess(true);

    xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
    xs3_client.setEndpoint(xs3_endpoint);
    xs3_client.setS3ClientOptions(xs3_client_options);

    try {
      List<PartETag> rest_parts = listPartsXml(xs3_objname, uploadId);
      if (null == rest_parts) {
        return;
      }
      for (PartETag item : rest_parts) {
        System.out.println(item.getETag() + " -> " + item.getPartNumber());
      }

      CompleteMultipartUploadRequest comp_req =
          new CompleteMultipartUploadRequest(xs3_bucketname, xs3_objname, uploadId, rest_parts);

      CompleteMultipartUploadResult comp_result = xs3_client.completeMultipartUpload(comp_req);

      System.out.println(comp_result.getETag());
      System.out.println(comp_result.getKey());
    } catch (AmazonServiceException ase) {
      System.out.println("xs3_svr_error_message:" + ase.getMessage());
      System.out.println("xs3_svr_status_code:  " + ase.getStatusCode());
      System.out.println("xs3_svr_error_code:   " + ase.getErrorCode());
      System.out.println("xs3_svr_error_type:   " + ase.getErrorType());
      System.out.println("xs3_svr_request_id:   " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println("xs3_clt_error_message:" + ace.getMessage());
    }
  }
Ejemplo n.º 14
0
  /**
   * @param args
   * @throws IOException
   */
  public static void main(String[] args) throws IOException {
    try {

      String myQueueUrl = "https://queue.amazonaws.com/034307772076/MyQueue";

      AmazonSQS sqs =
          new AmazonSQSClient(
              new PropertiesCredentials(
                  SimpleQueueServiceSample.class.getResourceAsStream("AwsCredentials.properties")));

      System.out.println("Sending message 1.");
      sqs.sendMessage(new SendMessageRequest(myQueueUrl, "Test message."));

      System.out.println("Receiving messages:");
      ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(myQueueUrl);
      List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages();
      for (Message message : messages) {
        System.out.print(message.getMessageId());
        System.out.println(" : " + message.getBody());
      }
    } catch (AmazonServiceException ase) {
      System.out.println(
          "Caught an AmazonServiceException, which means your request made it "
              + "to Amazon SQS, but was rejected with an error response for some reason.");
      System.out.println("Error Message:    " + ase.getMessage());
      System.out.println("HTTP Status Code: " + ase.getStatusCode());
      System.out.println("AWS Error Code:   " + ase.getErrorCode());
      System.out.println("Error Type:       " + ase.getErrorType());
      System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println(
          "Caught an AmazonClientException, which means the client encountered "
              + "a serious internal problem while trying to communicate with SQS, such as not "
              + "being able to access the network.");
      System.out.println("Error Message: " + ace.getMessage());
    }
  }
  /** @param args */
  public static void main(String[] args) {
    // ============================================================================================//
    // =============================== Submitting a Request
    // =======================================//
    // ============================================================================================//

    // Retrieves the credentials from an AWSCredentials.properties file.
    AWSCredentials credentials = null;
    try {
      credentials =
          new PropertiesCredentials(
              InlineTaggingCodeSampleApp.class.getResourceAsStream("AwsCredentials.properties"));
    } catch (IOException e1) {
      System.out.println("Credentials were not properly entered into AwsCredentials.properties.");
      System.out.println(e1.getMessage());
      System.exit(-1);
    }

    // Create the AmazonEC2Client object so we can call various APIs.
    AmazonEC2 ec2 = new AmazonEC2Client(credentials);

    // Initializes a Spot Instance Request
    RequestSpotInstancesRequest requestRequest = new RequestSpotInstancesRequest();

    // Request 1 x t1.micro instance with a bid price of $0.03.
    requestRequest.setSpotPrice("0.03");
    requestRequest.setInstanceCount(Integer.valueOf(1));

    // Setup the specifications of the launch. This includes the instance type (e.g. t1.micro)
    // and the latest Amazon Linux AMI id available. Note, you should always use the latest
    // Amazon Linux AMI id or another of your choosing.
    LaunchSpecification launchSpecification = new LaunchSpecification();
    launchSpecification.setImageId("ami-8c1fece5");
    launchSpecification.setInstanceType("t1.micro");

    // Add the security group to the request.
    ArrayList<String> securityGroups = new ArrayList<String>();
    securityGroups.add("GettingStartedGroup");
    launchSpecification.setSecurityGroups(securityGroups);

    // Add the launch specifications to the request.
    requestRequest.setLaunchSpecification(launchSpecification);

    // ============================================================================================//
    // =========================== Getting the Request ID from the Request
    // ========================//
    // ============================================================================================//

    // Call the RequestSpotInstance API.
    RequestSpotInstancesResult requestResult = ec2.requestSpotInstances(requestRequest);
    List<SpotInstanceRequest> requestResponses = requestResult.getSpotInstanceRequests();

    // Setup an arraylist to collect all of the request ids we want to watch hit the running
    // state.
    ArrayList<String> spotInstanceRequestIds = new ArrayList<String>();

    // Add all of the request ids to the hashset, so we can determine when they hit the
    // active state.
    for (SpotInstanceRequest requestResponse : requestResponses) {
      System.out.println("Created Spot Request: " + requestResponse.getSpotInstanceRequestId());
      spotInstanceRequestIds.add(requestResponse.getSpotInstanceRequestId());
    }

    // ============================================================================================//
    // ====================================== Tag the Spot Requests
    // ===============================//
    // ============================================================================================//

    // Create the list of tags we want to create
    ArrayList<Tag> requestTags = new ArrayList<Tag>();
    requestTags.add(new Tag("keyname1", "value1"));

    // Create a tag request for requests.
    CreateTagsRequest createTagsRequest_requests = new CreateTagsRequest();
    createTagsRequest_requests.setResources(spotInstanceRequestIds);
    createTagsRequest_requests.setTags(requestTags);

    // Try to tag the Spot request submitted.
    try {
      ec2.createTags(createTagsRequest_requests);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error terminating instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }

    // ============================================================================================//
    // =========================== Determining the State of the Spot Request
    // ======================//
    // ============================================================================================//

    // Create a variable that will track whether there are any requests still in the open state.
    boolean anyOpen;

    // Initialize variables.
    ArrayList<String> instanceIds = new ArrayList<String>();

    do {
      // Create the describeRequest with tall of the request id to monitor (e.g. that we started).
      DescribeSpotInstanceRequestsRequest describeRequest =
          new DescribeSpotInstanceRequestsRequest();
      describeRequest.setSpotInstanceRequestIds(spotInstanceRequestIds);

      // Initialize the anyOpen variable to false – which assumes there are no requests open unless
      // we find one that is still open.
      anyOpen = false;

      try {
        // Retrieve all of the requests we want to monitor.
        DescribeSpotInstanceRequestsResult describeResult =
            ec2.describeSpotInstanceRequests(describeRequest);
        List<SpotInstanceRequest> describeResponses = describeResult.getSpotInstanceRequests();

        // Look through each request and determine if they are all in the active state.
        for (SpotInstanceRequest describeResponse : describeResponses) {
          // If the state is open, it hasn't changed since we attempted to request it.
          // There is the potential for it to transition almost immediately to closed or
          // cancelled so we compare against open instead of active.
          if (describeResponse.getState().equals("open")) {
            anyOpen = true;
            break;
          }

          // Add the instance id to the list we will eventually terminate.
          instanceIds.add(describeResponse.getInstanceId());
        }
      } catch (AmazonServiceException e) {
        // If we have an exception, ensure we don't break out of the loop.
        // This prevents the scenario where there was blip on the wire.
        anyOpen = true;
      }

      try {
        // Sleep for 60 seconds.
        Thread.sleep(60 * 1000);
      } catch (Exception e) {
        // Do nothing because it woke up early.
      }
    } while (anyOpen);

    // ============================================================================================//
    // ====================================== Tag the Spot Instances
    // ===============================//
    // ============================================================================================//

    // Create the list of tags we want to create
    ArrayList<Tag> instanceTags = new ArrayList<Tag>();
    instanceTags.add(new Tag("keyname1", "value1"));

    // Create a tag request for instances.
    CreateTagsRequest createTagsRequest_instances = new CreateTagsRequest();
    createTagsRequest_instances.setResources(instanceIds);
    createTagsRequest_instances.setTags(instanceTags);

    // Try to tag the Spot instance started.
    try {
      ec2.createTags(createTagsRequest_instances);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error terminating instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }

    // ============================================================================================//
    // ====================================== Canceling the Request ==============================//
    // ============================================================================================//

    try {
      // Cancel requests.
      CancelSpotInstanceRequestsRequest cancelRequest =
          new CancelSpotInstanceRequestsRequest(spotInstanceRequestIds);
      ec2.cancelSpotInstanceRequests(cancelRequest);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error cancelling instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }

    // ============================================================================================//
    // =================================== Terminating any Instances
    // ==============================//
    // ============================================================================================//
    try {
      // Terminate instances.
      TerminateInstancesRequest terminateRequest = new TerminateInstancesRequest(instanceIds);
      ec2.terminateInstances(terminateRequest);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error terminating instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }
  }
Ejemplo n.º 16
0
  public static void start() {
    AWSCredentials credentials = null;
    try {
      credentials =
          new PropertiesCredentials(
              CreateInitialVMs.class.getResourceAsStream("AwsCredentials.properties"));
    } catch (IOException e1) {
      // TODO Auto-generated catch block
      e1.printStackTrace();
    }

    /**
     * *******************************************
     *
     * <p>#1 Create Amazon Client object
     *
     * <p>*******************************************
     */
    System.out.println("#1 Create Amazon Client object");
    ec2 = new AmazonEC2Client(credentials);

    try {

      /**
       * *******************************************
       *
       * <p>#6 Create an Instance
       *
       * <p>*******************************************
       */
      System.out.println("#5 Create an Instance");

      String imageId = "ami-76f0061f"; // Basic 32-bit Amazon Linux AMI
      int minInstanceCount = 1;
      int maxInstanceCount = 2; // create 2 instances
      String keyPairName = "testElastic";
      String securityGroupName = "Prachi";
      ArrayList<String> securityGroup = new ArrayList<String>();
      securityGroup.add(securityGroupName);

      RunInstancesRequest rir =
          new RunInstancesRequest(imageId, minInstanceCount, maxInstanceCount);
      rir.setKeyName(keyPairName);
      rir.setSecurityGroups(securityGroup);
      Placement p = new Placement();
      p.setAvailabilityZone("us-east-1a");
      rir.setPlacement(p);

      RunInstancesResult result = ec2.runInstances(rir);

      // get instanceId from the result
      List<Instance> resultInstance = result.getReservation().getInstances();
      String createdInstanceId = null;
      for (Instance ins : resultInstance) {
        createdInstanceId = ins.getInstanceId();

        System.out.println("State of instance " + ins.getInstanceId() + " : ");
        while (true) {
          try {
            Thread.sleep(1000);
          } catch (InterruptedException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
          }
          System.out.print(ins.getState().getName() + "\n");

          ins = updatedInstance(ins);
          // if(ins.getPublicIpAddress()!= null){
          // if(ins.getState().getName().equals("running")){
          if (ins.getPublicIpAddress() != null && ins.getState().getName().equals("running")) {
            break;
          }
        }

        System.out.println("State: " + ins.getState().getName());

        System.out.println("New instance has been created: " + ins.getInstanceId());
        System.out.println("Instance Key: " + ins.getKeyName());
        System.out.println("Public DNS Name: " + ins.getPublicDnsName());
        System.out.println("Public DNS IP Address: " + ins.getPublicIpAddress());
        System.out.println("Instance ID: " + ins.getInstanceId());

        VMNode vmn = new VMNode();

        vmn.setInstanceId(ins.getInstanceId());

        /**
         * ******************************************* Allocate elastic IP addresses.
         * *******************************************
         */
        System.out.println("Public IP before association: " + ins.getPublicIpAddress());

        ins = allocateElasticIP(ins);

        vmn.setElasticIp(ins.getPublicIpAddress());

        System.out.println("Public IP after association: " + ins.getPublicIpAddress());

        /**
         * ******************************************* Create a volume
         * *******************************************
         */
        String createdVolumeId = createVolume();

        /**
         * ******************************************* Attach the volume to the instance
         * *******************************************
         */
        attachVolume(createdInstanceId, createdVolumeId);

        vmn.setVolumeId(createdVolumeId);

        /**
         * ******************************************* Detach the volume from the instance
         * *******************************************
         */
        detachVolume(createdInstanceId, createdVolumeId);

        /**
         * ********************************* Create an AMI from an instance
         * *******************************
         */
        String createdImageId = createAmi(createdInstanceId);

        vmn.setAmiId(createdImageId);
        nodes.add(vmn);

        /** ********************************* Stop the instance ******************************* */
        // stopInstance(ins.getInstanceId());
        // Diassociate Elastic IP from instance
        disassociateElasticIp(ins.getPublicIpAddress());
      }

      System.out.println("Listing Nodes: ");
      for (VMNode n : nodes) {
        System.out.println("Instance ID: " + n.getInstanceid());
        System.out.println("Elastic IP: " + n.getElasticIp());
        System.out.println("Volume ID: " + n.getVolumeId());
        System.out.println("AMI ID: " + n.getAmiId());
        System.out.println();
      }

      /**
       * *******************************************
       *
       * <p>#10 shutdown client object
       *
       * <p>*******************************************
       */
      ec2.shutdown();

    } catch (AmazonServiceException ase) {
      System.out.println("Caught Exception: " + ase.getMessage());
      System.out.println("Reponse Status Code: " + ase.getStatusCode());
      System.out.println("Error Code: " + ase.getErrorCode());
      System.out.println("Request ID: " + ase.getRequestId());
    }
  }
Ejemplo n.º 17
0
  public static String gen_part_url(
      String uploadId, int file_size, String file_name, String file_type) {
    AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
    ClientConfiguration xs3_clientconfig = new ClientConfiguration();
    xs3_clientconfig.setProtocol(Protocol.HTTP);

    S3ClientOptions xs3_client_options = new S3ClientOptions();
    xs3_client_options.setPathStyleAccess(true);

    xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
    xs3_client.setEndpoint(xs3_endpoint);
    xs3_client.setS3ClientOptions(xs3_client_options);

    try {

      final int xs3_part_size = 1024 * 1024 * 5;

      int xs3_part_count = (int) Math.ceil((double) (file_size) / (double) xs3_part_size);
      JSONArray jsonArray = new JSONArray();
      JSONObject jsonObject_1 = new JSONObject();
      jsonObject_1.put("total_num", xs3_part_count);
      jsonObject_1.put("upload_id", uploadId);

      JSONArray jsonArray_sub = new JSONArray();
      for (int part_no = 0; part_no < xs3_part_count; part_no++) {

        long xs3_offset_bytes = xs3_part_size * part_no;

        long part_size =
            xs3_part_size < (file_size - xs3_offset_bytes)
                ? xs3_part_size
                : (file_size - xs3_offset_bytes);

        java.util.Date expiration = new java.util.Date();
        long milliSeconds = expiration.getTime();
        milliSeconds += 1000 * 60 * 5;
        expiration.setTime(milliSeconds);

        GeneratePresignedUrlRequest xs3_genurl_req =
            new GeneratePresignedUrlRequest(xs3_bucketname, file_name);
        xs3_genurl_req.setMethod(HttpMethod.PUT);
        xs3_genurl_req.setExpiration(expiration);
        xs3_genurl_req.setContentType(file_type);
        xs3_genurl_req.addRequestParameter("uploadId", uploadId);
        xs3_genurl_req.addRequestParameter("partNumber", String.valueOf(part_no + 1));

        URL url = xs3_client.generatePresignedUrl(xs3_genurl_req);
        System.out.println(url.toString());

        JSONObject jsonObject = new JSONObject();
        jsonObject.put("part_idx", part_no);
        jsonObject.put("part_url", url.toString());
        jsonObject.put("upload_len", part_size);
        jsonObject.put("part_begin", xs3_offset_bytes);
        jsonObject.put("part_end", xs3_offset_bytes + part_size);

        jsonArray_sub.add(jsonObject);
      }

      jsonObject_1.put("multi_list", jsonArray_sub);
      jsonArray.add(jsonObject_1);

      return jsonArray.toString();

    } catch (AmazonServiceException ase) {
      System.out.println("xs3_svr_error_message:" + ase.getMessage());
      System.out.println("xs3_svr_status_code:  " + ase.getStatusCode());
      System.out.println("xs3_svr_error_code:   " + ase.getErrorCode());
      System.out.println("xs3_svr_error_type:   " + ase.getErrorType());
      System.out.println("xs3_svr_request_id:   " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println("xs3_clt_error_message:" + ace.getMessage());
    }

    return null;
  }
Ejemplo n.º 18
0
  /**
   * Internal method to execute the HTTP method given.
   *
   * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler)
   * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler,
   *     ExecutionContext)
   */
  private <T> Response<T> executeHelper(
      Request<?> request,
      HttpResponseHandler<AmazonWebServiceResponse<T>> responseHandler,
      HttpResponseHandler<AmazonServiceException> errorResponseHandler,
      ExecutionContext executionContext)
      throws AmazonClientException, AmazonServiceException {
    /*
     * Depending on which response handler we end up choosing to handle the
     * HTTP response, it might require us to leave the underlying HTTP
     * connection open, depending on whether or not it reads the complete
     * HTTP response stream from the HTTP connection, or if delays reading
     * any of the content until after a response is returned to the caller.
     */
    boolean leaveHttpConnectionOpen = false;
    AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics();
    /* add the service endpoint to the logs. You can infer service name from service endpoint */
    awsRequestMetrics.addProperty(Field.ServiceName, request.getServiceName());
    awsRequestMetrics.addProperty(Field.ServiceEndpoint, request.getEndpoint());
    // Apply whatever request options we know how to handle, such as user-agent.
    setUserAgent(request);
    int requestCount = 0;
    URI redirectedURI = null;
    HttpEntity entity = null;
    AmazonClientException retriedException = null;

    // Make a copy of the original request params and headers so that we can
    // permute it in this loop and start over with the original every time.
    Map<String, String> originalParameters = new LinkedHashMap<String, String>();
    originalParameters.putAll(request.getParameters());
    Map<String, String> originalHeaders = new HashMap<String, String>();
    originalHeaders.putAll(request.getHeaders());
    final AWSCredentials credentials = executionContext.getCredentials();
    Signer signer = null;

    while (true) {
      ++requestCount;
      awsRequestMetrics.setCounter(Field.RequestCount, requestCount);
      if (requestCount > 1) { // retry
        request.setParameters(originalParameters);
        request.setHeaders(originalHeaders);
      }
      HttpRequestBase httpRequest = null;
      org.apache.http.HttpResponse apacheResponse = null;

      try {
        // Sign the request if a signer was provided
        if (signer == null) signer = executionContext.getSignerByURI(request.getEndpoint());
        if (signer != null && credentials != null) {
          awsRequestMetrics.startEvent(Field.RequestSigningTime);
          try {
            signer.sign(request, credentials);
          } finally {
            awsRequestMetrics.endEvent(Field.RequestSigningTime);
          }
        }

        if (requestLog.isDebugEnabled()) {
          requestLog.debug("Sending Request: " + request.toString());
        }

        httpRequest = httpRequestFactory.createHttpRequest(request, config, executionContext);

        if (httpRequest instanceof HttpEntityEnclosingRequest) {
          entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity();
        }

        if (redirectedURI != null) {
          httpRequest.setURI(redirectedURI);
        }

        if (requestCount > 1) { // retry
          awsRequestMetrics.startEvent(Field.RetryPauseTime);
          try {
            pauseBeforeNextRetry(
                request.getOriginalRequest(),
                retriedException,
                requestCount,
                config.getRetryPolicy());
          } finally {
            awsRequestMetrics.endEvent(Field.RetryPauseTime);
          }
        }

        if (entity != null) {
          InputStream content = entity.getContent();
          if (requestCount > 1) { // retry
            if (content.markSupported()) {
              content.reset();
              content.mark(-1);
            }
          } else {
            if (content.markSupported()) {
              content.mark(-1);
            }
          }
        }

        captureConnectionPoolMetrics(httpClient.getConnectionManager(), awsRequestMetrics);
        HttpContext httpContext = new BasicHttpContext();
        httpContext.setAttribute(AWSRequestMetrics.class.getSimpleName(), awsRequestMetrics);
        retriedException = null;
        awsRequestMetrics.startEvent(Field.HttpRequestTime);
        try {
          apacheResponse = httpClient.execute(httpRequest, httpContext);
        } finally {
          awsRequestMetrics.endEvent(Field.HttpRequestTime);
        }

        if (isRequestSuccessful(apacheResponse)) {
          awsRequestMetrics.addProperty(
              Field.StatusCode, apacheResponse.getStatusLine().getStatusCode());
          /*
           * If we get back any 2xx status code, then we know we should
           * treat the service call as successful.
           */
          leaveHttpConnectionOpen = responseHandler.needsConnectionLeftOpen();
          HttpResponse httpResponse = createResponse(httpRequest, request, apacheResponse);
          T response =
              handleResponse(
                  request,
                  responseHandler,
                  httpRequest,
                  httpResponse,
                  apacheResponse,
                  executionContext);
          return new Response<T>(response, httpResponse);
        } else if (isTemporaryRedirect(apacheResponse)) {
          /*
           * S3 sends 307 Temporary Redirects if you try to delete an
           * EU bucket from the US endpoint. If we get a 307, we'll
           * point the HTTP method to the redirected location, and let
           * the next retry deliver the request to the right location.
           */
          Header[] locationHeaders = apacheResponse.getHeaders("location");
          String redirectedLocation = locationHeaders[0].getValue();
          log.debug("Redirecting to: " + redirectedLocation);
          redirectedURI = URI.create(redirectedLocation);
          httpRequest.setURI(redirectedURI);
          awsRequestMetrics.addProperty(
              Field.StatusCode, apacheResponse.getStatusLine().getStatusCode());
          awsRequestMetrics.addProperty(Field.RedirectLocation, redirectedLocation);
          awsRequestMetrics.addProperty(Field.AWSRequestID, null);

        } else {
          leaveHttpConnectionOpen = errorResponseHandler.needsConnectionLeftOpen();
          AmazonServiceException ase =
              handleErrorResponse(request, errorResponseHandler, httpRequest, apacheResponse);
          awsRequestMetrics.addProperty(Field.AWSRequestID, ase.getRequestId());
          awsRequestMetrics.addProperty(Field.AWSErrorCode, ase.getErrorCode());
          awsRequestMetrics.addProperty(Field.StatusCode, ase.getStatusCode());

          if (!shouldRetry(
              request.getOriginalRequest(),
              httpRequest,
              ase,
              requestCount,
              config.getRetryPolicy())) {
            throw ase;
          }

          // Cache the retryable exception
          retriedException = ase;
          /*
           * Checking for clock skew error again because we don't want to set the
           * global time offset for every service exception.
           */
          if (RetryUtils.isClockSkewError(ase)) {
            int timeOffset = parseClockSkewOffset(apacheResponse, ase);
            SDKGlobalConfiguration.setGlobalTimeOffset(timeOffset);
          }
          resetRequestAfterError(request, ase);
        }
      } catch (IOException ioe) {
        if (log.isInfoEnabled()) {
          log.info("Unable to execute HTTP request: " + ioe.getMessage(), ioe);
        }
        awsRequestMetrics.incrementCounter(Field.Exception);
        awsRequestMetrics.addProperty(Field.Exception, ioe);
        awsRequestMetrics.addProperty(Field.AWSRequestID, null);

        AmazonClientException ace =
            new AmazonClientException("Unable to execute HTTP request: " + ioe.getMessage(), ioe);
        if (!shouldRetry(
            request.getOriginalRequest(),
            httpRequest,
            ace,
            requestCount,
            config.getRetryPolicy())) {
          throw ace;
        }

        // Cache the retryable exception
        retriedException = ace;
        resetRequestAfterError(request, ioe);
      } catch (RuntimeException e) {
        throw handleUnexpectedFailure(e, awsRequestMetrics);
      } catch (Error e) {
        throw handleUnexpectedFailure(e, awsRequestMetrics);
      } finally {
        /*
         * Some response handlers need to manually manage the HTTP
         * connection and will take care of releasing the connection on
         * their own, but if this response handler doesn't need the
         * connection left open, we go ahead and release the it to free
         * up resources.
         */
        if (!leaveHttpConnectionOpen) {
          try {
            if (apacheResponse != null
                && apacheResponse.getEntity() != null
                && apacheResponse.getEntity().getContent() != null) {
              apacheResponse.getEntity().getContent().close();
            }
          } catch (IOException e) {
            log.warn("Cannot close the response content.", e);
          }
        }
      }
    } /* end while (true) */
  }
  /**
   * Internal method to execute the HTTP method given.
   *
   * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler)
   * @see AmazonHttpClient#execute(Request, HttpResponseHandler, HttpResponseHandler,
   *     ExecutionContext)
   */
  private <T extends Object> T executeHelper(
      Request<?> request,
      HttpResponseHandler<AmazonWebServiceResponse<T>> responseHandler,
      HttpResponseHandler<AmazonServiceException> errorResponseHandler,
      ExecutionContext executionContext)
      throws AmazonClientException, AmazonServiceException {

    /*
     * Depending on which response handler we end up choosing to handle the
     * HTTP response, it might require us to leave the underlying HTTP
     * connection open, depending on whether or not it reads the complete
     * HTTP response stream from the HTTP connection, or if delays reading
     * any of the content until after a response is returned to the caller.
     */
    boolean leaveHttpConnectionOpen = false;

    AWSRequestMetrics awsRequestMetrics = executionContext.getAwsRequestMetrics();
    /* add the service endpoint to the logs. You can infer service name from service endpoint */
    awsRequestMetrics.addProperty(Field.ServiceName.name(), request.getServiceName());
    awsRequestMetrics.addProperty(Field.ServiceEndpoint.name(), request.getEndpoint());

    // Apply whatever request options we know how to handle, such as user-agent.
    applyRequestData(request);

    int retryCount = 0;
    URI redirectedURI = null;
    HttpEntity entity = null;
    AmazonServiceException exception = null;

    // Make a copy of the original request params and headers so that we can
    // permute it in this loop and start over with the original every time.
    Map<String, String> originalParameters = new HashMap<String, String>();
    originalParameters.putAll(request.getParameters());
    Map<String, String> originalHeaders = new HashMap<String, String>();
    originalHeaders.putAll(request.getHeaders());

    while (true) {
      awsRequestMetrics.setCounter(Field.AttemptCount.name(), retryCount + 1);
      if (retryCount > 0) {
        request.setParameters(originalParameters);
        request.setHeaders(originalHeaders);
      }

      HttpRequestBase httpRequest = null;
      org.apache.http.HttpResponse response = null;

      try {
        // Sign the request if a signer was provided
        if (executionContext.getSigner() != null && executionContext.getCredentials() != null) {
          awsRequestMetrics.startEvent(Field.RequestSigningTime.name());
          executionContext.getSigner().sign(request, executionContext.getCredentials());
          awsRequestMetrics.endEvent(Field.RequestSigningTime.name());
        }

        if (requestLog.isDebugEnabled()) {
          requestLog.debug("Sending Request: " + request.toString());
        }

        httpRequest =
            httpRequestFactory.createHttpRequest(request, config, entity, executionContext);

        if (httpRequest instanceof HttpEntityEnclosingRequest) {
          entity = ((HttpEntityEnclosingRequest) httpRequest).getEntity();
        }

        if (redirectedURI != null) {
          httpRequest.setURI(redirectedURI);
        }

        if (retryCount > 0) {
          awsRequestMetrics.startEvent(Field.RetryPauseTime.name());
          pauseExponentially(retryCount, exception, executionContext.getCustomBackoffStrategy());
          awsRequestMetrics.endEvent(Field.RetryPauseTime.name());
        }

        if (entity != null) {
          InputStream content = entity.getContent();
          if (retryCount > 0) {
            if (content.markSupported()) {
              content.reset();
              content.mark(-1);
            }
          } else {
            if (content.markSupported()) {
              content.mark(-1);
            }
          }
        }

        exception = null;

        awsRequestMetrics.startEvent(Field.HttpRequestTime.name());
        response = httpClient.execute(httpRequest);
        awsRequestMetrics.endEvent(Field.HttpRequestTime.name());

        if (isRequestSuccessful(response)) {

          awsRequestMetrics.addProperty(
              Field.StatusCode.name(), response.getStatusLine().getStatusCode());

          /*
           * If we get back any 2xx status code, then we know we should
           * treat the service call as successful.
           */
          leaveHttpConnectionOpen = responseHandler.needsConnectionLeftOpen();
          return handleResponse(request, responseHandler, httpRequest, response, executionContext);
        } else if (isTemporaryRedirect(response)) {
          /*
           * S3 sends 307 Temporary Redirects if you try to delete an
           * EU bucket from the US endpoint. If we get a 307, we'll
           * point the HTTP method to the redirected location, and let
           * the next retry deliver the request to the right location.
           */
          Header[] locationHeaders = response.getHeaders("location");
          String redirectedLocation = locationHeaders[0].getValue();
          log.debug("Redirecting to: " + redirectedLocation);
          redirectedURI = URI.create(redirectedLocation);
          httpRequest.setURI(redirectedURI);
          awsRequestMetrics.addProperty(
              Field.StatusCode.name(), response.getStatusLine().getStatusCode());
          awsRequestMetrics.addProperty(Field.RedirectLocation.name(), redirectedLocation);
          awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null);

        } else {
          leaveHttpConnectionOpen = errorResponseHandler.needsConnectionLeftOpen();
          exception = handleErrorResponse(request, errorResponseHandler, httpRequest, response);
          awsRequestMetrics.addProperty(Field.AWSRequestID.name(), exception.getRequestId());
          awsRequestMetrics.addProperty(Field.AWSErrorCode.name(), exception.getErrorCode());
          awsRequestMetrics.addProperty(Field.StatusCode.name(), exception.getStatusCode());

          if (!shouldRetry(httpRequest, exception, retryCount)) {
            throw exception;
          }
          resetRequestAfterError(request, exception);
        }
      } catch (IOException ioe) {
        log.info("Unable to execute HTTP request: " + ioe.getMessage(), ioe);
        awsRequestMetrics.addProperty(Field.Exception.name(), ioe.toString());
        awsRequestMetrics.addProperty(Field.AWSRequestID.name(), null);

        if (!shouldRetry(httpRequest, ioe, retryCount)) {
          throw new AmazonClientException(
              "Unable to execute HTTP request: " + ioe.getMessage(), ioe);
        }
        resetRequestAfterError(request, ioe);
      } finally {
        retryCount++;

        /*
         * Some response handlers need to manually manage the HTTP
         * connection and will take care of releasing the connection on
         * their own, but if this response handler doesn't need the
         * connection left open, we go ahead and release the it to free
         * up resources.
         */
        if (!leaveHttpConnectionOpen) {
          try {
            response.getEntity().getContent().close();
          } catch (Throwable t) {
          }
        }
      }
    } /* end while (true) */
  }
Ejemplo n.º 20
0
  public static void main(String[] args) throws Exception {
    System.out.println("===========================================");
    System.out.println("Welcome to the AWS Java SDK!");
    System.out.println("===========================================");

    init();

    try {
      /*
       * The Amazon EC2 client allows you to easily launch and configure
       * computing capacity in AWS datacenters.
       *
       * In this sample, we use the EC2 client to list the availability zones
       * in a region, and then list the instances running in those zones.
       */
      DescribeAvailabilityZonesResult availabilityZonesResult = ec2.describeAvailabilityZones();
      List<AvailabilityZone> availabilityZones = availabilityZonesResult.getAvailabilityZones();
      System.out.println("You have access to " + availabilityZones.size() + " availability zones:");
      for (AvailabilityZone zone : availabilityZones) {
        System.out.println(" - " + zone.getZoneName() + " (" + zone.getRegionName() + ")");
      }

      DescribeInstancesResult describeInstancesResult = ec2.describeInstances();
      Set<Instance> instances = new HashSet<Instance>();
      for (Reservation reservation : describeInstancesResult.getReservations()) {
        instances.addAll(reservation.getInstances());
      }

      System.out.println("You have " + instances.size() + " Amazon EC2 instance(s) running.");

      /*
       * The Amazon S3 client allows you to manage and configure buckets
       * and to upload and download data.
       *
       * In this sample, we use the S3 client to list all the buckets in
       * your account, and then iterate over the object metadata for all
       * objects in one bucket to calculate the total object count and
       * space usage for that one bucket. Note that this sample only
       * retrieves the object's metadata and doesn't actually download the
       * object's content.
       *
       * In addition to the low-level Amazon S3 client in the SDK, there
       * is also a high-level TransferManager API that provides
       * asynchronous management of uploads and downloads with an easy to
       * use API:
       *   http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/transfer/TransferManager.html
       */
      List<Bucket> buckets = s3.listBuckets();
      System.out.println("You have " + buckets.size() + " Amazon S3 bucket(s).");

      if (buckets.size() > 0) {
        Bucket bucket = buckets.get(0);

        long totalSize = 0;
        long totalItems = 0;
        /*
         * The S3Objects and S3Versions classes provide convenient APIs
         * for iterating over the contents of your buckets, without
         * having to manually deal with response pagination.
         */
        for (S3ObjectSummary objectSummary : S3Objects.inBucket(s3, bucket.getName())) {
          totalSize += objectSummary.getSize();
          totalItems++;
        }

        System.out.println(
            "The bucket '"
                + bucket.getName()
                + "' contains "
                + totalItems
                + " objects "
                + "with a total size of "
                + totalSize
                + " bytes.");
      }
    } catch (AmazonServiceException ase) {
      /*
       * AmazonServiceExceptions represent an error response from an AWS
       * services, i.e. your request made it to AWS, but the AWS service
       * either found it invalid or encountered an error trying to execute
       * it.
       */
      System.out.println("Error Message:    " + ase.getMessage());
      System.out.println("HTTP Status Code: " + ase.getStatusCode());
      System.out.println("AWS Error Code:   " + ase.getErrorCode());
      System.out.println("Error Type:       " + ase.getErrorType());
      System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      /*
       * AmazonClientExceptions represent an error that occurred inside
       * the client on the local host, either while trying to send the
       * request to AWS or interpret the response. For example, if no
       * network connection is available, the client won't be able to
       * connect to AWS to execute a request and will throw an
       * AmazonClientException.
       */
      System.out.println("Error Message: " + ace.getMessage());
    }
  }
Ejemplo n.º 21
0
  public void run() {
    logger.debug("started GridManager");
    System.out.println("started GridManager");
    AmazonSQSClient sqsClient = new AmazonSQSClient(papa.getCredentials());
    AmazonEC2Client ec2Client = new AmazonEC2Client(papa.getCredentials());

    // Get current imageID from the metadata table
    String imageID = Constants.imageID; // get the default imageID from Constants
    try {
      Connection conn = PooledConnectionFactory.INSTANCE.getCumulusConnection();
      Statement stmt = conn.createStatement();
      String imageQuery = "SELECT mvalue FROM cumulus.metadata WHERE mkey='imageID';";
      ResultSet results = stmt.executeQuery(imageQuery);
      results.next(); // move the cursor into the results
      imageID = results.getString(1);
    } catch (SQLException e) {
      logger.warn(e);
    }
    if (imageID == null
        || imageID.equals("")) { // if we get an empty imageID revert to the hardcoded one
      imageID = Constants.imageID;
    }
    logger.debug(imageID);
    System.out.println(imageID);

    while (true) { // loop from startup to shutdown
      // check the number of messages in the queue.
      GetQueueAttributesRequest sqsRequest =
          new GetQueueAttributesRequest(papa.getDispatchQueue())
              .withAttributeNames("ApproximateNumberOfMessages");
      Integer sqsResult = null;
      try {
        sqsResult =
            Integer.valueOf(
                sqsClient
                    .getQueueAttributes(sqsRequest)
                    .getAttributes()
                    .get("ApproximateNumberOfMessages"));
      } catch (AmazonServiceException e) {
        // Write out any exceptions that may have occurred.
        System.err.println("Error getting list of instances");
        System.err.println("Caught Exception: " + e.getMessage());
        System.err.println("Reponse Status Code: " + e.getStatusCode());
        System.err.println("Error Code: " + e.getErrorCode());
        System.err.println("Request ID: " + e.getRequestId());
      } catch (AmazonClientException e) {
        logger.warn(e);
        continue; // if the error is due to being unable to connect to sqs it is likely transient,
                  // and we should keep going until SQS comes back
      } catch (Exception e) {
        // caught another exception
        System.err.println(e);
        System.err.println("Did not exit cleanly");
        // System.exit(1);
      }
      if (sqsResult == null) {
        System.err.println(
            "Couldn't get the number of items in the queue reverting to static functioning");
        logger.error(
            "Couldn't get the number of items in the queue reverting to static functioning");
        break;
      }
      System.out.println("Queue size=" + sqsResult);
      // check the number of running instances.
      DescribeInstancesRequest describeInstancesRequest = new DescribeInstancesRequest();
      // get the number of running instances with our image ID
      describeInstancesRequest.withFilters(
          new Filter("image-id").withValues(imageID),
          new Filter("instance-state-name").withValues("running"));
      logger.debug("Getting list of active cumulus drones");
      Integer numInstances = null;
      Integer numSpotRequests = null;
      Initializer.serialize("/tmp/dump.obj", papa.unitsOnServer); // TODO test this.
      // check number of open spot requests
      DescribeSpotInstanceRequestsRequest describeSpotInstancesRequest =
          new DescribeSpotInstanceRequestsRequest();
      describeSpotInstancesRequest.withFilters(new Filter("state").withValues("open"));
      try {
        // Parse returned instances
        DescribeInstancesResult describeInstancesResult =
            ec2Client.describeInstances(describeInstancesRequest);
        if (describeInstancesResult.getReservations().size() == 0) {
          numInstances = 0;
        } else {
          Integer totalInstances = 0;
          for (Reservation i : describeInstancesResult.getReservations()) {
            totalInstances += i.getInstances().size();
          }
          numInstances = totalInstances;
        }
        // Parse returned open spot requests
        DescribeSpotInstanceRequestsResult describeSpotReservationsResult =
            ec2Client.describeSpotInstanceRequests(describeSpotInstancesRequest);
        if (describeSpotReservationsResult.getSpotInstanceRequests().size() == 0) {
          numSpotRequests = 0;
        } else {
          Integer totalInstances = 0;
          for (SpotInstanceRequest i : describeSpotReservationsResult.getSpotInstanceRequests()) {
            totalInstances++;
          }
          numSpotRequests = totalInstances;
        }

        // System.out.println("num instances="+numInstances);
      } catch (AmazonServiceException e) {
        // Write out any exceptions that may have occurred.
        System.err.println("Error getting list of instances");
        System.err.println("Caught Exception: " + e.getMessage());
        System.err.println("Reponse Status Code: " + e.getStatusCode());
        System.err.println("Error Code: " + e.getErrorCode());
        System.err.println("Request ID: " + e.getRequestId());
        // System.exit(1);
      } catch (Exception e) {
        // caught another exception
        System.err.println(e);
        System.err.println("Did not exit cleanly");
        // System.exit(1);
      }
      if (numInstances == null || numSpotRequests == null) {
        logger.error(
            "Couldn't get the number of machines or requests reverting to static functioning");
        break;
      }
      System.out.println("num instances=" + (numInstances + numSpotRequests));

      // if queue is longer than the number of instances * idealMaxUnitsPerInstance
      Integer idealMaxUnitsPerInstance = Integer.valueOf(Constants.idealMaxUnitsPerInstance);
      if (sqsResult
          > (numInstances + numSpotRequests)
              * idealMaxUnitsPerInstance) { // we count pending spot instances too.
        Integer numToCreate = (sqsResult / idealMaxUnitsPerInstance) - numInstances + 1;
        // check current pricing of spot instances
        DescribeSpotPriceHistoryRequest historyRequest =
            new DescribeSpotPriceHistoryRequest()
                .withAvailabilityZone("us-east-1c")
                .withInstanceTypes(Constants.instanceType)
                .withProductDescriptions("Linux/UNIX (Amazon VPC)")
                .withStartTime(new Date(System.currentTimeMillis()));
        DescribeSpotPriceHistoryResult result = ec2Client.describeSpotPriceHistory(historyRequest);
        Double nowPrice = Double.valueOf(result.getSpotPriceHistory().get(0).getSpotPrice());
        // If spot price is < constants.spotPrice launch numToCreate*Constants.percentSpot spot
        // instances
        Integer spotToCreate = 0;
        if (nowPrice < Double.valueOf(Constants.spotPrice)) {
          Double percentSpot = Double.valueOf(Constants.percentSpot);
          spotToCreate = (int) (numToCreate * percentSpot);
          numToCreate = (int) (numToCreate * (1 - percentSpot));
        }
        try {
          System.out.println("regular instances: " + numToCreate);
          System.out.println("spot instances: " + spotToCreate);
          // Create regular and spot instances
          if (numToCreate > 0) {
            papa.createInstances(ec2Client, numToCreate, imageID);
          }
          if (spotToCreate > 0) {
            papa.createSpotInstances(ec2Client, spotToCreate, Constants.imageID); // FIXME
          }
          System.out.println("created instances");
        } catch (Exception e) {
          // Report ec2 exceptions
          logger.error(e);
          e.printStackTrace();
          continue;
        }
      }
      try {
        Thread.sleep(60000); // we wait for any new servers to start up
      } catch (InterruptedException e) {
        logger.error("Sqslistener was interrupted");
        if (papa.getShuttingDown()) {
          break;
        } else {
          continue;
        }
      }
    }
  }
Ejemplo n.º 22
0
  public static void main(String[] args) throws IOException {
    /*
     * This credentials provider implementation loads your AWS credentials
     * from a properties file at the root of your classpath.
     *
     * Important: Be sure to fill in your AWS access credentials in the
     *            AwsCredentials.properties file before you try to run this
     *            sample.
     * http://aws.amazon.com/security-credentials
     */
    AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());

    String bucketName = "my-first-s3-bucket-" + UUID.randomUUID();
    String key = "MyObjectKey";

    System.out.println("===========================================");
    System.out.println("Getting Started with Amazon S3");
    System.out.println("===========================================\n");

    try {
      /*
       * Create a new S3 bucket - Amazon S3 bucket names are globally unique,
       * so once a bucket name has been taken by any user, you can't create
       * another bucket with that same name.
       *
       * You can optionally specify a location for your bucket if you want to
       * keep your data closer to your applications or users.
       */
      System.out.println("Creating bucket " + bucketName + "\n");
      s3.createBucket(bucketName);

      /*
       * List the buckets in your account
       */
      System.out.println("Listing buckets");
      for (Bucket bucket : s3.listBuckets()) {
        System.out.println(" - " + bucket.getName());
      }
      System.out.println();

      /*
       * Upload an object to your bucket - You can easily upload a file to
       * S3, or upload directly an InputStream if you know the length of
       * the data in the stream. You can also specify your own metadata
       * when uploading to S3, which allows you set a variety of options
       * like content-type and content-encoding, plus additional metadata
       * specific to your applications.
       */
      System.out.println("Uploading a new object to S3 from a file\n");
      s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile()));

      /*
       * Download an object - When you download an object, you get all of
       * the object's metadata and a stream from which to read the contents.
       * It's important to read the contents of the stream as quickly as
       * possibly since the data is streamed directly from Amazon S3 and your
       * network connection will remain open until you read all the data or
       * close the input stream.
       *
       * GetObjectRequest also supports several other options, including
       * conditional downloading of objects based on modification times,
       * ETags, and selectively downloading a range of an object.
       */
      System.out.println("Downloading an object");
      S3Object object = s3.getObject(new GetObjectRequest(bucketName, key));
      System.out.println("Content-Type: " + object.getObjectMetadata().getContentType());
      displayTextInputStream(object.getObjectContent());

      /*
       * List objects in your bucket by prefix - There are many options for
       * listing the objects in your bucket.  Keep in mind that buckets with
       * many objects might truncate their results when listing their objects,
       * so be sure to check if the returned object listing is truncated, and
       * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve
       * additional results.
       */
      System.out.println("Listing objects");
      ObjectListing objectListing =
          s3.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My"));
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        System.out.println(
            " - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")");
      }
      System.out.println();

      /*
       * Delete an object - Unless versioning has been turned on for your bucket,
       * there is no way to undelete an object, so use caution when deleting objects.
       */
      System.out.println("Deleting an object\n");
      s3.deleteObject(bucketName, key);

      /*
       * Delete a bucket - A bucket must be completely empty before it can be
       * deleted, so remember to delete any objects from your buckets before
       * you try to delete them.
       */
      System.out.println("Deleting bucket " + bucketName + "\n");
      s3.deleteBucket(bucketName);
    } catch (AmazonServiceException ase) {
      System.out.println(
          "Caught an AmazonServiceException, which means your request made it "
              + "to Amazon S3, but was rejected with an error response for some reason.");
      System.out.println("Error Message:    " + ase.getMessage());
      System.out.println("HTTP Status Code: " + ase.getStatusCode());
      System.out.println("AWS Error Code:   " + ase.getErrorCode());
      System.out.println("Error Type:       " + ase.getErrorType());
      System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println(
          "Caught an AmazonClientException, which means the client encountered "
              + "a serious internal problem while trying to communicate with S3, "
              + "such as not being able to access the network.");
      System.out.println("Error Message: " + ace.getMessage());
    }
  }
  /** @param args */
  public static void main(String[] args) {
    // ============================================================================================//
    // =============================== Submitting a Request
    // =======================================//
    // ============================================================================================//

    // Create the AmazonEC2Client object so we can call various APIs.
    AmazonEC2 ec2 = new AmazonEC2Client(new ClasspathPropertiesFileCredentialsProvider());
    Region usWest2 = Region.getRegion(Regions.US_WEST_2);
    ec2.setRegion(usWest2);

    // Initializes a Spot Instance Request
    RequestSpotInstancesRequest requestRequest = new RequestSpotInstancesRequest();

    // *************************** Required Parameters Settings ************************//
    // Request 1 x t1.micro instance with a bid price of $0.03.
    requestRequest.setSpotPrice("0.03");
    requestRequest.setInstanceCount(Integer.valueOf(1));

    // Setup the specifications of the launch. This includes the instance type (e.g. t1.micro)
    // and the latest Amazon Linux AMI id available. Note, you should always use the latest
    // Amazon Linux AMI id or another of your choosing.
    LaunchSpecification launchSpecification = new LaunchSpecification();
    launchSpecification.setImageId("ami-8c1fece5");
    launchSpecification.setInstanceType("t1.micro");

    // Add the security group to the request.
    ArrayList<String> securityGroups = new ArrayList<String>();
    securityGroups.add("GettingStartedGroup");
    launchSpecification.setSecurityGroups(securityGroups);

    // *************************** Bid Type Settings ************************//
    // Set the type of the bid to persistent.
    requestRequest.setType("persistent");

    // *************************** Valid From/To Settings ************************//
    // Set the valid start time to be two minutes from now.
    Calendar from = Calendar.getInstance();
    from.add(Calendar.MINUTE, 2);
    requestRequest.setValidFrom(from.getTime());

    // Set the valid end time to be two minutes and two hours from now.
    Calendar until = (Calendar) from.clone();
    until.add(Calendar.HOUR, 2);
    requestRequest.setValidUntil(until.getTime());

    // *************************** Launch Group Settings ************************//
    // Set the launch group.
    requestRequest.setLaunchGroup("ADVANCED-DEMO-LAUNCH-GROUP");

    // *************************** Availability Zone Group Settings ************************//
    // Set the availability zone group.
    requestRequest.setAvailabilityZoneGroup("ADVANCED-DEMO-AZ-GROUP");

    // *************************** Add the block device mapping ************************//

    // Goal: Setup block device mappings to ensure that we will not delete
    // the root partition on termination.

    // Create the block device mapping to describe the root partition.
    BlockDeviceMapping blockDeviceMapping = new BlockDeviceMapping();
    blockDeviceMapping.setDeviceName("/dev/sda1");

    // Set the delete on termination flag to false.
    EbsBlockDevice ebs = new EbsBlockDevice();
    ebs.setDeleteOnTermination(Boolean.FALSE);
    blockDeviceMapping.setEbs(ebs);

    // Add the block device mapping to the block list.
    ArrayList<BlockDeviceMapping> blockList = new ArrayList<BlockDeviceMapping>();
    blockList.add(blockDeviceMapping);

    // Set the block device mapping configuration in the launch specifications.
    launchSpecification.setBlockDeviceMappings(blockList);

    // *************************** Add the availability zone ************************//
    // Setup the availability zone to use. Note we could retrieve the availability
    // zones using the ec2.describeAvailabilityZones() API. For this demo we will just use
    // us-east-1b.
    SpotPlacement placement = new SpotPlacement("us-east-1b");
    launchSpecification.setPlacement(placement);

    // *************************** Add the placement group ************************//
    // Setup the placement group to use with whatever name you desire.
    // For this demo we will just use "ADVANCED-DEMO-PLACEMENT-GROUP".
    // Note: We have commented this out, because we are not leveraging cc1.4xlarge or
    // cg1.4xlarge in this example.
    /*
    SpotPlacement pg = new SpotPlacement();
    pg.setGroupName("ADVANCED-DEMO-PLACEMENT-GROUP");
    launchSpecification.setPlacement(pg);
    */

    // *************************** Add the launch specification ************************//
    // Add the launch specification.
    requestRequest.setLaunchSpecification(launchSpecification);

    // ============================================================================================//
    // =========================== Getting the Request ID from the Request
    // ========================//
    // ============================================================================================//

    // Call the RequestSpotInstance API.
    RequestSpotInstancesResult requestResult = ec2.requestSpotInstances(requestRequest);
    List<SpotInstanceRequest> requestResponses = requestResult.getSpotInstanceRequests();

    // Setup an arraylist to collect all of the request ids we want to watch hit the running
    // state.
    ArrayList<String> spotInstanceRequestIds = new ArrayList<String>();

    // Add all of the request ids to the hashset, so we can determine when they hit the
    // active state.
    for (SpotInstanceRequest requestResponse : requestResponses) {
      System.out.println("Created Spot Request: " + requestResponse.getSpotInstanceRequestId());
      spotInstanceRequestIds.add(requestResponse.getSpotInstanceRequestId());
    }

    // ============================================================================================//
    // =========================== Determining the State of the Spot Request
    // ======================//
    // ============================================================================================//

    // Create a variable that will track whether there are any requests still in the open state.
    boolean anyOpen;

    // Initialize variables.
    ArrayList<String> instanceIds = new ArrayList<String>();

    do {
      // Create the describeRequest with tall of the request id to monitor (e.g. that we started).
      DescribeSpotInstanceRequestsRequest describeRequest =
          new DescribeSpotInstanceRequestsRequest();
      describeRequest.setSpotInstanceRequestIds(spotInstanceRequestIds);

      // Initialize the anyOpen variable to false ??? which assumes there are no requests open
      // unless
      // we find one that is still open.
      anyOpen = false;

      try {
        // Retrieve all of the requests we want to monitor.
        DescribeSpotInstanceRequestsResult describeResult =
            ec2.describeSpotInstanceRequests(describeRequest);
        List<SpotInstanceRequest> describeResponses = describeResult.getSpotInstanceRequests();

        // Look through each request and determine if they are all in the active state.
        for (SpotInstanceRequest describeResponse : describeResponses) {
          // If the state is open, it hasn't changed since we attempted to request it.
          // There is the potential for it to transition almost immediately to closed or
          // cancelled so we compare against open instead of active.
          if (describeResponse.getState().equals("open")) {
            anyOpen = true;
            break;
          }

          // Add the instance id to the list we will eventually terminate.
          instanceIds.add(describeResponse.getInstanceId());
        }
      } catch (AmazonServiceException e) {
        // If we have an exception, ensure we don't break out of the loop.
        // This prevents the scenario where there was blip on the wire.
        anyOpen = true;
      }

      try {
        // Sleep for 60 seconds.
        Thread.sleep(60 * 1000);
      } catch (Exception e) {
        // Do nothing because it woke up early.
      }
    } while (anyOpen);

    // ============================================================================================//
    // ====================================== Canceling the Request ==============================//
    // ============================================================================================//

    try {
      // Cancel requests.
      CancelSpotInstanceRequestsRequest cancelRequest =
          new CancelSpotInstanceRequestsRequest(spotInstanceRequestIds);
      ec2.cancelSpotInstanceRequests(cancelRequest);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error cancelling instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }

    // ============================================================================================//
    // =================================== Terminating any Instances
    // ==============================//
    // ============================================================================================//
    try {
      // Terminate instances.
      TerminateInstancesRequest terminateRequest = new TerminateInstancesRequest(instanceIds);
      ec2.terminateInstances(terminateRequest);
    } catch (AmazonServiceException e) {
      // Write out any exceptions that may have occurred.
      System.out.println("Error terminating instances");
      System.out.println("Caught Exception: " + e.getMessage());
      System.out.println("Reponse Status Code: " + e.getStatusCode());
      System.out.println("Error Code: " + e.getErrorCode());
      System.out.println("Request ID: " + e.getRequestId());
    }
  }
Ejemplo n.º 24
0
  public static void main(String[] args) throws Exception {
    init();

    try {
      String violationsTable = "violations-table";
      String citationsTable = "citations-table";
      String warrantsTable = "warrants-table";
      String contactsTable = "contacts-table";
      String notificationsTable = "notifications-table";

      // VIOLATIONS TABLE
      //
      //

      // Create violationsTable if it does not exist yet
      if (Tables.doesTableExist(dynamoDB, violationsTable)) {
        System.out.println("Table " + violationsTable + " is already ACTIVE");
      } else {
        // Create a table with a primary hash key named 'name', which holds a string
        CreateTableRequest createTableRequest =
            new CreateTableRequest()
                .withTableName(violationsTable)
                .withKeySchema(
                    new KeySchemaElement()
                        .withAttributeName("citation_number")
                        .withKeyType(KeyType.HASH))
                .withAttributeDefinitions(
                    new AttributeDefinition()
                        .withAttributeName("citation_number")
                        .withAttributeType(ScalarAttributeType.N))
                .withProvisionedThroughput(
                    new ProvisionedThroughput()
                        .withReadCapacityUnits(25L)
                        .withWriteCapacityUnits(25L));
        TableDescription createdTableDescription =
            dynamoDB.createTable(createTableRequest).getTableDescription();
        System.out.println("Created Table: " + createdTableDescription);

        // Wait for it to become active
        System.out.println("Waiting for " + violationsTable + " to become ACTIVE...");
        Tables.awaitTableToBecomeActive(dynamoDB, violationsTable);
      }

      // Describe our new table
      DescribeTableRequest describeTableRequest =
          new DescribeTableRequest().withTableName(violationsTable);
      TableDescription tableDescription = dynamoDB.describeTable(describeTableRequest).getTable();
      System.out.println("Table Description: " + tableDescription);

      // CITATIONS TABLE
      //
      //
      // Create citationsTable if it does not exist yet
      if (Tables.doesTableExist(dynamoDB, citationsTable)) {
        System.out.println("Table " + citationsTable + " is already ACTIVE");
      } else {
        // Create a table with a primary hash key named 'name', which holds a string
        /*
                        GlobalSecondaryIndex _firstLast = new GlobalSecondaryIndex()
                            .withIndexName("first_last")
                            .withProvisionedThroughput(new ProvisionedThroughput()
                                .withReadCapacityUnits((long) 10)
                                .withWriteCapacityUnits((long) 1))
                                .withProjection(new Projection().withProjectionType(ProjectionType.ALL));

                        GlobalSecondaryIndex _firstLastDOB = new GlobalSecondaryIndex()
                            .withIndexName("first_last_dob")
                            .withProvisionedThroughput(new ProvisionedThroughput()
                                .withReadCapacityUnits((long) 10)
                                .withWriteCapacityUnits((long) 1))
                                .withProjection(new Projection().withProjectionType(ProjectionType.ALL));
        */
        CreateTableRequest createTableRequest =
            new CreateTableRequest()
                .withTableName(citationsTable)
                .withKeySchema(
                    new KeySchemaElement()
                        .withAttributeName("first_last")
                        .withKeyType(KeyType.HASH))
                .withAttributeDefinitions(
                    new AttributeDefinition()
                        .withAttributeName("first_last")
                        .withAttributeType(ScalarAttributeType.S))
                .withProvisionedThroughput(
                    new ProvisionedThroughput()
                        .withReadCapacityUnits(25L)
                        .withWriteCapacityUnits(25L));
        // .withGlobalSecondaryIndexes(_firstLast, _firstLastDOB);
        TableDescription createdTableDescription =
            dynamoDB.createTable(createTableRequest).getTableDescription();
        System.out.println("Created Table: " + createdTableDescription);

        // Wait for it to become active
        System.out.println("Waiting for " + citationsTable + " to become ACTIVE...");
        Tables.awaitTableToBecomeActive(dynamoDB, citationsTable);
      }

      // Describe our new table
      describeTableRequest = new DescribeTableRequest().withTableName(citationsTable);
      tableDescription = dynamoDB.describeTable(describeTableRequest).getTable();
      System.out.println("Table Description: " + tableDescription);

      // WARRANTS TABLE
      //
      //
      // Create warrantsTable if it does not exist yet
      if (Tables.doesTableExist(dynamoDB, warrantsTable)) {
        System.out.println("Table " + warrantsTable + " is already ACTIVE");
      } else {
        /*
                        GlobalSecondaryIndex _firstLastDOB = new GlobalSecondaryIndex()
                            .withIndexName("first_last_dob")
                            .withProvisionedThroughput(new ProvisionedThroughput()
                                .withReadCapacityUnits((long) 10)
                                .withWriteCapacityUnits((long) 1))
                                .withProjection(new Projection().withProjectionType(ProjectionType.ALL));
        */
        // Create a table with a primary hash key named 'name', which holds a string
        CreateTableRequest createTableRequest =
            new CreateTableRequest()
                .withTableName(warrantsTable)
                .withKeySchema(
                    new KeySchemaElement().withAttributeName("id").withKeyType(KeyType.HASH))
                .withAttributeDefinitions(
                    new AttributeDefinition()
                        .withAttributeName("id")
                        .withAttributeType(ScalarAttributeType.S))
                .withProvisionedThroughput(
                    new ProvisionedThroughput()
                        .withReadCapacityUnits(25L)
                        .withWriteCapacityUnits(25L));
        // .withGlobalSecondaryIndexes(_firstLastDOB);
        TableDescription createdTableDescription =
            dynamoDB.createTable(createTableRequest).getTableDescription();
        System.out.println("Created Table: " + createdTableDescription);

        // Wait for it to become active
        System.out.println("Waiting for " + warrantsTable + " to become ACTIVE...");
        Tables.awaitTableToBecomeActive(dynamoDB, warrantsTable);
      }

      // Describe our new table
      describeTableRequest = new DescribeTableRequest().withTableName(warrantsTable);
      tableDescription = dynamoDB.describeTable(describeTableRequest).getTable();
      System.out.println("Table Description: " + tableDescription);

      // CONTACTS TABLE
      //
      //

      // Create violationsTable if it does not exist yet
      if (Tables.doesTableExist(dynamoDB, contactsTable)) {
        System.out.println("Table " + contactsTable + " is already ACTIVE");
      } else {
        // Create a table with a primary hash key named 'name', which holds a string
        CreateTableRequest createTableRequest =
            new CreateTableRequest()
                .withTableName(contactsTable)
                .withKeySchema(
                    new KeySchemaElement()
                        .withAttributeName("first_last")
                        .withKeyType(KeyType.HASH))
                .withAttributeDefinitions(
                    new AttributeDefinition()
                        .withAttributeName("first_last")
                        .withAttributeType(ScalarAttributeType.S))
                .withProvisionedThroughput(
                    new ProvisionedThroughput()
                        .withReadCapacityUnits(25L)
                        .withWriteCapacityUnits(25L));
        TableDescription createdTableDescription =
            dynamoDB.createTable(createTableRequest).getTableDescription();
        System.out.println("Created Table: " + createdTableDescription);

        // Wait for it to become active
        System.out.println("Waiting for " + contactsTable + " to become ACTIVE...");
        Tables.awaitTableToBecomeActive(dynamoDB, contactsTable);
      }

      // Describe our new table
      describeTableRequest = new DescribeTableRequest().withTableName(violationsTable);
      tableDescription = dynamoDB.describeTable(describeTableRequest).getTable();
      System.out.println("Table Description: " + tableDescription);

      ssnHashMap = new HashMap<>();
      saltHashMap = new HashMap<>();
      hashedssnHashMap = new HashMap<>();

      ssnList = new ArrayList<>();

      for (int i = 1010001; i < 1200001; i++) {
        SocialSecurityNumber ssn = new SocialSecurityNumber(i);
        String ssnString = ssn.toString();
        ssnList.add(ssnString);
      }
      /*
      for (String str : ssnList) {
          System.out.println("SSN: " + str);
      }
      */

      // oneTimeAddViolations();
      // oneTimeAddCitations();
      oneTimeAddWarrants();
      // oneTimeAddContacts();

    } catch (AmazonServiceException ase) {
      System.out.println(
          "Caught an AmazonServiceException, which means your request made it "
              + "to AWS, but was rejected with an error response for some reason.");
      System.out.println("Error Message:    " + ase.getMessage());
      System.out.println("HTTP Status Code: " + ase.getStatusCode());
      System.out.println("AWS Error Code:   " + ase.getErrorCode());
      System.out.println("Error Type:       " + ase.getErrorType());
      System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println(
          "Caught an AmazonClientException, which means the client encountered "
              + "a serious internal problem while trying to communicate with AWS, "
              + "such as not being able to access the network.");
      System.out.println("Error Message: " + ace.getMessage());
    }
  }
Ejemplo n.º 25
0
  private void jButton1ActionPerformed(
      java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jButton1ActionPerformed
    // TODO add your handling code here: upload file
    JFileChooser jfc = new JFileChooser();
    jfc.setAcceptAllFileFilterUsed(false);
    jfc.setFileFilter(new FileNameExtensionFilter("Text Files", "txt"));
    int returnVal = jfc.showOpenDialog(this);
    if (returnVal == JFileChooser.APPROVE_OPTION) {
      File file = jfc.getSelectedFile();
      try {
        filepath = file.getCanonicalPath();
        FileReader fr = new FileReader(filepath);

        BufferedReader br = new BufferedReader(fr);
        String line;
        int total_no_of_words = 0;

        while ((line = br.readLine()) != null) {

          String[] words_arr = line.split(" ");
          total_no_of_words += words_arr.length;
        }
        jLabel4.setText("word count :" + total_no_of_words);
        System.out.println("Total no of words = " + total_no_of_words);
        if (total_no_of_words != 0) {
          File userFile = new File(filepath);
          keyName = userFile.getName();
          String myAccessKeyID = "AKIAI7PATIX6LHJA3A3Q";
          String mySecretKey = "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";
          AWSCredentials myCredentials = new BasicAWSCredentials(myAccessKeyID, mySecretKey);
          ClientConfiguration clientConfig = new ClientConfiguration();
          clientConfig.setProtocol(Protocol.HTTP);
          AmazonS3 conn = new AmazonS3Client(myCredentials, clientConfig);
          try {

            File file1 = new File(filepath);
            file_size = file1.length();
            if (file_size < 100000) {
              conn.putObject(new PutObjectRequest(bucketName, keyName, file1));
              jLabel2.setText("Uploading complete !");
            } else {
              jLabel2.setText("Uploading failed size greater than 100kb...");
            }
          } catch (AmazonServiceException ase) {
            System.out.println(
                "Caught an AmazonServiceException, which "
                    + "means your request made it "
                    + "to Amazon S3, but was rejected with an error response"
                    + " for some reason.");
            System.out.println("Error Message:    " + ase.getMessage());
            System.out.println("HTTP Status Code: " + ase.getStatusCode());
            System.out.println("AWS Error Code:   " + ase.getErrorCode());
            System.out.println("Error Type:       " + ase.getErrorType());
            System.out.println("Request ID:       " + ase.getRequestId());
          } catch (AmazonClientException ace) {
            System.out.println(
                "Caught an AmazonClientException, which "
                    + "means the client encountered "
                    + "an internal error while trying to "
                    + "communicate with S3, "
                    + "such as not being able to access the network.");
            System.out.println("Error Message: " + ace.getMessage());
          } /*catch (InterruptedException ex) {
                Logger.getLogger(Upload.class.getName()).log(Level.SEVERE, null, ex);
            }*/
        } else {
          jLabel2.setText("Uploading failed file empty !");
        }

      } catch (IOException ex) {
        Logger.getLogger(Upload.class.getName()).log(Level.SEVERE, null, ex);
      }
    }
    if (returnVal == JFileChooser.CANCEL_OPTION) {
      System.exit(0);
    }
  } // GEN-LAST:event_jButton1ActionPerformed
Ejemplo n.º 26
0
  public static double getCPUUsage(AmazonCloudWatchClient cloudWatch, String instanceId) {

    try {
      // create request message
      GetMetricStatisticsRequest statRequest = new GetMetricStatisticsRequest();
      // set up request message
      statRequest.setNamespace("AWS/EC2"); // namespace
      statRequest.setPeriod(60); // period of data
      ArrayList<String> stats = new ArrayList<String>();
      // Use one of these strings: Average, Maximum, Minimum, SampleCount, Sum
      stats.add("Average");
      stats.add("Sum");
      stats.add("Maximum");
      statRequest.setStatistics(stats);
      // Use one of these strings: CPUUtilization, NetworkIn, NetworkOut, DiskReadBytes,
      // DiskWriteBytes, DiskReadOperations
      statRequest.setMetricName("CPUUtilization");
      // set time
      GregorianCalendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
      calendar.add(
          GregorianCalendar.SECOND, -1 * calendar.get(GregorianCalendar.SECOND)); // 1 second ago
      Date endTime = calendar.getTime();
      calendar.add(GregorianCalendar.MINUTE, -1); // 1 minutes ago
      Date startTime = calendar.getTime();
      statRequest.setStartTime(startTime);
      statRequest.setEndTime(endTime);

      // specify an instance
      ArrayList<Dimension> dimensions = new ArrayList<Dimension>();
      dimensions.add(new Dimension().withName("InstanceId").withValue(instanceId));
      statRequest.setDimensions(dimensions);

      // get statistics
      GetMetricStatisticsResult statResult = cloudWatch.getMetricStatistics(statRequest);

      // display
      // System.out.println(statResult.toString());
      List<Datapoint> dataList = statResult.getDatapoints();
      Double averageCPU = null;
      Date timeStamp = null;
      for (Datapoint data : dataList) {
        averageCPU = data.getAverage();
        timeStamp = data.getTimestamp();
        // System.out.println("Average CPU utlilization for last 1 minute since "
        // +timeStamp.toString() + ": " +averageCPU);
        // System.out.println("Max CPU utlilization for last 1 minute since "+timeStamp.toString() +
        // ": " + data.getMaximum());
        // System.out.println("Total CPU utlilization for last 1 minutes since
        // "+timeStamp.toString() + ": " + data.getSum());
      }
      if (averageCPU == null) {
        System.out.println("Average CPU utlilization for last 1 minute: 0");
        return 0;
      } else {
        System.out.println(
            "Average CPU utlilization for last 1 minute since "
                + timeStamp.toString()
                + ": "
                + averageCPU);
        return averageCPU;
      }
    } catch (AmazonServiceException ase) {
      System.out.println("Caught Exception: " + ase.getMessage());
      System.out.println("Reponse Status Code: " + ase.getStatusCode());
      System.out.println("Error Code: " + ase.getErrorCode());
      System.out.println("Request ID: " + ase.getRequestId());
      return 0;
    }
  }