private static void createResources() {
    ClasspathPropertiesFileCredentialsProvider provider =
        new ClasspathPropertiesFileCredentialsProvider();

    AmazonSQS sqs = new AmazonSQSClient(provider);
    sqs.setEndpoint(SQS_ENDPOINT);
    sqs.createQueue(new CreateQueueRequest().withQueueName(SQS_QUEUE));

    AmazonS3 s3 = new AmazonS3Client(provider);
    if (!s3.doesBucketExist(BUCKET_NAME)) {
      s3.createBucket(new CreateBucketRequest(BUCKET_NAME));
    }

    AmazonDynamoDBClient dynamo = new AmazonDynamoDBClient(provider);
    dynamo.setEndpoint(DYNAMO_ENDPOINT);

    if (!doesTableExist(dynamo, DYNAMO_TABLE_NAME)) {
      dynamo.createTable(
          new CreateTableRequest()
              .withTableName(DYNAMO_TABLE_NAME)
              .withProvisionedThroughput(
                  new ProvisionedThroughput()
                      .withReadCapacityUnits(50l)
                      .withWriteCapacityUnits(50l))
              .withKeySchema(
                  new KeySchema()
                      .withHashKeyElement(
                          new KeySchemaElement()
                              .withAttributeName("id")
                              .withAttributeType(ScalarAttributeType.S))));
      waitForTableToBecomeAvailable(dynamo, DYNAMO_TABLE_NAME);
    }
  }
Example #2
0
  @Override
  public int read(
      String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) {

    logger.debug("readkey: " + key + " from table: " + table);
    GetItemRequest req = new GetItemRequest(table, createPrimaryKey(key));
    req.setAttributesToGet(fields);
    req.setConsistentRead(consistentRead);
    GetItemResult res = null;

    try {
      res = dynamoDB.getItem(req);
    } catch (AmazonServiceException ex) {
      logger.error(ex.getMessage());
      return SERVER_ERROR;
    } catch (AmazonClientException ex) {
      logger.error(ex.getMessage());
      return CLIENT_ERROR;
    }

    if (null != res.getItem()) {
      result.putAll(extractResult(res.getItem()));
      logger.debug("Result: " + res.toString());
    }
    return OK;
  }
Example #3
0
  public ResourceHelper() {
    InputStream credentialsStream =
        getClass().getClassLoader().getResourceAsStream(AWS_CREDENTIALS_FILE);
    if (credentialsStream != null) {
      try {
        credentials = new PropertiesCredentials(credentialsStream);
      } catch (IOException e) {
        logger.error(AWS_CRED_ERROR_READING);
      }

      s3cli = new AmazonS3Client();
      ddb = new AmazonDynamoDBClient(credentials);
      ddb.setEndpoint("https://dynamodb.us-west-1.amazonaws.com", "dynamodb", "us-west-1");
      mapper = new DynamoDBMapper(ddb);

      swfClient = new AmazonSimpleWorkflowClient(credentials);
      swfClient.setEndpoint("http://swf.us-west-1.amazonaws.com");

      transClient = new AmazonElasticTranscoderClient(credentials);

      transClient.setEndpoint("elastictranscoder.us-west-1.amazonaws.com");

    } else {
      logger.error(AWS_CRED_ERROR_NOT_FOUND);
    }
  }
Example #4
0
  /*
   * (non-Javadoc)
   *
   * @see com.dtcc.awsticker.dao.IDaoTicker#getAllTickerRows()
   */
  public List<TickerRow> getAllTickerRows() {
    // return LocalTickerTable.getAll();
    List<TickerRow> rows = new ArrayList<TickerRow>();

    ScanRequest scanRequest = new ScanRequest(tableName);
    ScanResult scanResult = db.scan(scanRequest);
    List<Map<String, AttributeValue>> items = scanResult.getItems();
    if (items == null || items.isEmpty()) return rows;
    for (Map<String, AttributeValue> item : items) {

      Integer index = new Integer(0);
      try {
        index = Integer.decode(item.get("Index").getN());
      } catch (NumberFormatException e) {
        log.warning("bad key " + e);
      }

      TickerRow tr = new TickerRow();
      tr.setIndex(index);
      ItemToModel(item, tr);
      rows.add(tr);
    }

    return rows;
  }
Example #5
0
  public void deleteTickerRow(Integer index) {

    AttributeValue val = new AttributeValue().withN(index.toString());

    Key primaryKey = new Key().withHashKeyElement(val);

    DeleteItemRequest deleteItemRequest =
        new DeleteItemRequest().withTableName(tableName).withKey(primaryKey);

    db.deleteItem(deleteItemRequest);
  }
  private void run() throws IOException {
    InputStream resourceAsStream = getClass().getResourceAsStream("/AwsCredentials.properties");
    assert resourceAsStream != null;

    AWSCredentials credentials = new PropertiesCredentials(resourceAsStream);
    AmazonDynamoDBClient amazonDynamoDBClient = new AmazonDynamoDBClient(credentials);
    amazonDynamoDBClient.setEndpoint("https://dynamodb.us-west-1.amazonaws.com");

    String tableName = "wellsfargo-checking-01";
    //        String tableName = "table3";

    //        describeTable(amazonDynamoDBClient, tableName);

    List<String> strings = FileUtils.readLines(new File("data/subset.txt"));
    int i = 0;
    for (String record : strings) {
      //	1	1/2/00	Opening Balance	-	3504.65	X	-	[WF Checking]
      String[] ledgerEntry = record.split("\\t");
      Map<String, AttributeValue> item = new HashMap<String, AttributeValue>();
      AttributeValue txidAttribute = new AttributeValue();
      txidAttribute.setN(ledgerEntry[0]);

      item.put("txid", txidAttribute);
      item.put("date", new AttributeValue(ledgerEntry[1]));
      item.put("payee", new AttributeValue(ledgerEntry[2]));
      item.put("memo", new AttributeValue(ledgerEntry[3]));
      item.put("amount", new AttributeValue(ledgerEntry[4]));
      item.put("cleared", new AttributeValue(ledgerEntry[5]));
      item.put("checknumber", new AttributeValue(ledgerEntry[6]));
      item.put("category", new AttributeValue(ledgerEntry[7]));

      PutItemRequest putItemRequest = new PutItemRequest(tableName, item);
      PutItemResult putItemResult = amazonDynamoDBClient.putItem(putItemRequest);
      if (++i % 10 == 0) {
        System.out.println("putItemResult: " + putItemResult);
      }
    }
  }
Example #7
0
  /*
   * (non-Javadoc)
   *
   * @see com.dtcc.awsticker.dao.IDaoTicker#getTickerRow(java.lang.Integer)
   */
  public TickerRow getTickerRow(Integer index) {
    // return LocalTickerTable.get(index);

    AttributeValue id = new AttributeValue().withN(index + "");
    Key primaryKey = new Key().withHashKeyElement(id);
    GetItemRequest request = new GetItemRequest().withTableName(tableName).withKey(primaryKey);

    GetItemResult result = db.getItem(request);
    if (result.getItem() == null) return null;
    TickerRow tr = new TickerRow();
    tr.setIndex(index);
    ItemToModel(result.getItem(), tr);

    return tr;
  }
Example #8
0
  /*
   * (non-Javadoc)
   *
   * @see com.dtcc.awsticker.dao.IDaoTicker#putTickerRow(java.lang.Integer,
   * com.dtcc.awsticker.domain.TickerRow)
   */
  public void putTickerRow(Integer index, TickerRow tickerRow) {
    // LocalTickerTable.put(index, tickerRow);

    Map<String, AttributeValue> item = new HashMap<String, AttributeValue>();

    item.put("Index", new AttributeValue().withN("" + tickerRow.getIndex()));
    // item.put("Date", new AttributeValue(tickerRow.getDate()));
    item.put("Name", new AttributeValue(tickerRow.getName()));
    item.put("Number", new AttributeValue(tickerRow.getNumber()));
    item.put("Price", new AttributeValue(tickerRow.getPrice()));
    item.put("Time", new AttributeValue(tickerRow.getDateTime()));
    item.put("CreatedTimeMs", new AttributeValue("" + System.currentTimeMillis()));

    PutItemRequest putItemRequest = new PutItemRequest(tableName, item);
    PutItemResult putItemResult = db.putItem(putItemRequest);
  }
Example #9
0
  @Override
  public int delete(String table, String key) {
    logger.debug("deletekey: " + key + " from table: " + table);
    DeleteItemRequest req = new DeleteItemRequest(table, createPrimaryKey(key));
    DeleteItemResult res = null;

    try {
      res = dynamoDB.deleteItem(req);
    } catch (AmazonServiceException ex) {
      logger.error(ex.getMessage());
      return SERVER_ERROR;
    } catch (AmazonClientException ex) {
      logger.error(ex.getMessage());
      return CLIENT_ERROR;
    }
    return OK;
  }
Example #10
0
  /**
   * Initialize any state for this DB. Called once per DB instance; there is one DB instance per
   * client thread.
   */
  public void init() throws DBException {
    // initialize DynamoDb driver & table.
    String debug = getProperties().getProperty("dynamodb.debug", null);

    if (null != debug && "true".equalsIgnoreCase(debug)) {
      logger.setLevel(Level.DEBUG);
    }

    String endpoint = getProperties().getProperty("dynamodb.endpoint", null);
    String credentialsFile = getProperties().getProperty("dynamodb.awsCredentialsFile", null);
    String primaryKey = getProperties().getProperty("dynamodb.primaryKey", null);
    String consistentReads = getProperties().getProperty("dynamodb.consistentReads", null);
    String connectMax = getProperties().getProperty("dynamodb.connectMax", null);

    if (null != connectMax) {
      this.maxConnects = Integer.parseInt(connectMax);
    }

    if (null != consistentReads && "true".equalsIgnoreCase(consistentReads)) {
      this.consistentRead = true;
    }

    if (null != endpoint) {
      this.endpoint = endpoint;
    }

    if (null == primaryKey || primaryKey.length() < 1) {
      String errMsg = "Missing primary key attribute name, cannot continue";
      logger.error(errMsg);
    }

    try {
      AWSCredentials credentials = new PropertiesCredentials(new File(credentialsFile));
      ClientConfiguration cconfig = new ClientConfiguration();
      cconfig.setMaxConnections(maxConnects);
      dynamoDB = new AmazonDynamoDBClient(credentials, cconfig);
      dynamoDB.setEndpoint(this.endpoint);
      primaryKeyName = primaryKey;
      logger.info("dynamodb connection created with " + this.endpoint);
    } catch (Exception e1) {
      String errMsg =
          "DynamoDBClient.init(): Could not initialize DynamoDB client: " + e1.getMessage();
      logger.error(errMsg);
    }
  }
Example #11
0
  @Override
  public int insert(String table, String key, HashMap<String, ByteIterator> values) {
    logger.debug("insertkey: " + primaryKeyName + "-" + key + " from table: " + table);
    Map<String, AttributeValue> attributes = createAttributes(values);
    // adding primary key
    attributes.put(primaryKeyName, new AttributeValue(key));

    PutItemRequest putItemRequest = new PutItemRequest(table, attributes);
    PutItemResult res = null;
    try {
      res = dynamoDB.putItem(putItemRequest);
    } catch (AmazonServiceException ex) {
      logger.error(ex.getMessage());
      return SERVER_ERROR;
    } catch (AmazonClientException ex) {
      logger.error(ex.getMessage());
      return CLIENT_ERROR;
    }
    return OK;
  }
Example #12
0
  @Override
  public int update(String table, String key, HashMap<String, ByteIterator> values) {
    logger.debug("updatekey: " + key + " from table: " + table);

    Map<String, AttributeValueUpdate> attributes =
        new HashMap<String, AttributeValueUpdate>(values.size());
    for (Entry<String, ByteIterator> val : values.entrySet()) {
      AttributeValue v = new AttributeValue(val.getValue().toString());
      attributes.put(val.getKey(), new AttributeValueUpdate().withValue(v).withAction("PUT"));
    }

    UpdateItemRequest req = new UpdateItemRequest(table, createPrimaryKey(key), attributes);

    try {
      dynamoDB.updateItem(req);
    } catch (AmazonServiceException ex) {
      logger.error(ex.getMessage());
      return SERVER_ERROR;
    } catch (AmazonClientException ex) {
      logger.error(ex.getMessage());
      return CLIENT_ERROR;
    }
    return OK;
  }
Example #13
0
  @Override
  public int scan(
      String table,
      String startkey,
      int recordcount,
      Set<String> fields,
      Vector<HashMap<String, ByteIterator>> result) {
    logger.debug("scan " + recordcount + " records from key: " + startkey + " on table: " + table);
    /*
     * on DynamoDB's scan, startkey is *exclusive* so we need to
     * getItem(startKey) and then use scan for the res
     */
    GetItemRequest greq = new GetItemRequest(table, createPrimaryKey(startkey));
    greq.setAttributesToGet(fields);

    GetItemResult gres = null;

    try {
      gres = dynamoDB.getItem(greq);
    } catch (AmazonServiceException ex) {
      logger.error(ex.getMessage());
      return SERVER_ERROR;
    } catch (AmazonClientException ex) {
      logger.error(ex.getMessage());
      return CLIENT_ERROR;
    }

    if (null != gres.getItem()) {
      result.add(extractResult(gres.getItem()));
    }

    int count = 1; // startKey is done, rest to go.

    Key startKey = createPrimaryKey(startkey);
    ScanRequest req = new ScanRequest(table);
    req.setAttributesToGet(fields);
    while (count < recordcount) {
      req.setExclusiveStartKey(startKey);
      req.setLimit(recordcount - count);
      ScanResult res = null;
      try {
        res = dynamoDB.scan(req);
      } catch (AmazonServiceException ex) {
        logger.error(ex.getMessage());
        ex.printStackTrace();
        return SERVER_ERROR;
      } catch (AmazonClientException ex) {
        logger.error(ex.getMessage());
        ex.printStackTrace();
        return CLIENT_ERROR;
      }

      count += res.getCount();
      for (Map<String, AttributeValue> items : res.getItems()) {
        result.add(extractResult(items));
      }
      startKey = res.getLastEvaluatedKey();
    }

    return OK;
  }
 private void describeTable(AmazonDynamoDBClient dynamoDB, String tableName) {
   // Describe our new table
   DescribeTableRequest describeTableRequest = new DescribeTableRequest().withTableName(tableName);
   TableDescription tableDescription = dynamoDB.describeTable(describeTableRequest).getTable();
   System.out.println("Table Description: " + tableDescription);
 }