private static void createResources() { ClasspathPropertiesFileCredentialsProvider provider = new ClasspathPropertiesFileCredentialsProvider(); AmazonSQS sqs = new AmazonSQSClient(provider); sqs.setEndpoint(SQS_ENDPOINT); sqs.createQueue(new CreateQueueRequest().withQueueName(SQS_QUEUE)); AmazonS3 s3 = new AmazonS3Client(provider); if (!s3.doesBucketExist(BUCKET_NAME)) { s3.createBucket(new CreateBucketRequest(BUCKET_NAME)); } AmazonDynamoDBClient dynamo = new AmazonDynamoDBClient(provider); dynamo.setEndpoint(DYNAMO_ENDPOINT); if (!doesTableExist(dynamo, DYNAMO_TABLE_NAME)) { dynamo.createTable( new CreateTableRequest() .withTableName(DYNAMO_TABLE_NAME) .withProvisionedThroughput( new ProvisionedThroughput() .withReadCapacityUnits(50l) .withWriteCapacityUnits(50l)) .withKeySchema( new KeySchema() .withHashKeyElement( new KeySchemaElement() .withAttributeName("id") .withAttributeType(ScalarAttributeType.S)))); waitForTableToBecomeAvailable(dynamo, DYNAMO_TABLE_NAME); } }
@Override public int read( String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) { logger.debug("readkey: " + key + " from table: " + table); GetItemRequest req = new GetItemRequest(table, createPrimaryKey(key)); req.setAttributesToGet(fields); req.setConsistentRead(consistentRead); GetItemResult res = null; try { res = dynamoDB.getItem(req); } catch (AmazonServiceException ex) { logger.error(ex.getMessage()); return SERVER_ERROR; } catch (AmazonClientException ex) { logger.error(ex.getMessage()); return CLIENT_ERROR; } if (null != res.getItem()) { result.putAll(extractResult(res.getItem())); logger.debug("Result: " + res.toString()); } return OK; }
public ResourceHelper() { InputStream credentialsStream = getClass().getClassLoader().getResourceAsStream(AWS_CREDENTIALS_FILE); if (credentialsStream != null) { try { credentials = new PropertiesCredentials(credentialsStream); } catch (IOException e) { logger.error(AWS_CRED_ERROR_READING); } s3cli = new AmazonS3Client(); ddb = new AmazonDynamoDBClient(credentials); ddb.setEndpoint("https://dynamodb.us-west-1.amazonaws.com", "dynamodb", "us-west-1"); mapper = new DynamoDBMapper(ddb); swfClient = new AmazonSimpleWorkflowClient(credentials); swfClient.setEndpoint("http://swf.us-west-1.amazonaws.com"); transClient = new AmazonElasticTranscoderClient(credentials); transClient.setEndpoint("elastictranscoder.us-west-1.amazonaws.com"); } else { logger.error(AWS_CRED_ERROR_NOT_FOUND); } }
/* * (non-Javadoc) * * @see com.dtcc.awsticker.dao.IDaoTicker#getAllTickerRows() */ public List<TickerRow> getAllTickerRows() { // return LocalTickerTable.getAll(); List<TickerRow> rows = new ArrayList<TickerRow>(); ScanRequest scanRequest = new ScanRequest(tableName); ScanResult scanResult = db.scan(scanRequest); List<Map<String, AttributeValue>> items = scanResult.getItems(); if (items == null || items.isEmpty()) return rows; for (Map<String, AttributeValue> item : items) { Integer index = new Integer(0); try { index = Integer.decode(item.get("Index").getN()); } catch (NumberFormatException e) { log.warning("bad key " + e); } TickerRow tr = new TickerRow(); tr.setIndex(index); ItemToModel(item, tr); rows.add(tr); } return rows; }
public void deleteTickerRow(Integer index) { AttributeValue val = new AttributeValue().withN(index.toString()); Key primaryKey = new Key().withHashKeyElement(val); DeleteItemRequest deleteItemRequest = new DeleteItemRequest().withTableName(tableName).withKey(primaryKey); db.deleteItem(deleteItemRequest); }
private void run() throws IOException { InputStream resourceAsStream = getClass().getResourceAsStream("/AwsCredentials.properties"); assert resourceAsStream != null; AWSCredentials credentials = new PropertiesCredentials(resourceAsStream); AmazonDynamoDBClient amazonDynamoDBClient = new AmazonDynamoDBClient(credentials); amazonDynamoDBClient.setEndpoint("https://dynamodb.us-west-1.amazonaws.com"); String tableName = "wellsfargo-checking-01"; // String tableName = "table3"; // describeTable(amazonDynamoDBClient, tableName); List<String> strings = FileUtils.readLines(new File("data/subset.txt")); int i = 0; for (String record : strings) { // 1 1/2/00 Opening Balance - 3504.65 X - [WF Checking] String[] ledgerEntry = record.split("\\t"); Map<String, AttributeValue> item = new HashMap<String, AttributeValue>(); AttributeValue txidAttribute = new AttributeValue(); txidAttribute.setN(ledgerEntry[0]); item.put("txid", txidAttribute); item.put("date", new AttributeValue(ledgerEntry[1])); item.put("payee", new AttributeValue(ledgerEntry[2])); item.put("memo", new AttributeValue(ledgerEntry[3])); item.put("amount", new AttributeValue(ledgerEntry[4])); item.put("cleared", new AttributeValue(ledgerEntry[5])); item.put("checknumber", new AttributeValue(ledgerEntry[6])); item.put("category", new AttributeValue(ledgerEntry[7])); PutItemRequest putItemRequest = new PutItemRequest(tableName, item); PutItemResult putItemResult = amazonDynamoDBClient.putItem(putItemRequest); if (++i % 10 == 0) { System.out.println("putItemResult: " + putItemResult); } } }
/* * (non-Javadoc) * * @see com.dtcc.awsticker.dao.IDaoTicker#getTickerRow(java.lang.Integer) */ public TickerRow getTickerRow(Integer index) { // return LocalTickerTable.get(index); AttributeValue id = new AttributeValue().withN(index + ""); Key primaryKey = new Key().withHashKeyElement(id); GetItemRequest request = new GetItemRequest().withTableName(tableName).withKey(primaryKey); GetItemResult result = db.getItem(request); if (result.getItem() == null) return null; TickerRow tr = new TickerRow(); tr.setIndex(index); ItemToModel(result.getItem(), tr); return tr; }
/* * (non-Javadoc) * * @see com.dtcc.awsticker.dao.IDaoTicker#putTickerRow(java.lang.Integer, * com.dtcc.awsticker.domain.TickerRow) */ public void putTickerRow(Integer index, TickerRow tickerRow) { // LocalTickerTable.put(index, tickerRow); Map<String, AttributeValue> item = new HashMap<String, AttributeValue>(); item.put("Index", new AttributeValue().withN("" + tickerRow.getIndex())); // item.put("Date", new AttributeValue(tickerRow.getDate())); item.put("Name", new AttributeValue(tickerRow.getName())); item.put("Number", new AttributeValue(tickerRow.getNumber())); item.put("Price", new AttributeValue(tickerRow.getPrice())); item.put("Time", new AttributeValue(tickerRow.getDateTime())); item.put("CreatedTimeMs", new AttributeValue("" + System.currentTimeMillis())); PutItemRequest putItemRequest = new PutItemRequest(tableName, item); PutItemResult putItemResult = db.putItem(putItemRequest); }
@Override public int delete(String table, String key) { logger.debug("deletekey: " + key + " from table: " + table); DeleteItemRequest req = new DeleteItemRequest(table, createPrimaryKey(key)); DeleteItemResult res = null; try { res = dynamoDB.deleteItem(req); } catch (AmazonServiceException ex) { logger.error(ex.getMessage()); return SERVER_ERROR; } catch (AmazonClientException ex) { logger.error(ex.getMessage()); return CLIENT_ERROR; } return OK; }
/** * Initialize any state for this DB. Called once per DB instance; there is one DB instance per * client thread. */ public void init() throws DBException { // initialize DynamoDb driver & table. String debug = getProperties().getProperty("dynamodb.debug", null); if (null != debug && "true".equalsIgnoreCase(debug)) { logger.setLevel(Level.DEBUG); } String endpoint = getProperties().getProperty("dynamodb.endpoint", null); String credentialsFile = getProperties().getProperty("dynamodb.awsCredentialsFile", null); String primaryKey = getProperties().getProperty("dynamodb.primaryKey", null); String consistentReads = getProperties().getProperty("dynamodb.consistentReads", null); String connectMax = getProperties().getProperty("dynamodb.connectMax", null); if (null != connectMax) { this.maxConnects = Integer.parseInt(connectMax); } if (null != consistentReads && "true".equalsIgnoreCase(consistentReads)) { this.consistentRead = true; } if (null != endpoint) { this.endpoint = endpoint; } if (null == primaryKey || primaryKey.length() < 1) { String errMsg = "Missing primary key attribute name, cannot continue"; logger.error(errMsg); } try { AWSCredentials credentials = new PropertiesCredentials(new File(credentialsFile)); ClientConfiguration cconfig = new ClientConfiguration(); cconfig.setMaxConnections(maxConnects); dynamoDB = new AmazonDynamoDBClient(credentials, cconfig); dynamoDB.setEndpoint(this.endpoint); primaryKeyName = primaryKey; logger.info("dynamodb connection created with " + this.endpoint); } catch (Exception e1) { String errMsg = "DynamoDBClient.init(): Could not initialize DynamoDB client: " + e1.getMessage(); logger.error(errMsg); } }
@Override public int insert(String table, String key, HashMap<String, ByteIterator> values) { logger.debug("insertkey: " + primaryKeyName + "-" + key + " from table: " + table); Map<String, AttributeValue> attributes = createAttributes(values); // adding primary key attributes.put(primaryKeyName, new AttributeValue(key)); PutItemRequest putItemRequest = new PutItemRequest(table, attributes); PutItemResult res = null; try { res = dynamoDB.putItem(putItemRequest); } catch (AmazonServiceException ex) { logger.error(ex.getMessage()); return SERVER_ERROR; } catch (AmazonClientException ex) { logger.error(ex.getMessage()); return CLIENT_ERROR; } return OK; }
@Override public int update(String table, String key, HashMap<String, ByteIterator> values) { logger.debug("updatekey: " + key + " from table: " + table); Map<String, AttributeValueUpdate> attributes = new HashMap<String, AttributeValueUpdate>(values.size()); for (Entry<String, ByteIterator> val : values.entrySet()) { AttributeValue v = new AttributeValue(val.getValue().toString()); attributes.put(val.getKey(), new AttributeValueUpdate().withValue(v).withAction("PUT")); } UpdateItemRequest req = new UpdateItemRequest(table, createPrimaryKey(key), attributes); try { dynamoDB.updateItem(req); } catch (AmazonServiceException ex) { logger.error(ex.getMessage()); return SERVER_ERROR; } catch (AmazonClientException ex) { logger.error(ex.getMessage()); return CLIENT_ERROR; } return OK; }
@Override public int scan( String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String, ByteIterator>> result) { logger.debug("scan " + recordcount + " records from key: " + startkey + " on table: " + table); /* * on DynamoDB's scan, startkey is *exclusive* so we need to * getItem(startKey) and then use scan for the res */ GetItemRequest greq = new GetItemRequest(table, createPrimaryKey(startkey)); greq.setAttributesToGet(fields); GetItemResult gres = null; try { gres = dynamoDB.getItem(greq); } catch (AmazonServiceException ex) { logger.error(ex.getMessage()); return SERVER_ERROR; } catch (AmazonClientException ex) { logger.error(ex.getMessage()); return CLIENT_ERROR; } if (null != gres.getItem()) { result.add(extractResult(gres.getItem())); } int count = 1; // startKey is done, rest to go. Key startKey = createPrimaryKey(startkey); ScanRequest req = new ScanRequest(table); req.setAttributesToGet(fields); while (count < recordcount) { req.setExclusiveStartKey(startKey); req.setLimit(recordcount - count); ScanResult res = null; try { res = dynamoDB.scan(req); } catch (AmazonServiceException ex) { logger.error(ex.getMessage()); ex.printStackTrace(); return SERVER_ERROR; } catch (AmazonClientException ex) { logger.error(ex.getMessage()); ex.printStackTrace(); return CLIENT_ERROR; } count += res.getCount(); for (Map<String, AttributeValue> items : res.getItems()) { result.add(extractResult(items)); } startKey = res.getLastEvaluatedKey(); } return OK; }
private void describeTable(AmazonDynamoDBClient dynamoDB, String tableName) { // Describe our new table DescribeTableRequest describeTableRequest = new DescribeTableRequest().withTableName(tableName); TableDescription tableDescription = dynamoDB.describeTable(describeTableRequest).getTable(); System.out.println("Table Description: " + tableDescription); }