private static void createResources() {
ClasspathPropertiesFileCredentialsProvider provider =
new ClasspathPropertiesFileCredentialsProvider();
AmazonSQS sqs = new AmazonSQSClient(provider);
sqs.setEndpoint(SQS_ENDPOINT);
sqs.createQueue(new CreateQueueRequest().withQueueName(SQS_QUEUE));
AmazonS3 s3 = new AmazonS3Client(provider);
if (!s3.doesBucketExist(BUCKET_NAME)) {
s3.createBucket(new CreateBucketRequest(BUCKET_NAME));
}
AmazonDynamoDBClient dynamo = new AmazonDynamoDBClient(provider);
dynamo.setEndpoint(DYNAMO_ENDPOINT);
if (!doesTableExist(dynamo, DYNAMO_TABLE_NAME)) {
dynamo.createTable(
new CreateTableRequest()
.withTableName(DYNAMO_TABLE_NAME)
.withProvisionedThroughput(
new ProvisionedThroughput()
.withReadCapacityUnits(50l)
.withWriteCapacityUnits(50l))
.withKeySchema(
new KeySchema()
.withHashKeyElement(
new KeySchemaElement()
.withAttributeName("id")
.withAttributeType(ScalarAttributeType.S))));
waitForTableToBecomeAvailable(dynamo, DYNAMO_TABLE_NAME);
}
}
@Override
public int read(
String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) {
logger.debug("readkey: " + key + " from table: " + table);
GetItemRequest req = new GetItemRequest(table, createPrimaryKey(key));
req.setAttributesToGet(fields);
req.setConsistentRead(consistentRead);
GetItemResult res = null;
try {
res = dynamoDB.getItem(req);
} catch (AmazonServiceException ex) {
logger.error(ex.getMessage());
return SERVER_ERROR;
} catch (AmazonClientException ex) {
logger.error(ex.getMessage());
return CLIENT_ERROR;
}
if (null != res.getItem()) {
result.putAll(extractResult(res.getItem()));
logger.debug("Result: " + res.toString());
}
return OK;
}
public ResourceHelper() {
InputStream credentialsStream =
getClass().getClassLoader().getResourceAsStream(AWS_CREDENTIALS_FILE);
if (credentialsStream != null) {
try {
credentials = new PropertiesCredentials(credentialsStream);
} catch (IOException e) {
logger.error(AWS_CRED_ERROR_READING);
}
s3cli = new AmazonS3Client();
ddb = new AmazonDynamoDBClient(credentials);
ddb.setEndpoint("https://dynamodb.us-west-1.amazonaws.com", "dynamodb", "us-west-1");
mapper = new DynamoDBMapper(ddb);
swfClient = new AmazonSimpleWorkflowClient(credentials);
swfClient.setEndpoint("http://swf.us-west-1.amazonaws.com");
transClient = new AmazonElasticTranscoderClient(credentials);
transClient.setEndpoint("elastictranscoder.us-west-1.amazonaws.com");
} else {
logger.error(AWS_CRED_ERROR_NOT_FOUND);
}
}
/*
* (non-Javadoc)
*
* @see com.dtcc.awsticker.dao.IDaoTicker#getAllTickerRows()
*/
public List<TickerRow> getAllTickerRows() {
// return LocalTickerTable.getAll();
List<TickerRow> rows = new ArrayList<TickerRow>();
ScanRequest scanRequest = new ScanRequest(tableName);
ScanResult scanResult = db.scan(scanRequest);
List<Map<String, AttributeValue>> items = scanResult.getItems();
if (items == null || items.isEmpty()) return rows;
for (Map<String, AttributeValue> item : items) {
Integer index = new Integer(0);
try {
index = Integer.decode(item.get("Index").getN());
} catch (NumberFormatException e) {
log.warning("bad key " + e);
}
TickerRow tr = new TickerRow();
tr.setIndex(index);
ItemToModel(item, tr);
rows.add(tr);
}
return rows;
}
public void deleteTickerRow(Integer index) {
AttributeValue val = new AttributeValue().withN(index.toString());
Key primaryKey = new Key().withHashKeyElement(val);
DeleteItemRequest deleteItemRequest =
new DeleteItemRequest().withTableName(tableName).withKey(primaryKey);
db.deleteItem(deleteItemRequest);
}
private void run() throws IOException {
InputStream resourceAsStream = getClass().getResourceAsStream("/AwsCredentials.properties");
assert resourceAsStream != null;
AWSCredentials credentials = new PropertiesCredentials(resourceAsStream);
AmazonDynamoDBClient amazonDynamoDBClient = new AmazonDynamoDBClient(credentials);
amazonDynamoDBClient.setEndpoint("https://dynamodb.us-west-1.amazonaws.com");
String tableName = "wellsfargo-checking-01";
// String tableName = "table3";
// describeTable(amazonDynamoDBClient, tableName);
List<String> strings = FileUtils.readLines(new File("data/subset.txt"));
int i = 0;
for (String record : strings) {
// 1 1/2/00 Opening Balance - 3504.65 X - [WF Checking]
String[] ledgerEntry = record.split("\\t");
Map<String, AttributeValue> item = new HashMap<String, AttributeValue>();
AttributeValue txidAttribute = new AttributeValue();
txidAttribute.setN(ledgerEntry[0]);
item.put("txid", txidAttribute);
item.put("date", new AttributeValue(ledgerEntry[1]));
item.put("payee", new AttributeValue(ledgerEntry[2]));
item.put("memo", new AttributeValue(ledgerEntry[3]));
item.put("amount", new AttributeValue(ledgerEntry[4]));
item.put("cleared", new AttributeValue(ledgerEntry[5]));
item.put("checknumber", new AttributeValue(ledgerEntry[6]));
item.put("category", new AttributeValue(ledgerEntry[7]));
PutItemRequest putItemRequest = new PutItemRequest(tableName, item);
PutItemResult putItemResult = amazonDynamoDBClient.putItem(putItemRequest);
if (++i % 10 == 0) {
System.out.println("putItemResult: " + putItemResult);
}
}
}
/*
* (non-Javadoc)
*
* @see com.dtcc.awsticker.dao.IDaoTicker#getTickerRow(java.lang.Integer)
*/
public TickerRow getTickerRow(Integer index) {
// return LocalTickerTable.get(index);
AttributeValue id = new AttributeValue().withN(index + "");
Key primaryKey = new Key().withHashKeyElement(id);
GetItemRequest request = new GetItemRequest().withTableName(tableName).withKey(primaryKey);
GetItemResult result = db.getItem(request);
if (result.getItem() == null) return null;
TickerRow tr = new TickerRow();
tr.setIndex(index);
ItemToModel(result.getItem(), tr);
return tr;
}
/*
* (non-Javadoc)
*
* @see com.dtcc.awsticker.dao.IDaoTicker#putTickerRow(java.lang.Integer,
* com.dtcc.awsticker.domain.TickerRow)
*/
public void putTickerRow(Integer index, TickerRow tickerRow) {
// LocalTickerTable.put(index, tickerRow);
Map<String, AttributeValue> item = new HashMap<String, AttributeValue>();
item.put("Index", new AttributeValue().withN("" + tickerRow.getIndex()));
// item.put("Date", new AttributeValue(tickerRow.getDate()));
item.put("Name", new AttributeValue(tickerRow.getName()));
item.put("Number", new AttributeValue(tickerRow.getNumber()));
item.put("Price", new AttributeValue(tickerRow.getPrice()));
item.put("Time", new AttributeValue(tickerRow.getDateTime()));
item.put("CreatedTimeMs", new AttributeValue("" + System.currentTimeMillis()));
PutItemRequest putItemRequest = new PutItemRequest(tableName, item);
PutItemResult putItemResult = db.putItem(putItemRequest);
}
@Override
public int delete(String table, String key) {
logger.debug("deletekey: " + key + " from table: " + table);
DeleteItemRequest req = new DeleteItemRequest(table, createPrimaryKey(key));
DeleteItemResult res = null;
try {
res = dynamoDB.deleteItem(req);
} catch (AmazonServiceException ex) {
logger.error(ex.getMessage());
return SERVER_ERROR;
} catch (AmazonClientException ex) {
logger.error(ex.getMessage());
return CLIENT_ERROR;
}
return OK;
}
/**
* Initialize any state for this DB. Called once per DB instance; there is one DB instance per
* client thread.
*/
public void init() throws DBException {
// initialize DynamoDb driver & table.
String debug = getProperties().getProperty("dynamodb.debug", null);
if (null != debug && "true".equalsIgnoreCase(debug)) {
logger.setLevel(Level.DEBUG);
}
String endpoint = getProperties().getProperty("dynamodb.endpoint", null);
String credentialsFile = getProperties().getProperty("dynamodb.awsCredentialsFile", null);
String primaryKey = getProperties().getProperty("dynamodb.primaryKey", null);
String consistentReads = getProperties().getProperty("dynamodb.consistentReads", null);
String connectMax = getProperties().getProperty("dynamodb.connectMax", null);
if (null != connectMax) {
this.maxConnects = Integer.parseInt(connectMax);
}
if (null != consistentReads && "true".equalsIgnoreCase(consistentReads)) {
this.consistentRead = true;
}
if (null != endpoint) {
this.endpoint = endpoint;
}
if (null == primaryKey || primaryKey.length() < 1) {
String errMsg = "Missing primary key attribute name, cannot continue";
logger.error(errMsg);
}
try {
AWSCredentials credentials = new PropertiesCredentials(new File(credentialsFile));
ClientConfiguration cconfig = new ClientConfiguration();
cconfig.setMaxConnections(maxConnects);
dynamoDB = new AmazonDynamoDBClient(credentials, cconfig);
dynamoDB.setEndpoint(this.endpoint);
primaryKeyName = primaryKey;
logger.info("dynamodb connection created with " + this.endpoint);
} catch (Exception e1) {
String errMsg =
"DynamoDBClient.init(): Could not initialize DynamoDB client: " + e1.getMessage();
logger.error(errMsg);
}
}
@Override
public int insert(String table, String key, HashMap<String, ByteIterator> values) {
logger.debug("insertkey: " + primaryKeyName + "-" + key + " from table: " + table);
Map<String, AttributeValue> attributes = createAttributes(values);
// adding primary key
attributes.put(primaryKeyName, new AttributeValue(key));
PutItemRequest putItemRequest = new PutItemRequest(table, attributes);
PutItemResult res = null;
try {
res = dynamoDB.putItem(putItemRequest);
} catch (AmazonServiceException ex) {
logger.error(ex.getMessage());
return SERVER_ERROR;
} catch (AmazonClientException ex) {
logger.error(ex.getMessage());
return CLIENT_ERROR;
}
return OK;
}
@Override
public int update(String table, String key, HashMap<String, ByteIterator> values) {
logger.debug("updatekey: " + key + " from table: " + table);
Map<String, AttributeValueUpdate> attributes =
new HashMap<String, AttributeValueUpdate>(values.size());
for (Entry<String, ByteIterator> val : values.entrySet()) {
AttributeValue v = new AttributeValue(val.getValue().toString());
attributes.put(val.getKey(), new AttributeValueUpdate().withValue(v).withAction("PUT"));
}
UpdateItemRequest req = new UpdateItemRequest(table, createPrimaryKey(key), attributes);
try {
dynamoDB.updateItem(req);
} catch (AmazonServiceException ex) {
logger.error(ex.getMessage());
return SERVER_ERROR;
} catch (AmazonClientException ex) {
logger.error(ex.getMessage());
return CLIENT_ERROR;
}
return OK;
}
@Override
public int scan(
String table,
String startkey,
int recordcount,
Set<String> fields,
Vector<HashMap<String, ByteIterator>> result) {
logger.debug("scan " + recordcount + " records from key: " + startkey + " on table: " + table);
/*
* on DynamoDB's scan, startkey is *exclusive* so we need to
* getItem(startKey) and then use scan for the res
*/
GetItemRequest greq = new GetItemRequest(table, createPrimaryKey(startkey));
greq.setAttributesToGet(fields);
GetItemResult gres = null;
try {
gres = dynamoDB.getItem(greq);
} catch (AmazonServiceException ex) {
logger.error(ex.getMessage());
return SERVER_ERROR;
} catch (AmazonClientException ex) {
logger.error(ex.getMessage());
return CLIENT_ERROR;
}
if (null != gres.getItem()) {
result.add(extractResult(gres.getItem()));
}
int count = 1; // startKey is done, rest to go.
Key startKey = createPrimaryKey(startkey);
ScanRequest req = new ScanRequest(table);
req.setAttributesToGet(fields);
while (count < recordcount) {
req.setExclusiveStartKey(startKey);
req.setLimit(recordcount - count);
ScanResult res = null;
try {
res = dynamoDB.scan(req);
} catch (AmazonServiceException ex) {
logger.error(ex.getMessage());
ex.printStackTrace();
return SERVER_ERROR;
} catch (AmazonClientException ex) {
logger.error(ex.getMessage());
ex.printStackTrace();
return CLIENT_ERROR;
}
count += res.getCount();
for (Map<String, AttributeValue> items : res.getItems()) {
result.add(extractResult(items));
}
startKey = res.getLastEvaluatedKey();
}
return OK;
}
private void describeTable(AmazonDynamoDBClient dynamoDB, String tableName) {
// Describe our new table
DescribeTableRequest describeTableRequest = new DescribeTableRequest().withTableName(tableName);
TableDescription tableDescription = dynamoDB.describeTable(describeTableRequest).getTable();
System.out.println("Table Description: " + tableDescription);
}