private static void basicCassandraSession(CassandraConnector connector) {
try (Session session = connector.openSession()) {
session.execute(
"CREATE KEYSPACE IF NOT EXISTS test WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1 }");
session.execute(
"CREATE TABLE IF NOT EXISTS test.key_value (key INT PRIMARY KEY, value VARCHAR)");
session.execute("TRUNCATE test.key_value");
session.execute("INSERT INTO test.key_value(key, value) VALUES (1, 'first row')");
session.execute("INSERT INTO test.key_value(key, value) VALUES (2, 'second row')");
session.execute("INSERT INTO test.key_value(key, value) VALUES (3, 'third row')");
// this will be used in the next test
session.execute(
"CREATE TABLE IF NOT EXISTS test.people (id INT, name TEXT, birth_date TIMESTAMP, PRIMARY KEY (id))");
// secondary indexes are not recommended in production. This is just a sample to demonstrate
// the connector
session.execute("CREATE INDEX IF NOT EXISTS people_name_idx ON test.people(name)");
}
}
public JavaStreamingContext createContext() {
CassandraConnectionContext connectionContext = connectToCassandra();
JavaDStream<String> rdd = connectionContext.getRDD();
JavaStreamingContext sparkContext = connectionContext.getStreamingContext();
final CassandraConnector connector = connectionContext.getCassandraConnector();
final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
JavaDStream<Tuple2<String, Transaction>> transactionList =
rdd.map(
new Function<String, Tuple2<String, Transaction>>() {
@Override
public Tuple2<String, Transaction> call(String line) throws Exception {
String[] columns = line.split(",");
String taxId = columns[0];
String name = columns[1];
String merchant = columns[2];
BigDecimal amount = new BigDecimal(columns[3]);
Date transactionDate;
try {
transactionDate = format.parse(columns[4]);
} catch (ParseException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
String tranId = columns[5];
System.out.println(line);
Tuple2<String, Transaction> transaction =
new Tuple2<>(
taxId, new Transaction(name, merchant, amount, transactionDate, tranId));
return transaction;
}
});
transactionList.cache();
final String warningsTableName = "warnings";
try (Session session = connector.openSession()) {
session.execute(
String.format("DROP TABLE IF EXISTS %s.%s", getKeySpaceName(), warningsTableName));
session.execute(
String.format(
"CREATE TABLE IF NOT EXISTS %s.%s (ssn text, "
+ "id uuid, amount decimal, rule text, PRIMARY KEY(ssn, id))",
getKeySpaceName(), warningsTableName));
}
// setup warning on more than certain number of transactions by user in a 60 second window,
// every 10 seconds
JavaPairDStream<String, BigDecimal> warnings =
transactionList
.window(new Duration(60000), new Duration(10000))
.mapToPair(
new PairFunction<Tuple2<String, Transaction>, String, BigDecimal>() {
@Override
public Tuple2<String, BigDecimal> call(Tuple2<String, Transaction> transaction)
throws Exception {
String taxId = transaction._1();
BigDecimal amount = transaction._2().getAmount();
return new Tuple2<>(taxId, amount);
}
})
.reduceByKey(
new Function2<BigDecimal, BigDecimal, BigDecimal>() {
@Override
public BigDecimal call(
BigDecimal transactionAmount1, BigDecimal transactionAmount2)
throws Exception {
return transactionAmount1.add(transactionAmount2);
}
})
.filter(
new Function<Tuple2<String, BigDecimal>, Boolean>() {
@Override
public Boolean call(Tuple2<String, BigDecimal> transactionSumByTaxId)
throws Exception {
// returns true if total is greater than 999
Boolean result = transactionSumByTaxId._2().compareTo(new BigDecimal(999)) == 1;
System.out.println(
"tran "
+ transactionSumByTaxId._1()
+ " with amount "
+ transactionSumByTaxId._2());
if (result) {
System.out.println(
"warning "
+ transactionSumByTaxId._1()
+ " has value "
+ transactionSumByTaxId._2()
+ " is greater than 999");
}
return result;
}
});
JavaDStream<Warning> mappedWarnings =
warnings.map(
new Function<Tuple2<String, BigDecimal>, Warning>() {
@Override
public Warning call(Tuple2<String, BigDecimal> warnings) throws Exception {
Warning warning = new Warning();
warning.setSsn(warnings._1());
warning.setId(UUIDs.timeBased());
warning.setAmount(warnings._2());
warning.setRule("OVER_DOLLAR_AMOUNT");
return warning;
}
});
javaFunctions(mappedWarnings)
.writerBuilder(getKeySpaceName(), warningsTableName, mapToRow(Warning.class))
.saveToCassandra();
return sparkContext;
}