private static void basicCassandraSession(CassandraConnector connector) { try (Session session = connector.openSession()) { session.execute( "CREATE KEYSPACE IF NOT EXISTS test WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1 }"); session.execute( "CREATE TABLE IF NOT EXISTS test.key_value (key INT PRIMARY KEY, value VARCHAR)"); session.execute("TRUNCATE test.key_value"); session.execute("INSERT INTO test.key_value(key, value) VALUES (1, 'first row')"); session.execute("INSERT INTO test.key_value(key, value) VALUES (2, 'second row')"); session.execute("INSERT INTO test.key_value(key, value) VALUES (3, 'third row')"); // this will be used in the next test session.execute( "CREATE TABLE IF NOT EXISTS test.people (id INT, name TEXT, birth_date TIMESTAMP, PRIMARY KEY (id))"); // secondary indexes are not recommended in production. This is just a sample to demonstrate // the connector session.execute("CREATE INDEX IF NOT EXISTS people_name_idx ON test.people(name)"); } }
public JavaStreamingContext createContext() { CassandraConnectionContext connectionContext = connectToCassandra(); JavaDStream<String> rdd = connectionContext.getRDD(); JavaStreamingContext sparkContext = connectionContext.getStreamingContext(); final CassandraConnector connector = connectionContext.getCassandraConnector(); final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); JavaDStream<Tuple2<String, Transaction>> transactionList = rdd.map( new Function<String, Tuple2<String, Transaction>>() { @Override public Tuple2<String, Transaction> call(String line) throws Exception { String[] columns = line.split(","); String taxId = columns[0]; String name = columns[1]; String merchant = columns[2]; BigDecimal amount = new BigDecimal(columns[3]); Date transactionDate; try { transactionDate = format.parse(columns[4]); } catch (ParseException e) { e.printStackTrace(); throw new RuntimeException(e); } String tranId = columns[5]; System.out.println(line); Tuple2<String, Transaction> transaction = new Tuple2<>( taxId, new Transaction(name, merchant, amount, transactionDate, tranId)); return transaction; } }); transactionList.cache(); final String warningsTableName = "warnings"; try (Session session = connector.openSession()) { session.execute( String.format("DROP TABLE IF EXISTS %s.%s", getKeySpaceName(), warningsTableName)); session.execute( String.format( "CREATE TABLE IF NOT EXISTS %s.%s (ssn text, " + "id uuid, amount decimal, rule text, PRIMARY KEY(ssn, id))", getKeySpaceName(), warningsTableName)); } // setup warning on more than certain number of transactions by user in a 60 second window, // every 10 seconds JavaPairDStream<String, BigDecimal> warnings = transactionList .window(new Duration(60000), new Duration(10000)) .mapToPair( new PairFunction<Tuple2<String, Transaction>, String, BigDecimal>() { @Override public Tuple2<String, BigDecimal> call(Tuple2<String, Transaction> transaction) throws Exception { String taxId = transaction._1(); BigDecimal amount = transaction._2().getAmount(); return new Tuple2<>(taxId, amount); } }) .reduceByKey( new Function2<BigDecimal, BigDecimal, BigDecimal>() { @Override public BigDecimal call( BigDecimal transactionAmount1, BigDecimal transactionAmount2) throws Exception { return transactionAmount1.add(transactionAmount2); } }) .filter( new Function<Tuple2<String, BigDecimal>, Boolean>() { @Override public Boolean call(Tuple2<String, BigDecimal> transactionSumByTaxId) throws Exception { // returns true if total is greater than 999 Boolean result = transactionSumByTaxId._2().compareTo(new BigDecimal(999)) == 1; System.out.println( "tran " + transactionSumByTaxId._1() + " with amount " + transactionSumByTaxId._2()); if (result) { System.out.println( "warning " + transactionSumByTaxId._1() + " has value " + transactionSumByTaxId._2() + " is greater than 999"); } return result; } }); JavaDStream<Warning> mappedWarnings = warnings.map( new Function<Tuple2<String, BigDecimal>, Warning>() { @Override public Warning call(Tuple2<String, BigDecimal> warnings) throws Exception { Warning warning = new Warning(); warning.setSsn(warnings._1()); warning.setId(UUIDs.timeBased()); warning.setAmount(warnings._2()); warning.setRule("OVER_DOLLAR_AMOUNT"); return warning; } }); javaFunctions(mappedWarnings) .writerBuilder(getKeySpaceName(), warningsTableName, mapToRow(Warning.class)) .saveToCassandra(); return sparkContext; }