private static void publishDb(
KarmaTopologyBuilder builder, String id, List<String> tables, Publisher factory) {
for (String schema : tables) {
Schema s = Schema.parse(schema);
String table = s.getName();
Fields idFields = new Fields(s.getIdFields());
builder
.setBolt(id + "_" + table, factory.newBolt(table, idFields))
.fieldsGrouping(table, idFields);
}
}
private static StormTopology buildTopology(KarmaConfig karmaConfig) {
KarmaTopologyBuilder builder = new KarmaTopologyBuilder(karmaConfig, "bankdemo");
// *** setup all the source queues
builder.setSpout("users", "users", new JsonScheme("id", "name"), 4);
builder.setSpout("deposits", "deposits", new JsonScheme("id", "account", "amount"), 4);
builder.setSpout("withdrawals", "withdrawals", new JsonScheme("id", "account", "amount"), 4);
builder.setSpout(
"transfers", "transfers", new JsonScheme("id", "srcAccount", "dstAccount", "amount"), 4);
// *** User Count: { count }
// *** SELECT COUNT(*) as userCount FROM users WHERE name IS NOT NULL
builder.map("{ name }", "users(id)", new Count()).red("{ count }", "userCount()", new Sum());
// buildSniffer(builder, "userCount");
/*
Account Balance: { accountId balance }
Equivalent to:
SELECT account, SUM(balance) as balance FROM {
SELECT account AS account, amount AS balance FROM deposits
UNION ALL
SELECT account AS account, -amount AS balance FROM withdrawals
UNION ALL
SELECT dstAccount AS account, amount AS balance FROM transfers
UNION ALL
SELECT srcAccount AS account, -amount AS balance FROM transfers
} WHERE account IN NOT NULL AND balance IS NOT NULL GROUP BY account
*/
builder
.map("{ account amount }", "deposits(id)")
.map("{ account amount }", "withdrawals(id)", "{ d -> emit(d.account, -d.amount) }")
.map("{ dstAccount amount }", "transfers(id)")
.map("{ srcAccount amount }", "transfers(id)", "{ d -> emit(d.srcAccount, -d.amount) }")
.red("{ balance }", "accounts(account)", new Sum());
/*
Account Summary: { accountId, username, accountBalance }
Equivalent to: TODO
*/
builder
.map(
"{ account balance }", "accounts(account)", "{ d -> emit(d.account, null, d.balance) }")
.map("{ id name }", "users(id)", "{ u -> emit(u.id, u.name, null) }")
.red(
"{ userName balance }",
"accountJoin(id)",
"{ a, b -> [a.userName ?: b.userName, a.balance ?: b.balance ] }");
builder
.map("{ id userName balance }", "accountJoin(id)")
.red("{ userName balance }", "accountSummary(id)", new MinValue());
// buildSniffer(builder, "accountSummary");
/*
Top Account: a temp table used for calculating top customer
SELECT balance, account FROM {
SELECT account, SUM(balance) as balance FROM {
SELECT id AS account, amount AS balance FROM deposits
UNION ALL
SELECT id AS account, -amount AS balance FROM withdrawals
UNION ALL
SELECT dstAccount AS account, amount AS balance FROM transfers
UNION ALL
SELECT srcAccount AS account, -amount AS balance FROM transfers
} WHERE account IS NOT NULL AND balance IS NOT NULL GROUP BY account
} ORDER BY balance DESC, account DESC LIMIT 1
*/
// karma.map("{ balance account }", "accountBalance(account)", "{ d -> emit([[d.balance,
// d.account]]) }")
builder
.map("{ balance account }", "accounts(account)", new TopNMapper())
.red(
"{ topN }",
"topAccounts()",
new TopN(10)); // *** TopN expects a single column with a list of tuples in it
/*
Top Customer: { name }
SELECT u.name as name FROM user u INNER JOIN {
SELECT balance, account FROM {
SELECT account as account, SUM(balance) as balance FROM {
SELECT id AS account, amount AS balance FROM deposits
UNION ALL
SELECT id AS account, -amount AS balance FROM withdrawals
UNION ALL
SELECT dstAccount AS account, amount AS balance FROM transfers
UNION ALL
SELECT srcAccount AS account, -amount AS balance FROM transfers
} WHERE account IS NOT NULL AND balance IS NOT NULL GROUP BY account
} ORDER BY balance DESC, account DESC LIMIT 1
} j ON u.id = j.account
*/
builder
.map("{ topN }", "topAccounts()", "{ t -> t.topN.each { emit(it[1], null, it[0]) } }")
.map("{ id name }", "users(id)", "{ u -> emit(u.id, u.name, null) }")
.red(
"{ name balance }",
"topCustomerStep1(account)",
"{ a, b -> [a.name ?: b.name, a.balance ?: b.balance] }");
builder
.map("{ balance name account }", "topCustomerStep1(account)", new TopNMapper())
.red("{ topN }", "topCustomers()", new TopN(10));
// buildSniffer(builder, "topCustomers");
builder
.map("{ amount id }", "deposits(id)", new TopNMapper())
.red("{ topN }", "topDeposits()", new TopN(10));
/*
Transaction Count: { accountId, transactions }
Equivalent to:
SELECT account as account, count(*) as transactions FROM {
SELECT id as account, amount as amount FROM deposits
UNION ALL
SELECT id as account, amount as amount FROM withdrawals
UNION ALL
SELECT srcAccount as account, amount as amount FROM transfers
UNION ALL
SELECT dstAccount AS account, amount as amount FROM transfers
} WHERE account IS NOT NULL AND amount IS NOT NULL GROUP BY account
*/
builder
.map("{ account amount }", "deposits(id) ", new Count("account"))
.map("{ account amount }", "withdrawals(id) ", new Count("account"))
.map("{ dstAccount amount }", "transfers(id) ", new Count("dstAccount"))
.map("{ srcAccount amount }", "transfers(id) ", new Count("srcAccount"))
.red("{ transactions }", "transactionCounts(account)", new Sum());
// buildSniffer(builder, "transactionCounts");
builder
.map("{ account amount }", "deposits(id)", new Count())
.map("{ account amount }", "withdrawals(id) ", new Count())
.map("{ srcAccount dstAccount amount }", "transfers(id) ", new Count())
.red("{ transactions }", "totalTransactions()", new Sum());
// buildSniffer(builder, "totalTransactions");
/*
Summary: { users transactions totalAssets topCustomer }
SELECT SUM(amount) FROM {
SELECT COUNT(*) as users FROM users WHERE name IS NOT NULL
SELECT COUNT(*) as accounts FROM accounts WHERE balance IS NOT NULL
SELECT SUM(counts) as transactions FROM {
SELECT COUNT(*) as counts FROM deposits WHERE amount IS NOT NULL
UNION ALL
SELECT COUNT(*) as counts FROM withdrawals WHERE amount IS NOT NULL
UNION ALL
SELECT COUNT(*) as counts FROM transfers WHERE amount IS NOT NULL
}
SELECT SUM(balance) as totalAssets FROM {
SELECT SUM(amount) as balance FROM deposits WHERE amount IS NOT NUlL
UNION ALL
SELECT SUM(-amount) as balance FROM withdrawals WHERE amount IS NOT NULL
} WHERE balance IN NOT NULL
} WHERE ..., ..., ... TODO: finish this
*/
builder
.map("{ amount }", "deposits(id)", "{ d -> emit(0, 1, d.amount, null) }")
.map("{ amount }", "withdrawals(id)", "{ d -> emit(0, 1, -d.amount, null) }")
.map("{ srcAccount dstAccount }", "transfers(id)", "{ d -> emit(0, 1, 0, null) }")
.map("{ name }", "users(id)", "{ d -> emit(1, 0, 0, null) }")
.map("{ topN }", "topCustomers()", "{ d -> emit(0, 0, 0, d.topN) }")
.red(
"{ users transactions totalAssets topCustomers }",
"summary()",
new Union(new Sum(), new Sum(), new Sum(), new MinValue()));
// buildSniffer(builder, "summary");
String duckStream = "default";
builder
.setBolt("duckAssets", new DucksBoardCounterBolt(new Fields(), "115222", "totalAssets"))
.globalGrouping("summary", duckStream);
builder
.setBolt(
"duckTransactions", new DucksBoardCounterBolt(new Fields(), "115268", "transactions"))
.globalGrouping("totalTransactions", duckStream);
builder
.setBolt(
"duckTopCustomers", new DucksBoardLeaderboardBolt(new Fields(), "115264", "topN", 1, 0))
.globalGrouping("topCustomers", duckStream);
builder
.setBolt(
"duckTopDeposits", new DucksBoardLeaderboardBolt(new Fields(), "116744", "topN", 1, 0))
.globalGrouping("topDeposits", duckStream);
builder
.map("{ amount }", "deposits(id)", new Count())
.red("{ value }", "depositCount()", new Sum());
builder
.map("{ amount }", "withdrawals(id)", new Count())
.red("{ value }", "withdrawalCount()", new Sum());
builder
.map("{ amount }", "transfers(id)", new Count())
.red("{ value }", "transferCount()", new Sum());
builder
.setBolt("duckDeposits", new DucksBoardCounterBolt(new Fields(), "116794", "value"))
.globalGrouping("depositCount", duckStream);
builder
.setBolt("duckWithdrawals", new DucksBoardCounterBolt(new Fields(), "116795", "value"))
.globalGrouping("withdrawalCount", duckStream);
builder
.setBolt("duckTransfers", new DucksBoardCounterBolt(new Fields(), "116796", "value"))
.globalGrouping("transferCount", duckStream);
List<String> tables =
L(
"users(id)",
"userCount()",
"accounts(account)",
"topCustomers()",
"transactionCounts(account)",
"totalTransactions()",
"summary()",
"transfers(id)");
publishDb(builder, "cass", tables, new CassandraPublisher("bank"));
publishDb(
builder,
"cloud",
tables,
new CouchDbPublisher("https://eldenbishop.cloudant.com", "eldenbishop", "shinobu", "bank"));
return builder.createTopology();
}
private static void buildSniffer(
KarmaTopologyBuilder builder, String srcComponentId, String stream) {
builder
.setBolt(srcComponentId + "Sniffer", new PrintBolt("### " + srcComponentId, 1000, false))
.globalGrouping(srcComponentId, stream);
}
public static void main(String[] args) throws IOException {
Logger.getRootLogger().setLevel(Level.WARN);
// *** start the storm cluster
LocalCluster cluster = new LocalCluster();
// *** start the embedded kafka service
LocalKafkaBroker broker = new LocalKafkaBroker(0, 9090, 1, "localhost:2000");
// *** configure replay and karma to use the local kafka instance
ReplayConfig replay =
new ReplayConfig().staticHosts(broker.getHostPortStrings(), broker.getNumPartitions());
KarmaConfig karmaConfig =
new KarmaConfig("a").replay(replay).reducerState(new InMemoryReducerState());
KarmaTopologyBuilder karma = new KarmaTopologyBuilder(karmaConfig, "testA");
karma.setSpout(
"orgSpout",
new KafkaSpout(replay.buildReplaySpoutConfig("org", ORG_SCHEME, "orgSpoutId")),
4);
karma.setSpout(
"userSpout",
new KafkaSpout(replay.buildReplaySpoutConfig("user", USER_SCHEME, "userSpoutId")),
4);
karma
.map("{ orgId }", "userSpout(id)", new Count("orgId"))
.red("{ userCount }", "orgUserCounts(orgId)", new Sum());
karma
.map("{ userCount }", "orgUserCounts(orgId)", new Count("userCount"))
.red("{ total samples }", "allOrgs()", new Sum())
.fmt("{ totalUsers averagePerOrg }", "{ d -> [d.total, d.total / d.samples] }");
buildSniffer("allOrgs", karma);
karma
.map("{ orgId }", "userSpout(id)", "{ u -> emit(u.orgId, 1) }")
.red("{ userCount }", "orgUserCounts2(orgId)", "{ a, b -> [a.userCount + b.userCount] }");
karma
.map("{ userCount }", "orgUserCounts2(orgId)", "{ d -> emit(d.userCount, 1) }")
.red(
"{ total samples }",
"allOrgs2()",
"{ a, b -> [a.total + b.total, a.samples + b.samples]}")
.fmt("{ totalUsers averagePerOrg }", "{ d -> [d.total, d.total / d.samples] }");
buildSniffer("allOrgs2", karma);
// *** build a name count using the scripting support
karma
.map("{ name }", "userSpout(id)", "{ u -> emit(u.name, 1L) }")
.red("{ count }", "nameCounts(name)", "{ a, b -> [a.count + b.count] }");
buildSniffer("nameCounts", karma);
karma
.map("{ name }", "userSpout(id)", "{ u -> emit(u.name, 1L) }")
.red("{ count }", "nameCounts2(name)", "{ a, b -> [a.count + b.count] }");
buildSniffer("nameCounts2", karma);
karma
.map("{ orgId }", "userSpout(id)", "{ u -> emit(u.orgId, 1L) }")
.red("{ count }", "empCounts(orgId)", "{ a, b -> [a.count + b.count] }");
buildSniffer("empCounts", karma);
karma
.map("{ name }", "userSpout(id)", "{ u -> emit(1L) }")
.red("{ count }", "userCount()", "{ a, b -> [a.count + b.count] }");
buildSniffer("userCount", karma);
karma
.map("{ name }", "userSpout(id)", "{ u -> emit(1L) }")
.red("{ count }", "userCount2()", "{ a, b -> [a.count + b.count] }");
buildSniffer("userCount2", karma);
karma
.map(
"{ id name }",
"orgSpout(id)",
new Mapper() {
@Override
public void map(Tuple t, Emitter e) {
e.emit(t.getValueByField("id"), t.getStringByField("name"), L());
}
})
.map(
"{ orgId name }",
"userSpout(id)",
new Mapper() {
@Override
public void map(Tuple t, Emitter e) {
e.emit(t.getValueByField("orgId"), null, L(t.getStringByField("name")));
}
})
.red(
"{ orgName userNames }",
"orgToUsernames(orgId)",
new Reducer() {
@Override
public List reduce(Tuple key, Tuple a, Tuple b) {
Set<String> names = new TreeSet<String>();
names.addAll((List) (a.getValueByField("userNames")));
names.addAll((List) (b.getValueByField("userNames")));
return L(
a.getString(0) != null ? a.getString(0) : b.getString(0), new ArrayList(names));
}
});
karma
.map(
"orgSpout",
L("id"),
L("id", "name"),
new Mapper() {
@Override
public void map(Tuple t, Emitter e) {
e.emit(t.getValueByField("id"), t.getStringByField("name"), L());
}
})
.map(
"userSpout",
L("id"),
L("orgId", "name"),
new Mapper() {
@Override
public void map(Tuple t, Emitter e) {
e.emit(t.getValueByField("orgId"), null, L(t.getStringByField("name")));
}
})
.red(
"{ orgName userNames }",
"org2Usernames(orgId)",
new Reducer() {
@Override
public List reduce(Tuple key, Tuple a, Tuple b) {
Set<String> names = new TreeSet<String>();
names.addAll((List) (a.getValueByField("userNames")));
names.addAll((List) (b.getValueByField("userNames")));
return L(
a.getString(0) != null ? a.getString(0) : b.getString(0), new ArrayList(names));
}
});
buildSniffer("org2Usernames", karma);
karma
.map(
"{ orgName userNames }",
"org2Usernames(orgId)",
new Mapper() {
@Override
public void map(Tuple t, Emitter e) {
String orgName = t.getStringByField("orgName");
if (orgName != null)
for (String userName : (List<String>) t.getValueByField("userNames")) {
e.emit(userName, L(orgName));
}
}
})
.red(
"{ orgNames }",
"userNames2OrgNames(userName)",
new Reducer() {
@Override
public List reduce(Tuple key, Tuple a, Tuple b) {
System.out.println("userNames2OrgNames reducing: a: " + a + ", b: " + b);
Set<String> orgNames = new TreeSet<String>();
orgNames.addAll((List) a.getValue(0));
orgNames.addAll((List) b.getValue(0));
return L(new ArrayList(orgNames));
}
});
buildSniffer("userNames2OrgNames", karma);
cluster.submitTopology("karma", new Config(), karma.createTopology());
Producer<Long, Message> producer = broker.buildSyncProducer();
// Demo.countdown("Adding orgs and users in ", 5);
// sendOrgs(producer);
// send100Users(producer);
// Demo.readEnter("*** Adding acme", 3);
sendOrg(producer, 1000, "Acme");
// Demo.readEnter("*** Adding 10 greggs", 3);
for (int i = 0; i < 10; i++) {
// Demo.readEnter("** Adding gregg " + (i + 1), 1);
sendUser(producer, 2000 + i, "Gregg", 1000);
}
// Demo.readEnter("*** Changing greggs to seth and assigning to org 1 in", 3);
Utils.sleep(2000);
sendOrg(producer, 1, "Kfc");
for (int i = 0; i < 10; i++) {
// Demo.readEnter("** Changing gregg " + (i + 1) + " to seth and Kfc in", 1);
sendUser(producer, 2000 + i, "Seth", 1);
}
// Demo.readEnter("*** Deleting acme", 3);
for (int i = 0; i < 10; i++) deleteUser(producer, 2000 + i);
deleteOrg(producer, 1000);
deleteOrg(producer, 1);
Utils.sleep(100000);
producer.close();
}
private static void buildSniffer(String srcComponentId, KarmaTopologyBuilder builder) {
builder
.setBolt(srcComponentId + "Sniffer", new PrintBolt("### " + srcComponentId))
.globalGrouping(srcComponentId, "changes");
}
