Esempio n. 1
0
 private static void publishDb(
     KarmaTopologyBuilder builder, String id, List<String> tables, Publisher factory) {
   for (String schema : tables) {
     Schema s = Schema.parse(schema);
     String table = s.getName();
     Fields idFields = new Fields(s.getIdFields());
     builder
         .setBolt(id + "_" + table, factory.newBolt(table, idFields))
         .fieldsGrouping(table, idFields);
   }
 }
Esempio n. 2
0
  private static StormTopology buildTopology(KarmaConfig karmaConfig) {

    KarmaTopologyBuilder builder = new KarmaTopologyBuilder(karmaConfig, "bankdemo");

    // *** setup all the source queues
    builder.setSpout("users", "users", new JsonScheme("id", "name"), 4);
    builder.setSpout("deposits", "deposits", new JsonScheme("id", "account", "amount"), 4);
    builder.setSpout("withdrawals", "withdrawals", new JsonScheme("id", "account", "amount"), 4);
    builder.setSpout(
        "transfers", "transfers", new JsonScheme("id", "srcAccount", "dstAccount", "amount"), 4);

    // *** User Count: { count }
    // *** SELECT COUNT(*) as userCount FROM users WHERE name IS NOT NULL
    builder.map("{ name }", "users(id)", new Count()).red("{ count }", "userCount()", new Sum());
    //        buildSniffer(builder, "userCount");

    /*

    Account Balance: { accountId balance }

    Equivalent to:
    SELECT account, SUM(balance) as balance FROM {
        SELECT account AS account, amount AS balance FROM deposits
        UNION ALL
        SELECT account AS account, -amount AS balance FROM withdrawals
        UNION ALL
        SELECT dstAccount AS account, amount AS balance FROM transfers
        UNION ALL
        SELECT srcAccount AS account, -amount AS balance FROM transfers
    } WHERE account IN NOT NULL AND balance IS NOT NULL GROUP BY account
     */
    builder
        .map("{ account amount }", "deposits(id)")
        .map("{ account amount }", "withdrawals(id)", "{ d -> emit(d.account, -d.amount) }")
        .map("{ dstAccount amount }", "transfers(id)")
        .map("{ srcAccount amount }", "transfers(id)", "{ d -> emit(d.srcAccount, -d.amount) }")
        .red("{ balance }", "accounts(account)", new Sum());

    /*

    Account Summary: { accountId, username, accountBalance }

    Equivalent to: TODO
     */
    builder
        .map(
            "{ account balance }", "accounts(account)", "{ d -> emit(d.account, null, d.balance) }")
        .map("{ id name }", "users(id)", "{ u -> emit(u.id, u.name, null) }")
        .red(
            "{ userName balance }",
            "accountJoin(id)",
            "{ a, b -> [a.userName ?: b.userName, a.balance ?: b.balance ] }");

    builder
        .map("{ id userName balance }", "accountJoin(id)")
        .red("{ userName balance }", "accountSummary(id)", new MinValue());
    //        buildSniffer(builder, "accountSummary");

    /*
    Top Account: a temp table used for calculating top customer

    SELECT balance, account FROM {
        SELECT account, SUM(balance) as balance FROM {
            SELECT id AS account, amount AS balance FROM deposits
            UNION ALL
            SELECT id AS account, -amount AS balance FROM withdrawals
            UNION ALL
            SELECT dstAccount AS account, amount AS balance FROM transfers
            UNION ALL
            SELECT srcAccount AS account, -amount AS balance FROM transfers
        } WHERE account IS NOT NULL AND balance IS NOT NULL GROUP BY account
    } ORDER BY balance DESC, account DESC LIMIT 1
    */
    //        karma.map("{ balance account }", "accountBalance(account)", "{ d -> emit([[d.balance,
    // d.account]]) }")
    builder
        .map("{ balance account }", "accounts(account)", new TopNMapper())
        .red(
            "{ topN }",
            "topAccounts()",
            new TopN(10)); // *** TopN expects a single column with a list of tuples in it

    /*
    Top Customer: { name }

    SELECT u.name as name FROM user u INNER JOIN {
        SELECT balance, account FROM {
            SELECT account as account, SUM(balance) as balance FROM {
                SELECT id AS account, amount AS balance FROM deposits
                UNION ALL
                SELECT id AS account, -amount AS balance FROM withdrawals
                UNION ALL
                SELECT dstAccount AS account, amount AS balance FROM transfers
                UNION ALL
                SELECT srcAccount AS account, -amount AS balance FROM transfers
            } WHERE account IS NOT NULL AND balance IS NOT NULL GROUP BY account
        } ORDER BY balance DESC, account DESC LIMIT 1
    } j ON u.id = j.account
     */
    builder
        .map("{ topN }", "topAccounts()", "{ t -> t.topN.each { emit(it[1], null, it[0]) } }")
        .map("{ id name }", "users(id)", "{ u -> emit(u.id, u.name, null) }")
        .red(
            "{ name balance }",
            "topCustomerStep1(account)",
            "{ a, b -> [a.name ?: b.name, a.balance ?: b.balance] }");

    builder
        .map("{ balance name account }", "topCustomerStep1(account)", new TopNMapper())
        .red("{ topN }", "topCustomers()", new TopN(10));
    //        buildSniffer(builder, "topCustomers");

    builder
        .map("{ amount id }", "deposits(id)", new TopNMapper())
        .red("{ topN }", "topDeposits()", new TopN(10));

    /*

    Transaction Count: { accountId, transactions }

    Equivalent to:
    SELECT account as account, count(*) as transactions FROM {
        SELECT id as account, amount as amount FROM deposits
        UNION ALL
        SELECT id as account, amount as amount FROM withdrawals
        UNION ALL
        SELECT srcAccount as account, amount as amount FROM transfers
        UNION ALL
        SELECT dstAccount AS account, amount as amount FROM transfers
    } WHERE account IS NOT NULL AND amount IS NOT NULL GROUP BY account
     */
    builder
        .map("{ account amount }", "deposits(id) ", new Count("account"))
        .map("{ account amount }", "withdrawals(id) ", new Count("account"))
        .map("{ dstAccount amount }", "transfers(id) ", new Count("dstAccount"))
        .map("{ srcAccount amount }", "transfers(id) ", new Count("srcAccount"))
        .red("{ transactions }", "transactionCounts(account)", new Sum());
    //        buildSniffer(builder, "transactionCounts");

    builder
        .map("{ account amount }", "deposits(id)", new Count())
        .map("{ account amount }", "withdrawals(id) ", new Count())
        .map("{ srcAccount dstAccount amount }", "transfers(id) ", new Count())
        .red("{ transactions }", "totalTransactions()", new Sum());
    //        buildSniffer(builder, "totalTransactions");

    /*
    Summary: { users transactions totalAssets topCustomer }

    SELECT SUM(amount) FROM {
        SELECT COUNT(*) as users FROM users WHERE name IS NOT NULL
        SELECT COUNT(*) as accounts FROM accounts WHERE balance IS NOT NULL
        SELECT SUM(counts) as transactions FROM {
            SELECT COUNT(*) as counts FROM deposits WHERE amount IS NOT NULL
            UNION ALL
            SELECT COUNT(*) as counts FROM withdrawals WHERE amount IS NOT NULL
            UNION ALL
            SELECT COUNT(*) as counts FROM transfers WHERE amount IS NOT NULL
        }
        SELECT SUM(balance) as totalAssets FROM {
            SELECT SUM(amount) as balance FROM deposits WHERE amount IS NOT NUlL
            UNION ALL
            SELECT SUM(-amount) as balance FROM withdrawals WHERE amount IS NOT NULL
        } WHERE balance IN NOT NULL
    } WHERE ..., ..., ... TODO: finish this
     */
    builder
        .map("{ amount }", "deposits(id)", "{ d -> emit(0, 1, d.amount, null) }")
        .map("{ amount }", "withdrawals(id)", "{ d -> emit(0, 1, -d.amount, null) }")
        .map("{ srcAccount dstAccount }", "transfers(id)", "{ d -> emit(0, 1, 0, null) }")
        .map("{ name }", "users(id)", "{ d -> emit(1, 0, 0, null) }")
        .map("{ topN }", "topCustomers()", "{ d -> emit(0, 0, 0, d.topN) }")
        .red(
            "{ users transactions totalAssets topCustomers }",
            "summary()",
            new Union(new Sum(), new Sum(), new Sum(), new MinValue()));
    //        buildSniffer(builder, "summary");

    String duckStream = "default";
    builder
        .setBolt("duckAssets", new DucksBoardCounterBolt(new Fields(), "115222", "totalAssets"))
        .globalGrouping("summary", duckStream);
    builder
        .setBolt(
            "duckTransactions", new DucksBoardCounterBolt(new Fields(), "115268", "transactions"))
        .globalGrouping("totalTransactions", duckStream);
    builder
        .setBolt(
            "duckTopCustomers", new DucksBoardLeaderboardBolt(new Fields(), "115264", "topN", 1, 0))
        .globalGrouping("topCustomers", duckStream);
    builder
        .setBolt(
            "duckTopDeposits", new DucksBoardLeaderboardBolt(new Fields(), "116744", "topN", 1, 0))
        .globalGrouping("topDeposits", duckStream);

    builder
        .map("{ amount }", "deposits(id)", new Count())
        .red("{ value }", "depositCount()", new Sum());
    builder
        .map("{ amount }", "withdrawals(id)", new Count())
        .red("{ value }", "withdrawalCount()", new Sum());
    builder
        .map("{ amount }", "transfers(id)", new Count())
        .red("{ value }", "transferCount()", new Sum());
    builder
        .setBolt("duckDeposits", new DucksBoardCounterBolt(new Fields(), "116794", "value"))
        .globalGrouping("depositCount", duckStream);
    builder
        .setBolt("duckWithdrawals", new DucksBoardCounterBolt(new Fields(), "116795", "value"))
        .globalGrouping("withdrawalCount", duckStream);
    builder
        .setBolt("duckTransfers", new DucksBoardCounterBolt(new Fields(), "116796", "value"))
        .globalGrouping("transferCount", duckStream);

    List<String> tables =
        L(
            "users(id)",
            "userCount()",
            "accounts(account)",
            "topCustomers()",
            "transactionCounts(account)",
            "totalTransactions()",
            "summary()",
            "transfers(id)");
    publishDb(builder, "cass", tables, new CassandraPublisher("bank"));
    publishDb(
        builder,
        "cloud",
        tables,
        new CouchDbPublisher("https://eldenbishop.cloudant.com", "eldenbishop", "shinobu", "bank"));

    return builder.createTopology();
  }
Esempio n. 3
0
 private static void buildSniffer(
     KarmaTopologyBuilder builder, String srcComponentId, String stream) {
   builder
       .setBolt(srcComponentId + "Sniffer", new PrintBolt("### " + srcComponentId, 1000, false))
       .globalGrouping(srcComponentId, stream);
 }
Esempio n. 4
0
  public static void main(String[] args) throws IOException {

    Logger.getRootLogger().setLevel(Level.WARN);

    // *** start the storm cluster
    LocalCluster cluster = new LocalCluster();

    // *** start the embedded kafka service
    LocalKafkaBroker broker = new LocalKafkaBroker(0, 9090, 1, "localhost:2000");

    // *** configure replay and karma to use the local kafka instance
    ReplayConfig replay =
        new ReplayConfig().staticHosts(broker.getHostPortStrings(), broker.getNumPartitions());
    KarmaConfig karmaConfig =
        new KarmaConfig("a").replay(replay).reducerState(new InMemoryReducerState());

    KarmaTopologyBuilder karma = new KarmaTopologyBuilder(karmaConfig, "testA");

    karma.setSpout(
        "orgSpout",
        new KafkaSpout(replay.buildReplaySpoutConfig("org", ORG_SCHEME, "orgSpoutId")),
        4);
    karma.setSpout(
        "userSpout",
        new KafkaSpout(replay.buildReplaySpoutConfig("user", USER_SCHEME, "userSpoutId")),
        4);

    karma
        .map("{ orgId }", "userSpout(id)", new Count("orgId"))
        .red("{ userCount }", "orgUserCounts(orgId)", new Sum());
    karma
        .map("{ userCount }", "orgUserCounts(orgId)", new Count("userCount"))
        .red("{ total samples }", "allOrgs()", new Sum())
        .fmt("{ totalUsers averagePerOrg }", "{ d -> [d.total, d.total / d.samples] }");
    buildSniffer("allOrgs", karma);

    karma
        .map("{ orgId }", "userSpout(id)", "{ u -> emit(u.orgId, 1) }")
        .red("{ userCount }", "orgUserCounts2(orgId)", "{ a, b -> [a.userCount + b.userCount] }");
    karma
        .map("{ userCount }", "orgUserCounts2(orgId)", "{ d -> emit(d.userCount, 1) }")
        .red(
            "{ total samples }",
            "allOrgs2()",
            "{ a, b -> [a.total + b.total, a.samples + b.samples]}")
        .fmt("{ totalUsers averagePerOrg }", "{ d -> [d.total, d.total / d.samples] }");
    buildSniffer("allOrgs2", karma);

    // *** build a name count using the scripting support
    karma
        .map("{ name }", "userSpout(id)", "{ u -> emit(u.name, 1L) }")
        .red("{ count }", "nameCounts(name)", "{ a, b -> [a.count + b.count] }");
    buildSniffer("nameCounts", karma);

    karma
        .map("{ name }", "userSpout(id)", "{ u -> emit(u.name, 1L) }")
        .red("{ count }", "nameCounts2(name)", "{ a, b -> [a.count + b.count] }");
    buildSniffer("nameCounts2", karma);

    karma
        .map("{ orgId }", "userSpout(id)", "{ u -> emit(u.orgId, 1L) }")
        .red("{ count }", "empCounts(orgId)", "{ a, b -> [a.count + b.count] }");
    buildSniffer("empCounts", karma);

    karma
        .map("{ name }", "userSpout(id)", "{ u -> emit(1L) }")
        .red("{ count }", "userCount()", "{ a, b -> [a.count + b.count] }");
    buildSniffer("userCount", karma);

    karma
        .map("{ name }", "userSpout(id)", "{ u -> emit(1L) }")
        .red("{ count }", "userCount2()", "{ a, b -> [a.count + b.count] }");
    buildSniffer("userCount2", karma);

    karma
        .map(
            "{ id name }",
            "orgSpout(id)",
            new Mapper() {
              @Override
              public void map(Tuple t, Emitter e) {
                e.emit(t.getValueByField("id"), t.getStringByField("name"), L());
              }
            })
        .map(
            "{ orgId name }",
            "userSpout(id)",
            new Mapper() {
              @Override
              public void map(Tuple t, Emitter e) {
                e.emit(t.getValueByField("orgId"), null, L(t.getStringByField("name")));
              }
            })
        .red(
            "{ orgName userNames }",
            "orgToUsernames(orgId)",
            new Reducer() {
              @Override
              public List reduce(Tuple key, Tuple a, Tuple b) {
                Set<String> names = new TreeSet<String>();
                names.addAll((List) (a.getValueByField("userNames")));
                names.addAll((List) (b.getValueByField("userNames")));
                return L(
                    a.getString(0) != null ? a.getString(0) : b.getString(0), new ArrayList(names));
              }
            });

    karma
        .map(
            "orgSpout",
            L("id"),
            L("id", "name"),
            new Mapper() {
              @Override
              public void map(Tuple t, Emitter e) {
                e.emit(t.getValueByField("id"), t.getStringByField("name"), L());
              }
            })
        .map(
            "userSpout",
            L("id"),
            L("orgId", "name"),
            new Mapper() {
              @Override
              public void map(Tuple t, Emitter e) {
                e.emit(t.getValueByField("orgId"), null, L(t.getStringByField("name")));
              }
            })
        .red(
            "{ orgName userNames }",
            "org2Usernames(orgId)",
            new Reducer() {
              @Override
              public List reduce(Tuple key, Tuple a, Tuple b) {
                Set<String> names = new TreeSet<String>();
                names.addAll((List) (a.getValueByField("userNames")));
                names.addAll((List) (b.getValueByField("userNames")));
                return L(
                    a.getString(0) != null ? a.getString(0) : b.getString(0), new ArrayList(names));
              }
            });
    buildSniffer("org2Usernames", karma);

    karma
        .map(
            "{ orgName userNames }",
            "org2Usernames(orgId)",
            new Mapper() {
              @Override
              public void map(Tuple t, Emitter e) {
                String orgName = t.getStringByField("orgName");
                if (orgName != null)
                  for (String userName : (List<String>) t.getValueByField("userNames")) {
                    e.emit(userName, L(orgName));
                  }
              }
            })
        .red(
            "{ orgNames }",
            "userNames2OrgNames(userName)",
            new Reducer() {
              @Override
              public List reduce(Tuple key, Tuple a, Tuple b) {
                System.out.println("userNames2OrgNames reducing: a: " + a + ", b: " + b);
                Set<String> orgNames = new TreeSet<String>();
                orgNames.addAll((List) a.getValue(0));
                orgNames.addAll((List) b.getValue(0));
                return L(new ArrayList(orgNames));
              }
            });

    buildSniffer("userNames2OrgNames", karma);

    cluster.submitTopology("karma", new Config(), karma.createTopology());

    Producer<Long, Message> producer = broker.buildSyncProducer();

    //        Demo.countdown("Adding orgs and users in ", 5);

    //        sendOrgs(producer);
    //        send100Users(producer);

    //        Demo.readEnter("*** Adding acme", 3);
    sendOrg(producer, 1000, "Acme");

    //        Demo.readEnter("*** Adding 10 greggs", 3);
    for (int i = 0; i < 10; i++) {
      //            Demo.readEnter("** Adding gregg " + (i + 1), 1);
      sendUser(producer, 2000 + i, "Gregg", 1000);
    }

    //        Demo.readEnter("*** Changing greggs to seth and assigning to org 1 in", 3);
    Utils.sleep(2000);
    sendOrg(producer, 1, "Kfc");
    for (int i = 0; i < 10; i++) {
      //            Demo.readEnter("** Changing gregg " + (i + 1) + " to seth and Kfc in", 1);
      sendUser(producer, 2000 + i, "Seth", 1);
    }

    //        Demo.readEnter("*** Deleting acme", 3);
    for (int i = 0; i < 10; i++) deleteUser(producer, 2000 + i);
    deleteOrg(producer, 1000);
    deleteOrg(producer, 1);

    Utils.sleep(100000);

    producer.close();
  }
Esempio n. 5
0
 private static void buildSniffer(String srcComponentId, KarmaTopologyBuilder builder) {
   builder
       .setBolt(srcComponentId + "Sniffer", new PrintBolt("### " + srcComponentId))
       .globalGrouping(srcComponentId, "changes");
 }