@Override public void execute(Tuple tuple) { LOG.info( "TopologyKiller: Received EOF message from componentId: " + tuple.getSourceComponent() + ", taskId: " + tuple.getSourceTask()); _numberRegisteredTasks--; LOG.info("TopologyKiller: " + _numberRegisteredTasks + " remaining"); if (_numberRegisteredTasks == 0) { LOG.info("TopologyKiller: Received EOF from all the registered tasks. Killing cluster..."); // EVENT WHEN ALL THE SPOUTS FINISHED EMITTING AND ACKED or // WHEN ALL THE TASKS FROM THE LAST COMPONENTS SENT EOF SIGNAL // Instrument all the components for which printOut is set to dump their results _collector.emit( SystemParameters.DUMP_RESULTS_STREAM, new Values(SystemParameters.DUMP_RESULTS)); long timeout = SystemParameters.LOCAL_SLEEP_BEFORE_KILL_MILLIS; if (SystemParameters.getBoolean(_conf, "DIP_DISTRIBUTED")) { // write down statistics (the same which is shown in Storm UI web interface) StormWrapper.writeStats(_conf); timeout = SystemParameters.CLUSTER_SLEEP_BEFORE_KILL_MILLIS; } if (SystemParameters.getBoolean(_conf, "DIP_KILL_AT_THE_END")) { /* Give enough time to dump the results * We couldn't use Storm ack mechanism for dumping results, * since our final result might be on Spout (StormDataSource). * Spouts cannot send ack to other spout (TopologyKiller spout). * They use EOF boolean to indicate when done. */ Utils.sleep(timeout); StormWrapper.killExecution(_conf); } } }
// String[] sizes: {"1G", "2G", "4G", ...} public Map createConfig(String parserConfPath) { Map map = SystemParameters.fileToMap(parserConfPath); if (!SystemParameters.getBoolean(map, "DIP_ACK_EVERY_TUPLE")) { // we don't ack after each tuple is sent, // so we don't need any node to be dedicated for acking CLUSTER_ACKERS = 0; LOCAL_ACKERS = 0; } if (SystemParameters.getBoolean(map, "DIP_DISTRIBUTED")) { // default value is already set, but for scheduling we might need to change that // SystemParameters.putInMap(map, "DIP_NUM_WORKERS", CLUSTER_WORKERS); SystemParameters.putInMap(map, "DIP_NUM_ACKERS", CLUSTER_ACKERS); } else { SystemParameters.putInMap(map, "DIP_NUM_ACKERS", LOCAL_ACKERS); } String dbSize = SystemParameters.getString(map, "DIP_DB_SIZE") + "G"; String dataRoot = SystemParameters.getString(map, "DIP_DATA_ROOT"); String dataPath = dataRoot + "/" + dbSize + "/"; SystemParameters.putInMap(map, "DIP_DATA_PATH", dataPath); return map; }
private Optimizer pickOptimizer(Map map) { String optStr = SystemParameters.getString(map, "DIP_OPTIMIZER_TYPE"); System.out.println("Selected optimizer: " + optStr); if ("INDEX_SIMPLE".equalsIgnoreCase(optStr)) { // Simple optimizer provides lefty plans return new IndexSimpleOptimizer(map); } else if ("INDEX_RULE_BUSHY".equalsIgnoreCase(optStr)) { return new IndexRuleOptimizer(map); } else if ("NAME_MANUAL_PAR_LEFTY".equalsIgnoreCase(optStr)) { return new NameManualParOptimizer(map); } else if ("NAME_MANUAL_COST_LEFTY".equalsIgnoreCase(optStr)) { return new NameManualOptimizer(map); } else if ("NAME_RULE_LEFTY".equalsIgnoreCase(optStr)) { return new NameRuleOptimizer(map); } else if ("NAME_COST_LEFTY".equalsIgnoreCase(optStr)) { return new NameCostOptimizer(map); } throw new RuntimeException("Unknown " + optStr + " optimizer!"); }