private void printExecutorLocation(Client client) throws Exception { ClusterSummary summary = client.getClusterInfo(); StringBuilder executorBuilder = new StringBuilder(); for (TopologySummary ts : summary.get_topologies()) { String id = ts.get_id(); TopologyInfo info = client.getTopologyInfo(id); executorBuilder.append("~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"); for (ExecutorSummary es : info.get_executors()) { executorBuilder.append( es.get_executor_info().get_task_start() + "," + es.get_component_id() + "," + es.get_host() + "\n"); } executorBuilder.append("~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"); } System.out.println(executorBuilder.toString()); }
public boolean metrics(Client client, long now, MetricsState state, String message) throws Exception { ClusterSummary summary = client.getClusterInfo(); long time = now - state.lastTime; state.lastTime = now; int numSupervisors = summary.get_supervisors_size(); int totalSlots = 0; int totalUsedSlots = 0; ////////// // String namaSupervisor = ""; for (SupervisorSummary sup : summary.get_supervisors()) { totalSlots += sup.get_num_workers(); totalUsedSlots += sup.get_num_used_workers(); // namaSupervisor = namaSupervisor + sup.get_host() + ","; } // System.out.println(namaSupervisor); int slotsUsedDiff = totalUsedSlots - state.slotsUsed; state.slotsUsed = totalUsedSlots; int numTopologies = summary.get_topologies_size(); long totalTransferred = 0; int totalExecutors = 0; int executorsWithMetrics = 0; for (TopologySummary ts : summary.get_topologies()) { String id = ts.get_id(); TopologyInfo info = client.getTopologyInfo(id); //// SOE Addition PerftestWriter.print(summary, info, new HashMap<String, Long>()); //// for (ExecutorSummary es : info.get_executors()) { ExecutorStats stats = es.get_stats(); totalExecutors++; if (stats != null) { Map<String, Map<String, Long>> transferred = stats.get_emitted(); /* .get_transferred();*/ if (transferred != null) { Map<String, Long> e2 = transferred.get(":all-time"); if (e2 != null) { executorsWithMetrics++; // The SOL messages are always on the default stream, so just count those Long dflt = e2.get("default"); if (dflt != null) { totalTransferred += dflt; } } } } } } // long transferredDiff = totalTransferred - state.transferred; state.transferred = totalTransferred; // double throughput = (transferredDiff == 0 || time == 0) ? 0.0 : (transferredDiff * // size)/(1024.0 * 1024.0)/(time/1000.0); // System.out.println(message+"\t"+numTopologies+"\t"+totalSlots+"\t"+totalUsedSlots+"\t"+totalExecutors+"\t"+executorsWithMetrics+"\t"+now+"\t"+time+"\t"+transferredDiff+"\t"+throughput); System.out.println( message + "," + totalSlots + "," + totalUsedSlots + "," + totalExecutors + "," + executorsWithMetrics + "," + time + ",NOLIMIT"); if ("WAITING".equals(message)) { // System.err.println(" !("+totalUsedSlots+" > 0 && "+slotsUsedDiff+" == 0 && // "+totalExecutors+" > 0 && "+executorsWithMetrics+" >= "+totalExecutors+")"); } return !(totalUsedSlots > 0 && slotsUsedDiff == 0 && totalExecutors > 0 && executorsWithMetrics >= totalExecutors); }