コード例 #1
0
  public void metrics(Nimbus.Client client, int size, int poll, int total) throws Exception {
    System.out.println(
        "status\ttopologies\ttotalSlots\tslotsUsed\ttotalExecutors\texecutorsWithMetrics\ttime\ttime-diff ms\ttransferred\tthroughput (MB/s)");
    MetricsState state = new MetricsState();
    long pollMs = poll * 1000;
    long now = System.currentTimeMillis();
    state.lastTime = now;
    long startTime = now;
    long cycle = 0;
    long sleepTime;
    long wakeupTime;
    while (metrics(client, size, now, state, "WAITING")) {
      now = System.currentTimeMillis();
      cycle = (now - startTime) / pollMs;
      wakeupTime = startTime + (pollMs * (cycle + 1));
      sleepTime = wakeupTime - now;
      if (sleepTime > 0) {
        Thread.sleep(sleepTime);
      }
      now = System.currentTimeMillis();
    }

    now = System.currentTimeMillis();
    cycle = (now - startTime) / pollMs;
    wakeupTime = startTime + (pollMs * (cycle + 1));
    sleepTime = wakeupTime - now;
    if (sleepTime > 0) {
      Thread.sleep(sleepTime);
    }
    now = System.currentTimeMillis();
    long end = now + (total * 1000);
    do {

      /// one time print addition
      if (printOnce) {
        printExecutorLocation(client);
      }
      printOnce = false;
      ///

      metrics(client, size, now, state, "RUNNING");

      now = System.currentTimeMillis();
      cycle = (now - startTime) / pollMs;
      wakeupTime = startTime + (pollMs * (cycle + 1));
      sleepTime = wakeupTime - now;
      if (sleepTime > 0) {
        Thread.sleep(sleepTime);
      }
      now = System.currentTimeMillis();
    } while (now < end);
  }
コード例 #2
0
  public boolean metrics(
      Nimbus.Client client, int size, long now, MetricsState state, String message)
      throws Exception {
    ClusterSummary summary = client.getClusterInfo();
    long time = now - state.lastTime;
    state.lastTime = now;
    int numSupervisors = summary.get_supervisors_size();
    int totalSlots = 0;
    int totalUsedSlots = 0;

    //////////
    // String namaSupervisor = "";
    for (SupervisorSummary sup : summary.get_supervisors()) {
      totalSlots += sup.get_num_workers();
      totalUsedSlots += sup.get_num_used_workers();
      // namaSupervisor = namaSupervisor + sup.get_host() + ",";
    }
    // System.out.println(namaSupervisor);

    int slotsUsedDiff = totalUsedSlots - state.slotsUsed;
    state.slotsUsed = totalUsedSlots;

    int numTopologies = summary.get_topologies_size();
    long totalTransferred = 0;
    int totalExecutors = 0;
    int executorsWithMetrics = 0;
    for (TopologySummary ts : summary.get_topologies()) {
      String id = ts.get_id();
      TopologyInfo info = client.getTopologyInfo(id);

      //// SOE Addition
      PerftestWriter.print(summary, info, new HashMap<String, Long>());
      ////

      for (ExecutorSummary es : info.get_executors()) {
        ExecutorStats stats = es.get_stats();
        totalExecutors++;
        if (stats != null) {
          Map<String, Map<String, Long>> transferred = stats.get_emitted(); /* .get_transferred();*/
          if (transferred != null) {
            Map<String, Long> e2 = transferred.get(":all-time");
            if (e2 != null) {
              executorsWithMetrics++;
              // The SOL messages are always on the default stream, so just count those
              Long dflt = e2.get("default");
              if (dflt != null) {
                totalTransferred += dflt;
              }
            }
          }
        }
      }
    }
    // long transferredDiff = totalTransferred - state.transferred;
    state.transferred = totalTransferred;
    // double throughput = (transferredDiff == 0 || time == 0) ? 0.0 : (transferredDiff *
    // size)/(1024.0 * 1024.0)/(time/1000.0);
    // System.out.println(message+"\t"+numTopologies+"\t"+totalSlots+"\t"+totalUsedSlots+"\t"+totalExecutors+"\t"+executorsWithMetrics+"\t"+now+"\t"+time+"\t"+transferredDiff+"\t"+throughput);
    System.out.println(
        message
            + ","
            + totalSlots
            + ","
            + totalUsedSlots
            + ","
            + totalExecutors
            + ","
            + executorsWithMetrics
            + ","
            + time);
    if ("WAITING".equals(message)) {
      // System.err.println(" !("+totalUsedSlots+" > 0 && "+slotsUsedDiff+" == 0 &&
      // "+totalExecutors+" > 0 && "+executorsWithMetrics+" >= "+totalExecutors+")");
    }
    return !(totalUsedSlots > 0
        && slotsUsedDiff == 0
        && totalExecutors > 0
        && executorsWithMetrics >= totalExecutors);
  }