Пример #1
0
  public boolean equals(ClusterSummary that) {
    if (that == null) return false;

    boolean this_present_supervisors = true && this.is_set_supervisors();
    boolean that_present_supervisors = true && that.is_set_supervisors();
    if (this_present_supervisors || that_present_supervisors) {
      if (!(this_present_supervisors && that_present_supervisors)) return false;
      if (!this.supervisors.equals(that.supervisors)) return false;
    }

    boolean this_present_nimbus_uptime_secs = true;
    boolean that_present_nimbus_uptime_secs = true;
    if (this_present_nimbus_uptime_secs || that_present_nimbus_uptime_secs) {
      if (!(this_present_nimbus_uptime_secs && that_present_nimbus_uptime_secs)) return false;
      if (this.nimbus_uptime_secs != that.nimbus_uptime_secs) return false;
    }

    boolean this_present_topologies = true && this.is_set_topologies();
    boolean that_present_topologies = true && that.is_set_topologies();
    if (this_present_topologies || that_present_topologies) {
      if (!(this_present_topologies && that_present_topologies)) return false;
      if (!this.topologies.equals(that.topologies)) return false;
    }

    boolean this_present_version = true && this.is_set_version();
    boolean that_present_version = true && that.is_set_version();
    if (this_present_version || that_present_version) {
      if (!(this_present_version && that_present_version)) return false;
      if (!this.version.equals(that.version)) return false;
    }

    return true;
  }
Пример #2
0
  @Test
  public void testGetTopologySummary() {
    ClusterSummary cs = mock(ClusterSummary.class);
    TopologySummary ts = mock(TopologySummary.class);
    String tsName = "benchmarks";
    String fakeName = "fake";

    when(cs.get_topologies()).thenReturn(Lists.newArrayList(ts));
    when(ts.get_name()).thenReturn(tsName);

    assertThat(MetricsUtils.getTopologySummary(cs, tsName)).isEqualTo(ts);
    assertThat(MetricsUtils.getTopologySummary(cs, fakeName)).isNull();
  }
  @Override
  public void monitor() {
    if (connection.open()) {
      try {
        ClusterSummary summary = connection.getClusterSummary();
        List<SupervisorSummary> supervisors = summary.get_supervisors();
        PlatformSystemPart part = getState().getPlatform();

        aggregate(part, supervisors);
        sendSummaryEvent(part, null, MonitoringManager.DEMO_MSG_INFRASTRUCTURE);
      } catch (TException e) {
        LOGGER.error("Cannot obtain thrift data " + e.getMessage(), e);
      } catch (IllegalStateException e) {
        // monitoring runs longer than topology exists... ignore
      }
    }
  }
Пример #4
0
  public int compareTo(ClusterSummary other) {
    if (!getClass().equals(other.getClass())) {
      return getClass().getName().compareTo(other.getClass().getName());
    }

    int lastComparison = 0;
    ClusterSummary typedOther = (ClusterSummary) other;

    lastComparison =
        Boolean.valueOf(is_set_supervisors()).compareTo(typedOther.is_set_supervisors());
    if (lastComparison != 0) {
      return lastComparison;
    }
    if (is_set_supervisors()) {
      lastComparison =
          org.apache.thrift7.TBaseHelper.compareTo(this.supervisors, typedOther.supervisors);
      if (lastComparison != 0) {
        return lastComparison;
      }
    }
    lastComparison =
        Boolean.valueOf(is_set_nimbus_uptime_secs())
            .compareTo(typedOther.is_set_nimbus_uptime_secs());
    if (lastComparison != 0) {
      return lastComparison;
    }
    if (is_set_nimbus_uptime_secs()) {
      lastComparison =
          org.apache.thrift7.TBaseHelper.compareTo(
              this.nimbus_uptime_secs, typedOther.nimbus_uptime_secs);
      if (lastComparison != 0) {
        return lastComparison;
      }
    }
    lastComparison = Boolean.valueOf(is_set_topologies()).compareTo(typedOther.is_set_topologies());
    if (lastComparison != 0) {
      return lastComparison;
    }
    if (is_set_topologies()) {
      lastComparison =
          org.apache.thrift7.TBaseHelper.compareTo(this.topologies, typedOther.topologies);
      if (lastComparison != 0) {
        return lastComparison;
      }
    }
    lastComparison = Boolean.valueOf(is_set_version()).compareTo(typedOther.is_set_version());
    if (lastComparison != 0) {
      return lastComparison;
    }
    if (is_set_version()) {
      lastComparison = org.apache.thrift7.TBaseHelper.compareTo(this.version, typedOther.version);
      if (lastComparison != 0) {
        return lastComparison;
      }
    }
    return 0;
  }
Пример #5
0
 /** Performs a deep copy on <i>other</i>. */
 public ClusterSummary(ClusterSummary other) {
   __isset_bit_vector.clear();
   __isset_bit_vector.or(other.__isset_bit_vector);
   if (other.is_set_supervisors()) {
     List<SupervisorSummary> __this__supervisors = new ArrayList<SupervisorSummary>();
     for (SupervisorSummary other_element : other.supervisors) {
       __this__supervisors.add(new SupervisorSummary(other_element));
     }
     this.supervisors = __this__supervisors;
   }
   this.nimbus_uptime_secs = other.nimbus_uptime_secs;
   if (other.is_set_topologies()) {
     List<TopologySummary> __this__topologies = new ArrayList<TopologySummary>();
     for (TopologySummary other_element : other.topologies) {
       __this__topologies.add(new TopologySummary(other_element));
     }
     this.topologies = __this__topologies;
   }
   if (other.is_set_version()) {
     this.version = other.version;
   }
 }
  private void printExecutorLocation(Client client) throws Exception {
    ClusterSummary summary = client.getClusterInfo();
    StringBuilder executorBuilder = new StringBuilder();

    for (TopologySummary ts : summary.get_topologies()) {
      String id = ts.get_id();
      TopologyInfo info = client.getTopologyInfo(id);

      executorBuilder.append("~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n");
      for (ExecutorSummary es : info.get_executors()) {
        executorBuilder.append(
            es.get_executor_info().get_task_start()
                + ","
                + es.get_component_id()
                + ","
                + es.get_host()
                + "\n");
      }
      executorBuilder.append("~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n");
    }

    System.out.println(executorBuilder.toString());
  }
  public boolean metrics(
      Nimbus.Client client, int size, long now, MetricsState state, String message)
      throws Exception {
    ClusterSummary summary = client.getClusterInfo();
    long time = now - state.lastTime;
    state.lastTime = now;
    int numSupervisors = summary.get_supervisors_size();
    int totalSlots = 0;
    int totalUsedSlots = 0;

    //////////
    // String namaSupervisor = "";
    for (SupervisorSummary sup : summary.get_supervisors()) {
      totalSlots += sup.get_num_workers();
      totalUsedSlots += sup.get_num_used_workers();
      // namaSupervisor = namaSupervisor + sup.get_host() + ",";
    }
    // System.out.println(namaSupervisor);

    int slotsUsedDiff = totalUsedSlots - state.slotsUsed;
    state.slotsUsed = totalUsedSlots;

    int numTopologies = summary.get_topologies_size();
    long totalTransferred = 0;
    int totalExecutors = 0;
    int executorsWithMetrics = 0;
    for (TopologySummary ts : summary.get_topologies()) {
      String id = ts.get_id();
      TopologyInfo info = client.getTopologyInfo(id);

      //// SOE Addition
      PerftestWriter.print(summary, info, new HashMap<String, Long>());
      ////

      for (ExecutorSummary es : info.get_executors()) {
        ExecutorStats stats = es.get_stats();
        totalExecutors++;
        if (stats != null) {
          Map<String, Map<String, Long>> transferred = stats.get_emitted(); /* .get_transferred();*/
          if (transferred != null) {
            Map<String, Long> e2 = transferred.get(":all-time");
            if (e2 != null) {
              executorsWithMetrics++;
              // The SOL messages are always on the default stream, so just count those
              Long dflt = e2.get("default");
              if (dflt != null) {
                totalTransferred += dflt;
              }
            }
          }
        }
      }
    }
    // long transferredDiff = totalTransferred - state.transferred;
    state.transferred = totalTransferred;
    // double throughput = (transferredDiff == 0 || time == 0) ? 0.0 : (transferredDiff *
    // size)/(1024.0 * 1024.0)/(time/1000.0);
    // System.out.println(message+"\t"+numTopologies+"\t"+totalSlots+"\t"+totalUsedSlots+"\t"+totalExecutors+"\t"+executorsWithMetrics+"\t"+now+"\t"+time+"\t"+transferredDiff+"\t"+throughput);
    System.out.println(
        message
            + ","
            + totalSlots
            + ","
            + totalUsedSlots
            + ","
            + totalExecutors
            + ","
            + executorsWithMetrics
            + ","
            + time);
    if ("WAITING".equals(message)) {
      // System.err.println(" !("+totalUsedSlots+" > 0 && "+slotsUsedDiff+" == 0 &&
      // "+totalExecutors+" > 0 && "+executorsWithMetrics+" >= "+totalExecutors+")");
    }
    return !(totalUsedSlots > 0
        && slotsUsedDiff == 0
        && totalExecutors > 0
        && executorsWithMetrics >= totalExecutors);
  }