@Test
  @InSequence(1)
  public void gaugeCalledWithDefaultValue() {
    assertThat("Gauge is not registered correctly", registry.getGauges(), hasKey(GAUGE_NAME));
    @SuppressWarnings("unchecked")
    Gauge<Long> gauge = registry.getGauges().get(GAUGE_NAME);

    // Make sure that the gauge has the expected value
    assertThat("Gauge value is incorrect", gauge.getValue(), is(equalTo(0L)));
  }
  /**
   * package metrics information for one tracked queue/app only support FairScheduler currently
   *
   * @throws java.io.IOException
   */
  private void printJsonTrack(HttpServletRequest request, HttpServletResponse response)
      throws IOException {
    response.setContentType("text/json");
    response.setStatus(HttpServletResponse.SC_OK);

    StringBuilder sb = new StringBuilder();
    if (schedulerMetrics instanceof FairSchedulerMetrics) {
      String para = request.getParameter("t");
      if (para.startsWith("Job ")) {
        String appId = para.substring("Job ".length());

        sb.append("{");
        sb.append("\"time\": ").append(System.currentTimeMillis()).append(",");
        sb.append("\"appId\": \"").append(appId).append("\"");
        for (String metric : this.schedulerMetrics.getAppTrackedMetrics()) {
          String key = "variable.app." + appId + "." + metric;
          sb.append(",\"").append(metric).append("\": ");
          if (metrics.getGauges().containsKey(key)) {
            double memoryGB =
                Double.parseDouble(metrics.getGauges().get(key).getValue().toString()) / 1024;
            sb.append(memoryGB);
          } else {
            sb.append(-1);
          }
        }
        sb.append("}");

      } else if (para.startsWith("Queue ")) {
        String queueName = para.substring("Queue ".length());
        sb.append("{");
        sb.append("\"time\": ").append(System.currentTimeMillis()).append(",");
        sb.append("\"queueName\": \"").append(queueName).append("\"");
        for (String metric : this.schedulerMetrics.getQueueTrackedMetrics()) {
          String key = "variable.queue." + queueName + "." + metric;
          sb.append(",\"").append(metric).append("\": ");
          if (metrics.getGauges().containsKey(key)) {
            double memoryGB =
                Double.parseDouble(metrics.getGauges().get(key).getValue().toString()) / 1024;
            sb.append(memoryGB);
          } else {
            sb.append(-1);
          }
        }
        sb.append("}");
      }
    }
    String output = sb.toString();
    if (output.isEmpty()) {
      output = "[]";
    }
    response.getWriter().println(output);
    // package result
    ((Request) request).setHandled(true);
  }
예제 #3
0
  @Test
  public void aPrivateGaugeInSuperclass() throws Exception {
    final Gauge<?> metric = registry.getGauges().get(name("gaugeParentPrivate"));

    assertNotNull(metric);
    assertEquals("gaugeParentPrivate", metric.getValue());
  }
 @Test
 public void test() throws Exception {
   Map<String, JobParameter> parameters = new HashMap<String, JobParameter>();
   JobExecution jobExecution = jobLauncherTestUtils.launchJob(new JobParameters(parameters));
   Assert.assertEquals(
       jobExecution.getExitStatus().getExitDescription(),
       BatchStatus.COMPLETED,
       jobExecution.getStatus());
   Map<String, Meter> meters = metricRegistry.getMeters();
   assertThat(meters).hasSize(2);
   assertThat(meters)
       .containsKey("batch.sampleJob.job.metered")
       .containsKey("batch.sampleJob.step.chunkStep.step.metered");
   assertThat(extractProperty("count", Number.class).from(meters.values()))
       .contains(1L)
       .doesNotContain(0L);
   Map<String, Timer> timers = metricRegistry.getTimers();
   assertThat(timers).hasSize(6);
   assertThat(timers)
       .containsKey("batch.sampleJob.job.timed")
       .containsKey("batch.sampleJob.step.chunkStep.chunk.timed")
       .containsKey("batch.sampleJob.step.chunkStep.step.timed")
       .containsKey("batch.sampleJob.step.chunkStep.read.timed")
       .containsKey("batch.sampleJob.step.chunkStep.process.timed")
       .containsKey("batch.sampleJob.step.chunkStep.write.timed");
   assertThat(extractProperty("count", Number.class).from(timers.values()))
       .contains(1L, 3L, 4L)
       .doesNotContain(0L);
   Map<String, Gauge> gauges = metricRegistry.getGauges();
   assertThat(gauges).hasSize(0);
 }
  /**
   * @summary get the worker metrics
   * @return the response object
   */
  @GET
  @Path(GET_METRICS)
  @ReturnType("java.util.SortedMap<String, Long>")
  public Response getMetrics() {
    MetricRegistry metricRegistry = mWorker.getWorkerMetricsSystem().getMetricRegistry();

    // Get all counters.
    Map<String, Counter> counters = metricRegistry.getCounters();

    // Only the gauge for cached blocks is retrieved here, other gauges are statistics of free/used
    // spaces, those statistics can be gotten via other REST apis.
    String blocksCachedProperty =
        CommonUtils.argsToString(
            ".", WorkerContext.getWorkerSource().getName(), WorkerSource.BLOCKS_CACHED);
    @SuppressWarnings("unchecked")
    Gauge<Integer> blocksCached =
        (Gauge<Integer>) metricRegistry.getGauges().get(blocksCachedProperty);

    // Get values of the counters and gauges and put them into a metrics map.
    SortedMap<String, Long> metrics = new TreeMap<>();
    for (Map.Entry<String, Counter> counter : counters.entrySet()) {
      metrics.put(counter.getKey(), counter.getValue().getCount());
    }
    metrics.put(blocksCachedProperty, blocksCached.getValue().longValue());

    return RestUtils.createResponse(metrics);
  }
예제 #6
0
  @Test
  public void aPrivateGauge() throws Exception {
    final Gauge<?> metric =
        registry.getGauges().get(name(InstrumentedWithGauge.class, "gaugePrivate"));

    assertNotNull(metric);
    assertEquals("gaugePrivate", metric.getValue());
  }
예제 #7
0
  @Test
  public void aGaugeAnnotatedMethodWithAbsoluteName() throws Exception {
    instance.doAThingWithAbsoluteName();

    final Gauge metric = registry.getGauges().get(name("absoluteName"));

    assertThat("Guice creates a metric", metric, is(notNullValue()));

    assertThat(
        "Guice creates a gauge with the given value",
        ((Gauge<String>) metric).getValue(),
        is("anotherThingWithAbsoluteName"));
  }
예제 #8
0
  @Test
  public void aGaugeAnnotatedMethod() throws Exception {
    instance.doAThing();

    final Gauge metric = registry.getGauges().get(name(InstrumentedWithGauge.class, "things"));

    assertThat("Guice creates a metric", metric, is(notNullValue()));

    assertThat(
        "Guice creates a gauge with the given value",
        ((Gauge<String>) metric).getValue(),
        is("poop"));
  }
예제 #9
0
  @Test
  public void aGaugeAnnotatedMethodWithDefaultName() throws Exception {
    instance.doAnotherThing();

    final Gauge metric =
        registry.getGauges().get(name(InstrumentedWithGauge.class, "doAnotherThing", GAUGE_SUFFIX));

    assertThat("Guice creates a metric", metric, is(notNullValue()));

    assertThat(
        "Guice creates a gauge with the given value",
        ((Gauge<String>) metric).getValue(),
        is("anotherThing"));
  }
  @Test
  @InSequence(2)
  public void callGaugeAfterSetterCall() throws InterruptedException {
    assertThat("Gauge is not registered correctly", registry.getGauges(), hasKey(GAUGE_NAME));
    @SuppressWarnings("unchecked")
    Gauge<Long> gauge = registry.getGauges().get(GAUGE_NAME);

    // Make sure that the gauge has the default value
    assertThat("Gauge value is incorrect", gauge.getValue(), is(equalTo(0L)));

    // Call the setter method
    long value = 1L + Math.round(Math.random() * (Long.MAX_VALUE - 1L));
    bean.setGauge(value);

    // Assert the gauge returns the cached value
    assertThat("Gauge value is incorrect", gauge.getValue(), is(equalTo(0L)));

    // Wait for two cache timeout periods
    Thread.sleep(2 * 500L);

    // Assert the gauge is refreshed and up-to-date
    assertThat("Gauge value is incorrect", gauge.getValue(), is(equalTo(value)));
  }
예제 #11
0
  public SerialCollector(MetricMonitorRegistry registry) {
    if (registry == null) {
      throw new NullPointerException("registry must not be null");
    }

    final MetricRegistry metricRegistry = registry.getRegistry();
    final SortedMap<String, Gauge> gauges = metricRegistry.getGauges();

    this.heapMax = getLongGauge(gauges, JVM_MEMORY_HEAP_MAX);
    this.heapUsed = getLongGauge(gauges, JVM_MEMORY_HEAP_USED);

    this.heapNonHeapMax = getLongGauge(gauges, JVM_MEMORY_NONHEAP_MAX);
    this.heapNonHeapUsed = getLongGauge(gauges, JVM_MEMORY_NONHEAP_USED);

    this.gcCount = getLongGauge(gauges, JVM_GC_SERIAL_MSC_COUNT);
    this.gcTime = getLongGauge(gauges, JVM_GC_SERIAL_MSC_TIME);
  }
예제 #12
0
 /**
  * Registers a gauge if it has not been registered.
  *
  * @param name the gauge name
  * @param metric the gauge
  * @param <T> the type
  */
 public static synchronized <T> void registerGaugeIfAbsent(String name, Gauge<T> metric) {
   if (!METRIC_REGISTRY.getGauges().containsKey(name)) {
     METRIC_REGISTRY.register(name, metric);
   }
 }
  public String generateRealTimeTrackingMetrics() {
    // JVM
    double jvmFreeMemoryGB, jvmMaxMemoryGB, jvmTotalMemoryGB;
    if (jvmFreeMemoryGauge == null && metrics.getGauges().containsKey("variable.jvm.free.memory")) {
      jvmFreeMemoryGauge = metrics.getGauges().get("variable.jvm.free.memory");
    }
    if (jvmMaxMemoryGauge == null && metrics.getGauges().containsKey("variable.jvm.max.memory")) {
      jvmMaxMemoryGauge = metrics.getGauges().get("variable.jvm.max.memory");
    }
    if (jvmTotalMemoryGauge == null
        && metrics.getGauges().containsKey("variable.jvm.total.memory")) {
      jvmTotalMemoryGauge = metrics.getGauges().get("variable.jvm.total.memory");
    }
    jvmFreeMemoryGB =
        jvmFreeMemoryGauge == null
            ? 0
            : Double.parseDouble(jvmFreeMemoryGauge.getValue().toString()) / 1024 / 1024 / 1024;
    jvmMaxMemoryGB =
        jvmMaxMemoryGauge == null
            ? 0
            : Double.parseDouble(jvmMaxMemoryGauge.getValue().toString()) / 1024 / 1024 / 1024;
    jvmTotalMemoryGB =
        jvmTotalMemoryGauge == null
            ? 0
            : Double.parseDouble(jvmTotalMemoryGauge.getValue().toString()) / 1024 / 1024 / 1024;

    // number of running applications/containers
    String numRunningApps, numRunningContainers;
    if (numRunningAppsGauge == null
        && metrics.getGauges().containsKey("variable.running.application")) {
      numRunningAppsGauge = metrics.getGauges().get("variable.running.application");
    }
    if (numRunningContainersGauge == null
        && metrics.getGauges().containsKey("variable.running.container")) {
      numRunningContainersGauge = metrics.getGauges().get("variable.running.container");
    }
    numRunningApps = numRunningAppsGauge == null ? "0" : numRunningAppsGauge.getValue().toString();
    numRunningContainers =
        numRunningContainersGauge == null ? "0" : numRunningContainersGauge.getValue().toString();

    // cluster available/allocate resource
    double allocatedMemoryGB, allocatedVCoresGB, availableMemoryGB, availableVCoresGB;
    if (allocatedMemoryGauge == null
        && metrics.getGauges().containsKey("variable.cluster.allocated.memory")) {
      allocatedMemoryGauge = metrics.getGauges().get("variable.cluster.allocated.memory");
    }
    if (allocatedVCoresGauge == null
        && metrics.getGauges().containsKey("variable.cluster.allocated.vcores")) {
      allocatedVCoresGauge = metrics.getGauges().get("variable.cluster.allocated.vcores");
    }
    if (availableMemoryGauge == null
        && metrics.getGauges().containsKey("variable.cluster.available.memory")) {
      availableMemoryGauge = metrics.getGauges().get("variable.cluster.available.memory");
    }
    if (availableVCoresGauge == null
        && metrics.getGauges().containsKey("variable.cluster.available.vcores")) {
      availableVCoresGauge = metrics.getGauges().get("variable.cluster.available.vcores");
    }
    allocatedMemoryGB =
        allocatedMemoryGauge == null
            ? 0
            : Double.parseDouble(allocatedMemoryGauge.getValue().toString()) / 1024;
    allocatedVCoresGB =
        allocatedVCoresGauge == null
            ? 0
            : Double.parseDouble(allocatedVCoresGauge.getValue().toString());
    availableMemoryGB =
        availableMemoryGauge == null
            ? 0
            : Double.parseDouble(availableMemoryGauge.getValue().toString()) / 1024;
    availableVCoresGB =
        availableVCoresGauge == null
            ? 0
            : Double.parseDouble(availableVCoresGauge.getValue().toString());

    // scheduler operation
    double allocateTimecost, handleTimecost;
    if (allocateTimecostHistogram == null
        && metrics.getHistograms().containsKey("sampler.scheduler.operation.allocate.timecost")) {
      allocateTimecostHistogram =
          metrics.getHistograms().get("sampler.scheduler.operation.allocate.timecost");
    }
    if (handleTimecostHistogram == null
        && metrics.getHistograms().containsKey("sampler.scheduler.operation.handle.timecost")) {
      handleTimecostHistogram =
          metrics.getHistograms().get("sampler.scheduler.operation.handle.timecost");
    }
    allocateTimecost =
        allocateTimecostHistogram == null
            ? 0.0
            : allocateTimecostHistogram.getSnapshot().getMean() / 1000000;
    handleTimecost =
        handleTimecostHistogram == null
            ? 0.0
            : handleTimecostHistogram.getSnapshot().getMean() / 1000000;
    // various handle operation
    Map<SchedulerEventType, Double> handleOperTimecostMap =
        new HashMap<SchedulerEventType, Double>();
    for (SchedulerEventType e : SchedulerEventType.values()) {
      String key = "sampler.scheduler.operation.handle." + e + ".timecost";
      if (!handleOperTimecostHistogramMap.containsKey(e)
          && metrics.getHistograms().containsKey(key)) {
        handleOperTimecostHistogramMap.put(e, metrics.getHistograms().get(key));
      }
      double timecost =
          handleOperTimecostHistogramMap.containsKey(e)
              ? handleOperTimecostHistogramMap.get(e).getSnapshot().getMean() / 1000000
              : 0;
      handleOperTimecostMap.put(e, timecost);
    }

    // allocated resource for each queue
    Map<String, Double> queueAllocatedMemoryMap = new HashMap<String, Double>();
    Map<String, Long> queueAllocatedVCoresMap = new HashMap<String, Long>();
    for (String queue : wrapper.getQueueSet()) {
      // memory
      String key = "counter.queue." + queue + ".allocated.memory";
      if (!queueAllocatedMemoryCounterMap.containsKey(queue)
          && metrics.getCounters().containsKey(key)) {
        queueAllocatedMemoryCounterMap.put(queue, metrics.getCounters().get(key));
      }
      double queueAllocatedMemoryGB =
          queueAllocatedMemoryCounterMap.containsKey(queue)
              ? queueAllocatedMemoryCounterMap.get(queue).getCount() / 1024.0
              : 0;
      queueAllocatedMemoryMap.put(queue, queueAllocatedMemoryGB);
      // vCores
      key = "counter.queue." + queue + ".allocated.cores";
      if (!queueAllocatedVCoresCounterMap.containsKey(queue)
          && metrics.getCounters().containsKey(key)) {
        queueAllocatedVCoresCounterMap.put(queue, metrics.getCounters().get(key));
      }
      long queueAllocatedVCores =
          queueAllocatedVCoresCounterMap.containsKey(queue)
              ? queueAllocatedVCoresCounterMap.get(queue).getCount()
              : 0;
      queueAllocatedVCoresMap.put(queue, queueAllocatedVCores);
    }

    // package results
    StringBuilder sb = new StringBuilder();
    sb.append("{");
    sb.append("\"time\":")
        .append(System.currentTimeMillis())
        .append(",\"jvm.free.memory\":")
        .append(jvmFreeMemoryGB)
        .append(",\"jvm.max.memory\":")
        .append(jvmMaxMemoryGB)
        .append(",\"jvm.total.memory\":")
        .append(jvmTotalMemoryGB)
        .append(",\"running.applications\":")
        .append(numRunningApps)
        .append(",\"running.containers\":")
        .append(numRunningContainers)
        .append(",\"cluster.allocated.memory\":")
        .append(allocatedMemoryGB)
        .append(",\"cluster.allocated.vcores\":")
        .append(allocatedVCoresGB)
        .append(",\"cluster.available.memory\":")
        .append(availableMemoryGB)
        .append(",\"cluster.available.vcores\":")
        .append(availableVCoresGB);

    for (String queue : wrapper.getQueueSet()) {
      sb.append(",\"queue.")
          .append(queue)
          .append(".allocated.memory\":")
          .append(queueAllocatedMemoryMap.get(queue));
      sb.append(",\"queue.")
          .append(queue)
          .append(".allocated.vcores\":")
          .append(queueAllocatedVCoresMap.get(queue));
    }
    // scheduler allocate & handle
    sb.append(",\"scheduler.allocate.timecost\":").append(allocateTimecost);
    sb.append(",\"scheduler.handle.timecost\":").append(handleTimecost);
    for (SchedulerEventType e : SchedulerEventType.values()) {
      sb.append(",\"scheduler.handle-")
          .append(e)
          .append(".timecost\":")
          .append(handleOperTimecostMap.get(e));
    }
    sb.append("}");
    return sb.toString();
  }