@Test
  @InSequence(1)
  public void countedMethodNotCalledYet() {
    assertThat("Counter is not registered correctly", registry.getCounters(), hasKey(COUNTER_NAME));
    Counter counter = registry.getCounters().get(COUNTER_NAME);

    // Make sure that the counter hasn't been called yet
    assertThat("Counter count is incorrect", counter.getCount(), is(equalTo(COUNTER_COUNT.get())));
  }
  @Test
  @InSequence(2)
  public void countedMethodNotCalledYet(
      @Metric(name = "countedMethod", absolute = true) Counter instance) {
    assertThat("Counter is not registered correctly", registry.getCounters(), hasKey(COUNTER_NAME));
    Counter counter = registry.getCounters().get(COUNTER_NAME);

    // Make sure that the counter registered and the bean instance are the same
    assertThat("Counter and bean instance are not equal", instance, is(equalTo(counter)));
  }
  /**
   * @summary get the worker metrics
   * @return the response object
   */
  @GET
  @Path(GET_METRICS)
  @ReturnType("java.util.SortedMap<String, Long>")
  public Response getMetrics() {
    MetricRegistry metricRegistry = mWorker.getWorkerMetricsSystem().getMetricRegistry();

    // Get all counters.
    Map<String, Counter> counters = metricRegistry.getCounters();

    // Only the gauge for cached blocks is retrieved here, other gauges are statistics of free/used
    // spaces, those statistics can be gotten via other REST apis.
    String blocksCachedProperty =
        CommonUtils.argsToString(
            ".", WorkerContext.getWorkerSource().getName(), WorkerSource.BLOCKS_CACHED);
    @SuppressWarnings("unchecked")
    Gauge<Integer> blocksCached =
        (Gauge<Integer>) metricRegistry.getGauges().get(blocksCachedProperty);

    // Get values of the counters and gauges and put them into a metrics map.
    SortedMap<String, Long> metrics = new TreeMap<>();
    for (Map.Entry<String, Counter> counter : counters.entrySet()) {
      metrics.put(counter.getKey(), counter.getValue().getCount());
    }
    metrics.put(blocksCachedProperty, blocksCached.getValue().longValue());

    return RestUtils.createResponse(metrics);
  }
 private void updateQueueMetrics(String queue, int releasedMemory, int releasedVCores) {
   // update queue counters
   SortedMap<String, Counter> counterMap = metrics.getCounters();
   if (releasedMemory != 0) {
     String name = "counter.queue." + queue + ".allocated.memory";
     if (!counterMap.containsKey(name)) {
       metrics.counter(name);
       counterMap = metrics.getCounters();
     }
     counterMap.get(name).inc(-releasedMemory);
   }
   if (releasedVCores != 0) {
     String name = "counter.queue." + queue + ".allocated.cores";
     if (!counterMap.containsKey(name)) {
       metrics.counter(name);
       counterMap = metrics.getCounters();
     }
     counterMap.get(name).inc(-releasedVCores);
   }
 }
  @Test
  @InSequence(4)
  public void removeCounterFromRegistry() {
    assertThat("Counter is not registered correctly", registry.getCounters(), hasKey(COUNTER_NAME));
    Counter counter = registry.getCounters().get(COUNTER_NAME);

    // Remove the counter from metrics registry
    registry.remove(COUNTER_NAME);

    try {
      // Call the counted method and assert an exception is thrown
      bean.countedMethod(
          new Callable<Long>() {
            @Override
            public Long call() throws Exception {
              return null;
            }
          });
    } catch (Exception cause) {
      assertThat(cause, is(instanceOf(IllegalStateException.class)));
      assertThat(
          cause.getMessage(),
          is(
              equalTo(
                  "No counter with name ["
                      + COUNTER_NAME
                      + "] found in registry ["
                      + registry
                      + "]")));
      // Make sure that the counter hasn't been called
      assertThat(
          "Counter count is incorrect", counter.getCount(), is(equalTo(COUNTER_COUNT.get())));
      return;
    }

    fail("No exception has been re-thrown!");
  }
Esempio n. 6
0
 /** Resets all the counters to 0 for testing. */
 public static void resetAllCounters() {
   for (Map.Entry<String, Counter> entry : METRIC_REGISTRY.getCounters().entrySet()) {
     entry.getValue().dec(entry.getValue().getCount());
   }
 }
  @Test
  @InSequence(3)
  public void callCountedMethodOnce() throws InterruptedException, TimeoutException {
    assertThat("Counter is not registered correctly", registry.getCounters(), hasKey(COUNTER_NAME));
    Counter counter = registry.getCounters().get(COUNTER_NAME);

    // Call the counted method, block and assert it's been counted
    final Exchanger<Long> exchanger = new Exchanger<>();
    Thread thread =
        new Thread(
            new Runnable() {
              @Override
              public void run() {
                try {
                  exchanger.exchange(
                      bean.countedMethod(
                          new Callable<Long>() {
                            @Override
                            public Long call() throws Exception {
                              exchanger.exchange(0L);
                              return exchanger.exchange(0L);
                            }
                          }));
                } catch (InterruptedException cause) {
                  throw new RuntimeException(cause);
                }
              }
            });
    final AtomicInteger uncaught = new AtomicInteger();
    thread.setUncaughtExceptionHandler(
        new Thread.UncaughtExceptionHandler() {
          @Override
          public void uncaughtException(Thread t, Throwable e) {
            uncaught.incrementAndGet();
          }
        });
    thread.start();

    // Wait until the method is executing and make sure that the counter has been incremented
    exchanger.exchange(0L, 5L, TimeUnit.SECONDS);
    assertThat(
        "Counter count is incorrect",
        counter.getCount(),
        is(equalTo(COUNTER_COUNT.incrementAndGet())));

    // Exchange the result and unblock the method execution
    Long random = 1 + Math.round(Math.random() * (Long.MAX_VALUE - 1));
    exchanger.exchange(random, 5L, TimeUnit.SECONDS);

    // Wait until the method has returned
    assertThat(
        "Counted method return value is incorrect", exchanger.exchange(0L), is(equalTo(random)));

    // Then make sure that the counter has been decremented
    assertThat(
        "Counter count is incorrect",
        counter.getCount(),
        is(equalTo(COUNTER_COUNT.decrementAndGet())));

    // Finally make sure calling thread is returns correctly
    thread.join();
    assertThat("Exception thrown in method call thread", uncaught.get(), is(equalTo(0)));
  }
  public String generateRealTimeTrackingMetrics() {
    // JVM
    double jvmFreeMemoryGB, jvmMaxMemoryGB, jvmTotalMemoryGB;
    if (jvmFreeMemoryGauge == null && metrics.getGauges().containsKey("variable.jvm.free.memory")) {
      jvmFreeMemoryGauge = metrics.getGauges().get("variable.jvm.free.memory");
    }
    if (jvmMaxMemoryGauge == null && metrics.getGauges().containsKey("variable.jvm.max.memory")) {
      jvmMaxMemoryGauge = metrics.getGauges().get("variable.jvm.max.memory");
    }
    if (jvmTotalMemoryGauge == null
        && metrics.getGauges().containsKey("variable.jvm.total.memory")) {
      jvmTotalMemoryGauge = metrics.getGauges().get("variable.jvm.total.memory");
    }
    jvmFreeMemoryGB =
        jvmFreeMemoryGauge == null
            ? 0
            : Double.parseDouble(jvmFreeMemoryGauge.getValue().toString()) / 1024 / 1024 / 1024;
    jvmMaxMemoryGB =
        jvmMaxMemoryGauge == null
            ? 0
            : Double.parseDouble(jvmMaxMemoryGauge.getValue().toString()) / 1024 / 1024 / 1024;
    jvmTotalMemoryGB =
        jvmTotalMemoryGauge == null
            ? 0
            : Double.parseDouble(jvmTotalMemoryGauge.getValue().toString()) / 1024 / 1024 / 1024;

    // number of running applications/containers
    String numRunningApps, numRunningContainers;
    if (numRunningAppsGauge == null
        && metrics.getGauges().containsKey("variable.running.application")) {
      numRunningAppsGauge = metrics.getGauges().get("variable.running.application");
    }
    if (numRunningContainersGauge == null
        && metrics.getGauges().containsKey("variable.running.container")) {
      numRunningContainersGauge = metrics.getGauges().get("variable.running.container");
    }
    numRunningApps = numRunningAppsGauge == null ? "0" : numRunningAppsGauge.getValue().toString();
    numRunningContainers =
        numRunningContainersGauge == null ? "0" : numRunningContainersGauge.getValue().toString();

    // cluster available/allocate resource
    double allocatedMemoryGB, allocatedVCoresGB, availableMemoryGB, availableVCoresGB;
    if (allocatedMemoryGauge == null
        && metrics.getGauges().containsKey("variable.cluster.allocated.memory")) {
      allocatedMemoryGauge = metrics.getGauges().get("variable.cluster.allocated.memory");
    }
    if (allocatedVCoresGauge == null
        && metrics.getGauges().containsKey("variable.cluster.allocated.vcores")) {
      allocatedVCoresGauge = metrics.getGauges().get("variable.cluster.allocated.vcores");
    }
    if (availableMemoryGauge == null
        && metrics.getGauges().containsKey("variable.cluster.available.memory")) {
      availableMemoryGauge = metrics.getGauges().get("variable.cluster.available.memory");
    }
    if (availableVCoresGauge == null
        && metrics.getGauges().containsKey("variable.cluster.available.vcores")) {
      availableVCoresGauge = metrics.getGauges().get("variable.cluster.available.vcores");
    }
    allocatedMemoryGB =
        allocatedMemoryGauge == null
            ? 0
            : Double.parseDouble(allocatedMemoryGauge.getValue().toString()) / 1024;
    allocatedVCoresGB =
        allocatedVCoresGauge == null
            ? 0
            : Double.parseDouble(allocatedVCoresGauge.getValue().toString());
    availableMemoryGB =
        availableMemoryGauge == null
            ? 0
            : Double.parseDouble(availableMemoryGauge.getValue().toString()) / 1024;
    availableVCoresGB =
        availableVCoresGauge == null
            ? 0
            : Double.parseDouble(availableVCoresGauge.getValue().toString());

    // scheduler operation
    double allocateTimecost, handleTimecost;
    if (allocateTimecostHistogram == null
        && metrics.getHistograms().containsKey("sampler.scheduler.operation.allocate.timecost")) {
      allocateTimecostHistogram =
          metrics.getHistograms().get("sampler.scheduler.operation.allocate.timecost");
    }
    if (handleTimecostHistogram == null
        && metrics.getHistograms().containsKey("sampler.scheduler.operation.handle.timecost")) {
      handleTimecostHistogram =
          metrics.getHistograms().get("sampler.scheduler.operation.handle.timecost");
    }
    allocateTimecost =
        allocateTimecostHistogram == null
            ? 0.0
            : allocateTimecostHistogram.getSnapshot().getMean() / 1000000;
    handleTimecost =
        handleTimecostHistogram == null
            ? 0.0
            : handleTimecostHistogram.getSnapshot().getMean() / 1000000;
    // various handle operation
    Map<SchedulerEventType, Double> handleOperTimecostMap =
        new HashMap<SchedulerEventType, Double>();
    for (SchedulerEventType e : SchedulerEventType.values()) {
      String key = "sampler.scheduler.operation.handle." + e + ".timecost";
      if (!handleOperTimecostHistogramMap.containsKey(e)
          && metrics.getHistograms().containsKey(key)) {
        handleOperTimecostHistogramMap.put(e, metrics.getHistograms().get(key));
      }
      double timecost =
          handleOperTimecostHistogramMap.containsKey(e)
              ? handleOperTimecostHistogramMap.get(e).getSnapshot().getMean() / 1000000
              : 0;
      handleOperTimecostMap.put(e, timecost);
    }

    // allocated resource for each queue
    Map<String, Double> queueAllocatedMemoryMap = new HashMap<String, Double>();
    Map<String, Long> queueAllocatedVCoresMap = new HashMap<String, Long>();
    for (String queue : wrapper.getQueueSet()) {
      // memory
      String key = "counter.queue." + queue + ".allocated.memory";
      if (!queueAllocatedMemoryCounterMap.containsKey(queue)
          && metrics.getCounters().containsKey(key)) {
        queueAllocatedMemoryCounterMap.put(queue, metrics.getCounters().get(key));
      }
      double queueAllocatedMemoryGB =
          queueAllocatedMemoryCounterMap.containsKey(queue)
              ? queueAllocatedMemoryCounterMap.get(queue).getCount() / 1024.0
              : 0;
      queueAllocatedMemoryMap.put(queue, queueAllocatedMemoryGB);
      // vCores
      key = "counter.queue." + queue + ".allocated.cores";
      if (!queueAllocatedVCoresCounterMap.containsKey(queue)
          && metrics.getCounters().containsKey(key)) {
        queueAllocatedVCoresCounterMap.put(queue, metrics.getCounters().get(key));
      }
      long queueAllocatedVCores =
          queueAllocatedVCoresCounterMap.containsKey(queue)
              ? queueAllocatedVCoresCounterMap.get(queue).getCount()
              : 0;
      queueAllocatedVCoresMap.put(queue, queueAllocatedVCores);
    }

    // package results
    StringBuilder sb = new StringBuilder();
    sb.append("{");
    sb.append("\"time\":")
        .append(System.currentTimeMillis())
        .append(",\"jvm.free.memory\":")
        .append(jvmFreeMemoryGB)
        .append(",\"jvm.max.memory\":")
        .append(jvmMaxMemoryGB)
        .append(",\"jvm.total.memory\":")
        .append(jvmTotalMemoryGB)
        .append(",\"running.applications\":")
        .append(numRunningApps)
        .append(",\"running.containers\":")
        .append(numRunningContainers)
        .append(",\"cluster.allocated.memory\":")
        .append(allocatedMemoryGB)
        .append(",\"cluster.allocated.vcores\":")
        .append(allocatedVCoresGB)
        .append(",\"cluster.available.memory\":")
        .append(availableMemoryGB)
        .append(",\"cluster.available.vcores\":")
        .append(availableVCoresGB);

    for (String queue : wrapper.getQueueSet()) {
      sb.append(",\"queue.")
          .append(queue)
          .append(".allocated.memory\":")
          .append(queueAllocatedMemoryMap.get(queue));
      sb.append(",\"queue.")
          .append(queue)
          .append(".allocated.vcores\":")
          .append(queueAllocatedVCoresMap.get(queue));
    }
    // scheduler allocate & handle
    sb.append(",\"scheduler.allocate.timecost\":").append(allocateTimecost);
    sb.append(",\"scheduler.handle.timecost\":").append(handleTimecost);
    for (SchedulerEventType e : SchedulerEventType.values()) {
      sb.append(",\"scheduler.handle-")
          .append(e)
          .append(".timecost\":")
          .append(handleOperTimecostMap.get(e));
    }
    sb.append("}");
    return sb.toString();
  }
  private void updateQueueWithAllocateRequest(
      Allocation allocation,
      ApplicationAttemptId attemptId,
      List<ResourceRequest> resourceRequests,
      List<ContainerId> containerIds)
      throws IOException {
    // update queue information
    Resource pendingResource = Resources.createResource(0, 0);
    Resource allocatedResource = Resources.createResource(0, 0);
    String queueName = appQueueMap.get(attemptId.getApplicationId());
    // container requested
    for (ResourceRequest request : resourceRequests) {
      if (request.getResourceName().equals(ResourceRequest.ANY)) {
        Resources.addTo(
            pendingResource,
            Resources.multiply(request.getCapability(), request.getNumContainers()));
      }
    }
    // container allocated
    for (Container container : allocation.getContainers()) {
      Resources.addTo(allocatedResource, container.getResource());
      Resources.subtractFrom(pendingResource, container.getResource());
    }
    // container released from AM
    SchedulerAppReport report = scheduler.getSchedulerAppInfo(attemptId);
    for (ContainerId containerId : containerIds) {
      Container container = null;
      for (RMContainer c : report.getLiveContainers()) {
        if (c.getContainerId().equals(containerId)) {
          container = c.getContainer();
          break;
        }
      }
      if (container != null) {
        // released allocated containers
        Resources.subtractFrom(allocatedResource, container.getResource());
      } else {
        for (RMContainer c : report.getReservedContainers()) {
          if (c.getContainerId().equals(containerId)) {
            container = c.getContainer();
            break;
          }
        }
        if (container != null) {
          // released reserved containers
          Resources.subtractFrom(pendingResource, container.getResource());
        }
      }
    }
    // containers released/preemption from scheduler
    Set<ContainerId> preemptionContainers = new HashSet<ContainerId>();
    if (allocation.getContainerPreemptions() != null) {
      preemptionContainers.addAll(allocation.getContainerPreemptions());
    }
    if (allocation.getStrictContainerPreemptions() != null) {
      preemptionContainers.addAll(allocation.getStrictContainerPreemptions());
    }
    if (!preemptionContainers.isEmpty()) {
      for (ContainerId containerId : preemptionContainers) {
        if (!preemptionContainerMap.containsKey(containerId)) {
          Container container = null;
          for (RMContainer c : report.getLiveContainers()) {
            if (c.getContainerId().equals(containerId)) {
              container = c.getContainer();
              break;
            }
          }
          if (container != null) {
            preemptionContainerMap.put(containerId, container.getResource());
          }
        }
      }
    }

    // update metrics
    SortedMap<String, Counter> counterMap = metrics.getCounters();
    String names[] =
        new String[] {
          "counter.queue." + queueName + ".pending.memory",
          "counter.queue." + queueName + ".pending.cores",
          "counter.queue." + queueName + ".allocated.memory",
          "counter.queue." + queueName + ".allocated.cores"
        };
    int values[] =
        new int[] {
          pendingResource.getMemory(),
          pendingResource.getVirtualCores(),
          allocatedResource.getMemory(),
          allocatedResource.getVirtualCores()
        };
    for (int i = names.length - 1; i >= 0; i--) {
      if (!counterMap.containsKey(names[i])) {
        metrics.counter(names[i]);
        counterMap = metrics.getCounters();
      }
      counterMap.get(names[i]).inc(values[i]);
    }

    queueLock.lock();
    try {
      if (!schedulerMetrics.isTracked(queueName)) {
        schedulerMetrics.trackQueue(queueName);
      }
    } finally {
      queueLock.unlock();
    }
  }