private void registerSchedulerMetrics() {
   samplerLock.lock();
   try {
     // counters for scheduler operations
     schedulerAllocateCounter = metrics.counter("counter.scheduler.operation.allocate");
     schedulerHandleCounter = metrics.counter("counter.scheduler.operation.handle");
     schedulerHandleCounterMap = new HashMap<SchedulerEventType, Counter>();
     for (SchedulerEventType e : SchedulerEventType.values()) {
       Counter counter = metrics.counter("counter.scheduler.operation.handle." + e);
       schedulerHandleCounterMap.put(e, counter);
     }
     // timers for scheduler operations
     int timeWindowSize =
         conf.getInt(
             SLSConfiguration.METRICS_TIMER_WINDOW_SIZE,
             SLSConfiguration.METRICS_TIMER_WINDOW_SIZE_DEFAULT);
     schedulerAllocateTimer = new Timer(new SlidingWindowReservoir(timeWindowSize));
     schedulerHandleTimer = new Timer(new SlidingWindowReservoir(timeWindowSize));
     schedulerHandleTimerMap = new HashMap<SchedulerEventType, Timer>();
     for (SchedulerEventType e : SchedulerEventType.values()) {
       Timer timer = new Timer(new SlidingWindowReservoir(timeWindowSize));
       schedulerHandleTimerMap.put(e, timer);
     }
     // histogram for scheduler operations (Samplers)
     schedulerHistogramList = new ArrayList<Histogram>();
     histogramTimerMap = new HashMap<Histogram, Timer>();
     Histogram schedulerAllocateHistogram =
         new Histogram(new SlidingWindowReservoir(SAMPLING_SIZE));
     metrics.register("sampler.scheduler.operation.allocate.timecost", schedulerAllocateHistogram);
     schedulerHistogramList.add(schedulerAllocateHistogram);
     histogramTimerMap.put(schedulerAllocateHistogram, schedulerAllocateTimer);
     Histogram schedulerHandleHistogram = new Histogram(new SlidingWindowReservoir(SAMPLING_SIZE));
     metrics.register("sampler.scheduler.operation.handle.timecost", schedulerHandleHistogram);
     schedulerHistogramList.add(schedulerHandleHistogram);
     histogramTimerMap.put(schedulerHandleHistogram, schedulerHandleTimer);
     for (SchedulerEventType e : SchedulerEventType.values()) {
       Histogram histogram = new Histogram(new SlidingWindowReservoir(SAMPLING_SIZE));
       metrics.register("sampler.scheduler.operation.handle." + e + ".timecost", histogram);
       schedulerHistogramList.add(histogram);
       histogramTimerMap.put(histogram, schedulerHandleTimerMap.get(e));
     }
   } finally {
     samplerLock.unlock();
   }
 }
  public String generateRealTimeTrackingMetrics() {
    // JVM
    double jvmFreeMemoryGB, jvmMaxMemoryGB, jvmTotalMemoryGB;
    if (jvmFreeMemoryGauge == null && metrics.getGauges().containsKey("variable.jvm.free.memory")) {
      jvmFreeMemoryGauge = metrics.getGauges().get("variable.jvm.free.memory");
    }
    if (jvmMaxMemoryGauge == null && metrics.getGauges().containsKey("variable.jvm.max.memory")) {
      jvmMaxMemoryGauge = metrics.getGauges().get("variable.jvm.max.memory");
    }
    if (jvmTotalMemoryGauge == null
        && metrics.getGauges().containsKey("variable.jvm.total.memory")) {
      jvmTotalMemoryGauge = metrics.getGauges().get("variable.jvm.total.memory");
    }
    jvmFreeMemoryGB =
        jvmFreeMemoryGauge == null
            ? 0
            : Double.parseDouble(jvmFreeMemoryGauge.getValue().toString()) / 1024 / 1024 / 1024;
    jvmMaxMemoryGB =
        jvmMaxMemoryGauge == null
            ? 0
            : Double.parseDouble(jvmMaxMemoryGauge.getValue().toString()) / 1024 / 1024 / 1024;
    jvmTotalMemoryGB =
        jvmTotalMemoryGauge == null
            ? 0
            : Double.parseDouble(jvmTotalMemoryGauge.getValue().toString()) / 1024 / 1024 / 1024;

    // number of running applications/containers
    String numRunningApps, numRunningContainers;
    if (numRunningAppsGauge == null
        && metrics.getGauges().containsKey("variable.running.application")) {
      numRunningAppsGauge = metrics.getGauges().get("variable.running.application");
    }
    if (numRunningContainersGauge == null
        && metrics.getGauges().containsKey("variable.running.container")) {
      numRunningContainersGauge = metrics.getGauges().get("variable.running.container");
    }
    numRunningApps = numRunningAppsGauge == null ? "0" : numRunningAppsGauge.getValue().toString();
    numRunningContainers =
        numRunningContainersGauge == null ? "0" : numRunningContainersGauge.getValue().toString();

    // cluster available/allocate resource
    double allocatedMemoryGB, allocatedVCoresGB, availableMemoryGB, availableVCoresGB;
    if (allocatedMemoryGauge == null
        && metrics.getGauges().containsKey("variable.cluster.allocated.memory")) {
      allocatedMemoryGauge = metrics.getGauges().get("variable.cluster.allocated.memory");
    }
    if (allocatedVCoresGauge == null
        && metrics.getGauges().containsKey("variable.cluster.allocated.vcores")) {
      allocatedVCoresGauge = metrics.getGauges().get("variable.cluster.allocated.vcores");
    }
    if (availableMemoryGauge == null
        && metrics.getGauges().containsKey("variable.cluster.available.memory")) {
      availableMemoryGauge = metrics.getGauges().get("variable.cluster.available.memory");
    }
    if (availableVCoresGauge == null
        && metrics.getGauges().containsKey("variable.cluster.available.vcores")) {
      availableVCoresGauge = metrics.getGauges().get("variable.cluster.available.vcores");
    }
    allocatedMemoryGB =
        allocatedMemoryGauge == null
            ? 0
            : Double.parseDouble(allocatedMemoryGauge.getValue().toString()) / 1024;
    allocatedVCoresGB =
        allocatedVCoresGauge == null
            ? 0
            : Double.parseDouble(allocatedVCoresGauge.getValue().toString());
    availableMemoryGB =
        availableMemoryGauge == null
            ? 0
            : Double.parseDouble(availableMemoryGauge.getValue().toString()) / 1024;
    availableVCoresGB =
        availableVCoresGauge == null
            ? 0
            : Double.parseDouble(availableVCoresGauge.getValue().toString());

    // scheduler operation
    double allocateTimecost, handleTimecost;
    if (allocateTimecostHistogram == null
        && metrics.getHistograms().containsKey("sampler.scheduler.operation.allocate.timecost")) {
      allocateTimecostHistogram =
          metrics.getHistograms().get("sampler.scheduler.operation.allocate.timecost");
    }
    if (handleTimecostHistogram == null
        && metrics.getHistograms().containsKey("sampler.scheduler.operation.handle.timecost")) {
      handleTimecostHistogram =
          metrics.getHistograms().get("sampler.scheduler.operation.handle.timecost");
    }
    allocateTimecost =
        allocateTimecostHistogram == null
            ? 0.0
            : allocateTimecostHistogram.getSnapshot().getMean() / 1000000;
    handleTimecost =
        handleTimecostHistogram == null
            ? 0.0
            : handleTimecostHistogram.getSnapshot().getMean() / 1000000;
    // various handle operation
    Map<SchedulerEventType, Double> handleOperTimecostMap =
        new HashMap<SchedulerEventType, Double>();
    for (SchedulerEventType e : SchedulerEventType.values()) {
      String key = "sampler.scheduler.operation.handle." + e + ".timecost";
      if (!handleOperTimecostHistogramMap.containsKey(e)
          && metrics.getHistograms().containsKey(key)) {
        handleOperTimecostHistogramMap.put(e, metrics.getHistograms().get(key));
      }
      double timecost =
          handleOperTimecostHistogramMap.containsKey(e)
              ? handleOperTimecostHistogramMap.get(e).getSnapshot().getMean() / 1000000
              : 0;
      handleOperTimecostMap.put(e, timecost);
    }

    // allocated resource for each queue
    Map<String, Double> queueAllocatedMemoryMap = new HashMap<String, Double>();
    Map<String, Long> queueAllocatedVCoresMap = new HashMap<String, Long>();
    for (String queue : wrapper.getQueueSet()) {
      // memory
      String key = "counter.queue." + queue + ".allocated.memory";
      if (!queueAllocatedMemoryCounterMap.containsKey(queue)
          && metrics.getCounters().containsKey(key)) {
        queueAllocatedMemoryCounterMap.put(queue, metrics.getCounters().get(key));
      }
      double queueAllocatedMemoryGB =
          queueAllocatedMemoryCounterMap.containsKey(queue)
              ? queueAllocatedMemoryCounterMap.get(queue).getCount() / 1024.0
              : 0;
      queueAllocatedMemoryMap.put(queue, queueAllocatedMemoryGB);
      // vCores
      key = "counter.queue." + queue + ".allocated.cores";
      if (!queueAllocatedVCoresCounterMap.containsKey(queue)
          && metrics.getCounters().containsKey(key)) {
        queueAllocatedVCoresCounterMap.put(queue, metrics.getCounters().get(key));
      }
      long queueAllocatedVCores =
          queueAllocatedVCoresCounterMap.containsKey(queue)
              ? queueAllocatedVCoresCounterMap.get(queue).getCount()
              : 0;
      queueAllocatedVCoresMap.put(queue, queueAllocatedVCores);
    }

    // package results
    StringBuilder sb = new StringBuilder();
    sb.append("{");
    sb.append("\"time\":")
        .append(System.currentTimeMillis())
        .append(",\"jvm.free.memory\":")
        .append(jvmFreeMemoryGB)
        .append(",\"jvm.max.memory\":")
        .append(jvmMaxMemoryGB)
        .append(",\"jvm.total.memory\":")
        .append(jvmTotalMemoryGB)
        .append(",\"running.applications\":")
        .append(numRunningApps)
        .append(",\"running.containers\":")
        .append(numRunningContainers)
        .append(",\"cluster.allocated.memory\":")
        .append(allocatedMemoryGB)
        .append(",\"cluster.allocated.vcores\":")
        .append(allocatedVCoresGB)
        .append(",\"cluster.available.memory\":")
        .append(availableMemoryGB)
        .append(",\"cluster.available.vcores\":")
        .append(availableVCoresGB);

    for (String queue : wrapper.getQueueSet()) {
      sb.append(",\"queue.")
          .append(queue)
          .append(".allocated.memory\":")
          .append(queueAllocatedMemoryMap.get(queue));
      sb.append(",\"queue.")
          .append(queue)
          .append(".allocated.vcores\":")
          .append(queueAllocatedVCoresMap.get(queue));
    }
    // scheduler allocate & handle
    sb.append(",\"scheduler.allocate.timecost\":").append(allocateTimecost);
    sb.append(",\"scheduler.handle.timecost\":").append(handleTimecost);
    for (SchedulerEventType e : SchedulerEventType.values()) {
      sb.append(",\"scheduler.handle-")
          .append(e)
          .append(".timecost\":")
          .append(handleOperTimecostMap.get(e));
    }
    sb.append("}");
    return sb.toString();
  }