@Test
@InSequence(1)
public void gaugeCalledWithDefaultValue() {
assertThat("Gauge is not registered correctly", registry.getGauges(), hasKey(GAUGE_NAME));
@SuppressWarnings("unchecked")
Gauge<Long> gauge = registry.getGauges().get(GAUGE_NAME);
// Make sure that the gauge has the expected value
assertThat("Gauge value is incorrect", gauge.getValue(), is(equalTo(0L)));
}
/**
* package metrics information for one tracked queue/app only support FairScheduler currently
*
* @throws java.io.IOException
*/
private void printJsonTrack(HttpServletRequest request, HttpServletResponse response)
throws IOException {
response.setContentType("text/json");
response.setStatus(HttpServletResponse.SC_OK);
StringBuilder sb = new StringBuilder();
if (schedulerMetrics instanceof FairSchedulerMetrics) {
String para = request.getParameter("t");
if (para.startsWith("Job ")) {
String appId = para.substring("Job ".length());
sb.append("{");
sb.append("\"time\": ").append(System.currentTimeMillis()).append(",");
sb.append("\"appId\": \"").append(appId).append("\"");
for (String metric : this.schedulerMetrics.getAppTrackedMetrics()) {
String key = "variable.app." + appId + "." + metric;
sb.append(",\"").append(metric).append("\": ");
if (metrics.getGauges().containsKey(key)) {
double memoryGB =
Double.parseDouble(metrics.getGauges().get(key).getValue().toString()) / 1024;
sb.append(memoryGB);
} else {
sb.append(-1);
}
}
sb.append("}");
} else if (para.startsWith("Queue ")) {
String queueName = para.substring("Queue ".length());
sb.append("{");
sb.append("\"time\": ").append(System.currentTimeMillis()).append(",");
sb.append("\"queueName\": \"").append(queueName).append("\"");
for (String metric : this.schedulerMetrics.getQueueTrackedMetrics()) {
String key = "variable.queue." + queueName + "." + metric;
sb.append(",\"").append(metric).append("\": ");
if (metrics.getGauges().containsKey(key)) {
double memoryGB =
Double.parseDouble(metrics.getGauges().get(key).getValue().toString()) / 1024;
sb.append(memoryGB);
} else {
sb.append(-1);
}
}
sb.append("}");
}
}
String output = sb.toString();
if (output.isEmpty()) {
output = "[]";
}
response.getWriter().println(output);
// package result
((Request) request).setHandled(true);
}
@Test
public void aPrivateGaugeInSuperclass() throws Exception {
final Gauge<?> metric = registry.getGauges().get(name("gaugeParentPrivate"));
assertNotNull(metric);
assertEquals("gaugeParentPrivate", metric.getValue());
}
@Test
public void test() throws Exception {
Map<String, JobParameter> parameters = new HashMap<String, JobParameter>();
JobExecution jobExecution = jobLauncherTestUtils.launchJob(new JobParameters(parameters));
Assert.assertEquals(
jobExecution.getExitStatus().getExitDescription(),
BatchStatus.COMPLETED,
jobExecution.getStatus());
Map<String, Meter> meters = metricRegistry.getMeters();
assertThat(meters).hasSize(2);
assertThat(meters)
.containsKey("batch.sampleJob.job.metered")
.containsKey("batch.sampleJob.step.chunkStep.step.metered");
assertThat(extractProperty("count", Number.class).from(meters.values()))
.contains(1L)
.doesNotContain(0L);
Map<String, Timer> timers = metricRegistry.getTimers();
assertThat(timers).hasSize(6);
assertThat(timers)
.containsKey("batch.sampleJob.job.timed")
.containsKey("batch.sampleJob.step.chunkStep.chunk.timed")
.containsKey("batch.sampleJob.step.chunkStep.step.timed")
.containsKey("batch.sampleJob.step.chunkStep.read.timed")
.containsKey("batch.sampleJob.step.chunkStep.process.timed")
.containsKey("batch.sampleJob.step.chunkStep.write.timed");
assertThat(extractProperty("count", Number.class).from(timers.values()))
.contains(1L, 3L, 4L)
.doesNotContain(0L);
Map<String, Gauge> gauges = metricRegistry.getGauges();
assertThat(gauges).hasSize(0);
}
/**
* @summary get the worker metrics
* @return the response object
*/
@GET
@Path(GET_METRICS)
@ReturnType("java.util.SortedMap<String, Long>")
public Response getMetrics() {
MetricRegistry metricRegistry = mWorker.getWorkerMetricsSystem().getMetricRegistry();
// Get all counters.
Map<String, Counter> counters = metricRegistry.getCounters();
// Only the gauge for cached blocks is retrieved here, other gauges are statistics of free/used
// spaces, those statistics can be gotten via other REST apis.
String blocksCachedProperty =
CommonUtils.argsToString(
".", WorkerContext.getWorkerSource().getName(), WorkerSource.BLOCKS_CACHED);
@SuppressWarnings("unchecked")
Gauge<Integer> blocksCached =
(Gauge<Integer>) metricRegistry.getGauges().get(blocksCachedProperty);
// Get values of the counters and gauges and put them into a metrics map.
SortedMap<String, Long> metrics = new TreeMap<>();
for (Map.Entry<String, Counter> counter : counters.entrySet()) {
metrics.put(counter.getKey(), counter.getValue().getCount());
}
metrics.put(blocksCachedProperty, blocksCached.getValue().longValue());
return RestUtils.createResponse(metrics);
}
@Test
public void aPrivateGauge() throws Exception {
final Gauge<?> metric =
registry.getGauges().get(name(InstrumentedWithGauge.class, "gaugePrivate"));
assertNotNull(metric);
assertEquals("gaugePrivate", metric.getValue());
}
@Test
public void aGaugeAnnotatedMethodWithAbsoluteName() throws Exception {
instance.doAThingWithAbsoluteName();
final Gauge metric = registry.getGauges().get(name("absoluteName"));
assertThat("Guice creates a metric", metric, is(notNullValue()));
assertThat(
"Guice creates a gauge with the given value",
((Gauge<String>) metric).getValue(),
is("anotherThingWithAbsoluteName"));
}
@Test
public void aGaugeAnnotatedMethod() throws Exception {
instance.doAThing();
final Gauge metric = registry.getGauges().get(name(InstrumentedWithGauge.class, "things"));
assertThat("Guice creates a metric", metric, is(notNullValue()));
assertThat(
"Guice creates a gauge with the given value",
((Gauge<String>) metric).getValue(),
is("poop"));
}
@Test
public void aGaugeAnnotatedMethodWithDefaultName() throws Exception {
instance.doAnotherThing();
final Gauge metric =
registry.getGauges().get(name(InstrumentedWithGauge.class, "doAnotherThing", GAUGE_SUFFIX));
assertThat("Guice creates a metric", metric, is(notNullValue()));
assertThat(
"Guice creates a gauge with the given value",
((Gauge<String>) metric).getValue(),
is("anotherThing"));
}
@Test
@InSequence(2)
public void callGaugeAfterSetterCall() throws InterruptedException {
assertThat("Gauge is not registered correctly", registry.getGauges(), hasKey(GAUGE_NAME));
@SuppressWarnings("unchecked")
Gauge<Long> gauge = registry.getGauges().get(GAUGE_NAME);
// Make sure that the gauge has the default value
assertThat("Gauge value is incorrect", gauge.getValue(), is(equalTo(0L)));
// Call the setter method
long value = 1L + Math.round(Math.random() * (Long.MAX_VALUE - 1L));
bean.setGauge(value);
// Assert the gauge returns the cached value
assertThat("Gauge value is incorrect", gauge.getValue(), is(equalTo(0L)));
// Wait for two cache timeout periods
Thread.sleep(2 * 500L);
// Assert the gauge is refreshed and up-to-date
assertThat("Gauge value is incorrect", gauge.getValue(), is(equalTo(value)));
}
public SerialCollector(MetricMonitorRegistry registry) {
if (registry == null) {
throw new NullPointerException("registry must not be null");
}
final MetricRegistry metricRegistry = registry.getRegistry();
final SortedMap<String, Gauge> gauges = metricRegistry.getGauges();
this.heapMax = getLongGauge(gauges, JVM_MEMORY_HEAP_MAX);
this.heapUsed = getLongGauge(gauges, JVM_MEMORY_HEAP_USED);
this.heapNonHeapMax = getLongGauge(gauges, JVM_MEMORY_NONHEAP_MAX);
this.heapNonHeapUsed = getLongGauge(gauges, JVM_MEMORY_NONHEAP_USED);
this.gcCount = getLongGauge(gauges, JVM_GC_SERIAL_MSC_COUNT);
this.gcTime = getLongGauge(gauges, JVM_GC_SERIAL_MSC_TIME);
}
/**
* Registers a gauge if it has not been registered.
*
* @param name the gauge name
* @param metric the gauge
* @param <T> the type
*/
public static synchronized <T> void registerGaugeIfAbsent(String name, Gauge<T> metric) {
if (!METRIC_REGISTRY.getGauges().containsKey(name)) {
METRIC_REGISTRY.register(name, metric);
}
}
public String generateRealTimeTrackingMetrics() {
// JVM
double jvmFreeMemoryGB, jvmMaxMemoryGB, jvmTotalMemoryGB;
if (jvmFreeMemoryGauge == null && metrics.getGauges().containsKey("variable.jvm.free.memory")) {
jvmFreeMemoryGauge = metrics.getGauges().get("variable.jvm.free.memory");
}
if (jvmMaxMemoryGauge == null && metrics.getGauges().containsKey("variable.jvm.max.memory")) {
jvmMaxMemoryGauge = metrics.getGauges().get("variable.jvm.max.memory");
}
if (jvmTotalMemoryGauge == null
&& metrics.getGauges().containsKey("variable.jvm.total.memory")) {
jvmTotalMemoryGauge = metrics.getGauges().get("variable.jvm.total.memory");
}
jvmFreeMemoryGB =
jvmFreeMemoryGauge == null
? 0
: Double.parseDouble(jvmFreeMemoryGauge.getValue().toString()) / 1024 / 1024 / 1024;
jvmMaxMemoryGB =
jvmMaxMemoryGauge == null
? 0
: Double.parseDouble(jvmMaxMemoryGauge.getValue().toString()) / 1024 / 1024 / 1024;
jvmTotalMemoryGB =
jvmTotalMemoryGauge == null
? 0
: Double.parseDouble(jvmTotalMemoryGauge.getValue().toString()) / 1024 / 1024 / 1024;
// number of running applications/containers
String numRunningApps, numRunningContainers;
if (numRunningAppsGauge == null
&& metrics.getGauges().containsKey("variable.running.application")) {
numRunningAppsGauge = metrics.getGauges().get("variable.running.application");
}
if (numRunningContainersGauge == null
&& metrics.getGauges().containsKey("variable.running.container")) {
numRunningContainersGauge = metrics.getGauges().get("variable.running.container");
}
numRunningApps = numRunningAppsGauge == null ? "0" : numRunningAppsGauge.getValue().toString();
numRunningContainers =
numRunningContainersGauge == null ? "0" : numRunningContainersGauge.getValue().toString();
// cluster available/allocate resource
double allocatedMemoryGB, allocatedVCoresGB, availableMemoryGB, availableVCoresGB;
if (allocatedMemoryGauge == null
&& metrics.getGauges().containsKey("variable.cluster.allocated.memory")) {
allocatedMemoryGauge = metrics.getGauges().get("variable.cluster.allocated.memory");
}
if (allocatedVCoresGauge == null
&& metrics.getGauges().containsKey("variable.cluster.allocated.vcores")) {
allocatedVCoresGauge = metrics.getGauges().get("variable.cluster.allocated.vcores");
}
if (availableMemoryGauge == null
&& metrics.getGauges().containsKey("variable.cluster.available.memory")) {
availableMemoryGauge = metrics.getGauges().get("variable.cluster.available.memory");
}
if (availableVCoresGauge == null
&& metrics.getGauges().containsKey("variable.cluster.available.vcores")) {
availableVCoresGauge = metrics.getGauges().get("variable.cluster.available.vcores");
}
allocatedMemoryGB =
allocatedMemoryGauge == null
? 0
: Double.parseDouble(allocatedMemoryGauge.getValue().toString()) / 1024;
allocatedVCoresGB =
allocatedVCoresGauge == null
? 0
: Double.parseDouble(allocatedVCoresGauge.getValue().toString());
availableMemoryGB =
availableMemoryGauge == null
? 0
: Double.parseDouble(availableMemoryGauge.getValue().toString()) / 1024;
availableVCoresGB =
availableVCoresGauge == null
? 0
: Double.parseDouble(availableVCoresGauge.getValue().toString());
// scheduler operation
double allocateTimecost, handleTimecost;
if (allocateTimecostHistogram == null
&& metrics.getHistograms().containsKey("sampler.scheduler.operation.allocate.timecost")) {
allocateTimecostHistogram =
metrics.getHistograms().get("sampler.scheduler.operation.allocate.timecost");
}
if (handleTimecostHistogram == null
&& metrics.getHistograms().containsKey("sampler.scheduler.operation.handle.timecost")) {
handleTimecostHistogram =
metrics.getHistograms().get("sampler.scheduler.operation.handle.timecost");
}
allocateTimecost =
allocateTimecostHistogram == null
? 0.0
: allocateTimecostHistogram.getSnapshot().getMean() / 1000000;
handleTimecost =
handleTimecostHistogram == null
? 0.0
: handleTimecostHistogram.getSnapshot().getMean() / 1000000;
// various handle operation
Map<SchedulerEventType, Double> handleOperTimecostMap =
new HashMap<SchedulerEventType, Double>();
for (SchedulerEventType e : SchedulerEventType.values()) {
String key = "sampler.scheduler.operation.handle." + e + ".timecost";
if (!handleOperTimecostHistogramMap.containsKey(e)
&& metrics.getHistograms().containsKey(key)) {
handleOperTimecostHistogramMap.put(e, metrics.getHistograms().get(key));
}
double timecost =
handleOperTimecostHistogramMap.containsKey(e)
? handleOperTimecostHistogramMap.get(e).getSnapshot().getMean() / 1000000
: 0;
handleOperTimecostMap.put(e, timecost);
}
// allocated resource for each queue
Map<String, Double> queueAllocatedMemoryMap = new HashMap<String, Double>();
Map<String, Long> queueAllocatedVCoresMap = new HashMap<String, Long>();
for (String queue : wrapper.getQueueSet()) {
// memory
String key = "counter.queue." + queue + ".allocated.memory";
if (!queueAllocatedMemoryCounterMap.containsKey(queue)
&& metrics.getCounters().containsKey(key)) {
queueAllocatedMemoryCounterMap.put(queue, metrics.getCounters().get(key));
}
double queueAllocatedMemoryGB =
queueAllocatedMemoryCounterMap.containsKey(queue)
? queueAllocatedMemoryCounterMap.get(queue).getCount() / 1024.0
: 0;
queueAllocatedMemoryMap.put(queue, queueAllocatedMemoryGB);
// vCores
key = "counter.queue." + queue + ".allocated.cores";
if (!queueAllocatedVCoresCounterMap.containsKey(queue)
&& metrics.getCounters().containsKey(key)) {
queueAllocatedVCoresCounterMap.put(queue, metrics.getCounters().get(key));
}
long queueAllocatedVCores =
queueAllocatedVCoresCounterMap.containsKey(queue)
? queueAllocatedVCoresCounterMap.get(queue).getCount()
: 0;
queueAllocatedVCoresMap.put(queue, queueAllocatedVCores);
}
// package results
StringBuilder sb = new StringBuilder();
sb.append("{");
sb.append("\"time\":")
.append(System.currentTimeMillis())
.append(",\"jvm.free.memory\":")
.append(jvmFreeMemoryGB)
.append(",\"jvm.max.memory\":")
.append(jvmMaxMemoryGB)
.append(",\"jvm.total.memory\":")
.append(jvmTotalMemoryGB)
.append(",\"running.applications\":")
.append(numRunningApps)
.append(",\"running.containers\":")
.append(numRunningContainers)
.append(",\"cluster.allocated.memory\":")
.append(allocatedMemoryGB)
.append(",\"cluster.allocated.vcores\":")
.append(allocatedVCoresGB)
.append(",\"cluster.available.memory\":")
.append(availableMemoryGB)
.append(",\"cluster.available.vcores\":")
.append(availableVCoresGB);
for (String queue : wrapper.getQueueSet()) {
sb.append(",\"queue.")
.append(queue)
.append(".allocated.memory\":")
.append(queueAllocatedMemoryMap.get(queue));
sb.append(",\"queue.")
.append(queue)
.append(".allocated.vcores\":")
.append(queueAllocatedVCoresMap.get(queue));
}
// scheduler allocate & handle
sb.append(",\"scheduler.allocate.timecost\":").append(allocateTimecost);
sb.append(",\"scheduler.handle.timecost\":").append(handleTimecost);
for (SchedulerEventType e : SchedulerEventType.values()) {
sb.append(",\"scheduler.handle-")
.append(e)
.append(".timecost\":")
.append(handleOperTimecostMap.get(e));
}
sb.append("}");
return sb.toString();
}
