@Override
public Jedis getResource() {
// add a simplistic retry strategy so we don't fail on the occasional pool timeout
Jedis resource = null;
try {
resource = super.getResource();
} catch (RuntimeException e) {
try {
Thread.sleep(500); // give it half a second to recover
} catch (InterruptedException ex) {
throw new IllegalStateException(ex);
}
try {
resource = super.getResource();
poolRetryCounter.inc();
} catch (RuntimeException e2) {
LOG.error(
"redis connect failure after retry once. Host: '"
+ redisHost
+ "' port: '"
+ redisPort
+ "' redis db: '"
+ redisDatabase
+ "' redis password: '******'");
poolFailCounter.inc();
// rethrow and let things escalate
throw e2;
}
}
return (Jedis) Enhancer.create(Jedis.class, new MeasuringJedisHandler(resource));
}
@Test
public void testCounterNoMatch() {
// create timer with id "testMetricAlerts" and register with metric registry, bump up value to
// 4.
Counter c =
MetricsConfigurator.createCounter(metrics, "testCounterNoMatch", PIPELINE_NAME, REVISION);
c.inc(100);
MetricsRuleDefinition metricsRuleDefinition =
new MetricsRuleDefinition(
"testCounterNoMatch",
"testCounterNoMatch",
"testCounterNoMatch",
MetricType.COUNTER,
MetricElement.COUNTER_COUNT,
"${value()>100}",
false,
true);
MetricRuleEvaluator metricRuleEvaluator =
new MetricRuleEvaluator(
metricsRuleDefinition,
metrics,
new AlertManager(
PIPELINE_NAME, REVISION, null, metrics, runtimeInfo, new EventListenerManager()),
Collections.<String>emptyList());
metricRuleEvaluator.checkForAlerts();
// get alert gauge
Gauge<Object> gauge =
MetricsConfigurator.getGauge(
metrics, AlertsUtil.getAlertGaugeName(metricsRuleDefinition.getId()));
Assert.assertNull(gauge);
}
@Override
protected void onComplete(Request request, Response response) {
activeRequestsCounter.dec();
Long duration =
computeDurationMillis(START_TIMES_BY_CORRELATION_ID.remove(request.getCorrelationId()));
if (duration != null && duration.longValue() > 0) {
allTimesTimer.update(duration, TimeUnit.MILLISECONDS);
String name = getRouteName(request);
if (name == null || name.isEmpty()) return;
ROUTE_TIMERS.get(name).update(duration, TimeUnit.MILLISECONDS);
}
Counter responseCounter = COUNTERS_BY_RESPONSE.get(response.getResponseStatus().getCode());
if (responseCounter == null) {
responseCounter = metrics.counter(getResponseCounterName(response.getResponseStatus()));
COUNTERS_BY_RESPONSE.putIfAbsent(response.getResponseStatus().getCode(), responseCounter);
}
responseCounter.inc();
publish(request, response, duration);
}
@Test
@InSequence(1)
public void countedMethodNotCalledYet() {
assertThat("Counter is not registered correctly", registry.getCounters(), hasKey(COUNTER_NAME));
Counter counter = registry.getCounters().get(COUNTER_NAME);
// Make sure that the counter hasn't been called yet
assertThat("Counter count is incorrect", counter.getCount(), is(equalTo(COUNTER_COUNT.get())));
}
@Override
@Timed("ninja.cache.get")
public Object get(String key) {
Object result = underlyingCache.get(key);
if (result == null) {
missCounter.inc();
} else {
hitCounter.inc();
}
return result;
}
@Override
@Timed("ninja.cache.getMany")
public Map<String, Object> get(String[] keys) {
Map<String, Object> result = underlyingCache.get(keys);
if (result == null || result.isEmpty()) {
missCounter.inc(keys.length);
} else {
hitCounter.inc(result.size());
missCounter.inc(keys.length - result.size());
}
return result;
}
@Override
protected void onException(Throwable exception, Request request, Response response) {
allExceptionsCounter.inc();
String name = getRouteName(request);
if (name == null || name.isEmpty()) return;
Counter exceptionCounter = EXCEPTION_COUNTERS_BY_ROUTE.get(name);
if (exceptionCounter == null) return;
exceptionCounter.inc();
}
/**
* Update record-level metrics.
*
* @param recordsWritten number of records written by the writer
* @param branchIndex fork branch index
*/
public void updateRecordMetrics(long recordsWritten, int branchIndex) {
TaskMetrics metrics = TaskMetrics.get(this);
String forkBranchId = ForkOperatorUtils.getForkId(this.taskId, branchIndex);
Counter taskRecordCounter =
metrics.getCounter(gobblin.runtime.util.MetricGroup.TASK.name(), forkBranchId, RECORDS);
long inc = recordsWritten - taskRecordCounter.getCount();
taskRecordCounter.inc(inc);
metrics.getMeter(MetricGroup.TASK.name(), forkBranchId, RECORDS_PER_SECOND).mark(inc);
metrics.getCounter(MetricGroup.JOB.name(), this.jobId, RECORDS).inc(inc);
metrics.getMeter(MetricGroup.JOB.name(), this.jobId, RECORDS_PER_SECOND).mark(inc);
}
@Test
public void testThatPercolationNotificationWorks() throws IOException, InterruptedException {
SimpleNotifier notifier = new SimpleNotifier();
MetricFilter percolationFilter =
new MetricFilter() {
@Override
public boolean matches(String name, Metric metric) {
return name.startsWith(prefix + ".foo");
}
};
elasticsearchReporter =
createElasticsearchReporterBuilder()
.percolationFilter(percolationFilter)
.percolationNotifier(notifier)
.build();
final Counter evictions = registry.counter("foo");
evictions.inc(18);
reportAndRefresh();
QueryBuilder queryBuilder =
QueryBuilders.boolQuery()
.must(QueryBuilders.matchAllQuery())
.filter(
QueryBuilders.boolQuery()
.must(QueryBuilders.rangeQuery("count").gte(20))
.must(QueryBuilders.termQuery("name", prefix + ".foo")));
String json = String.format("{ \"query\" : %s }", queryBuilder.buildAsBytes().toUtf8());
client()
.prepareIndex(indexWithDate, ".percolator", "myName")
.setRefresh(true)
.setSource(json)
.execute()
.actionGet();
evictions.inc(1);
reportAndRefresh();
assertThat(notifier.metrics.size(), is(0));
evictions.inc(2);
reportAndRefresh();
org.assertj.core.api.Assertions.assertThat(notifier.metrics.size()).isEqualTo(1);
org.assertj.core.api.Assertions.assertThat(notifier.metrics).containsKey("myName");
org.assertj.core.api.Assertions.assertThat(notifier.metrics.get("myName").name())
.isEqualTo(prefix + ".foo");
notifier.metrics.clear();
evictions.dec(2);
reportAndRefresh();
org.assertj.core.api.Assertions.assertThat(notifier.metrics.size()).isEqualTo(0);
}
@Test
public void testThatBulkIndexingWorks() {
for (int i = 0; i < 2020; i++) {
final Counter evictions = registry.counter(name("foo", "bar", String.valueOf(i)));
evictions.inc(i);
}
reportAndRefresh();
SearchResponse searchResponse =
client().prepareSearch(indexWithDate).setTypes("counter").execute().actionGet();
org.assertj.core.api.Assertions.assertThat(searchResponse.getHits().totalHits())
.isEqualTo(2020L);
}
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
throws IOException, ServletException {
final StatusExposingServletResponse wrappedResponse =
new StatusExposingServletResponse((HttpServletResponse) response);
activeRequests.inc();
final Timer.Context context = requestTimer.time();
try {
chain.doFilter(request, wrappedResponse);
} finally {
context.stop();
activeRequests.dec();
markMeterForStatusCode(wrappedResponse.getStatus());
}
}
@Override
public void run() {
Timer.Context rollupTimerContext = rollupTimer.time();
try {
Rollup.Type rollupComputer =
RollupRunnable.getRollupComputer(RollupType.BF_BASIC, Granularity.FULL);
Rollup rollup = rollupComputer.compute(points);
writer.enqueueRollupForWrite(
new SingleRollupWriteContext(
rollup, loc, Granularity.MIN_5, CassandraModel.CF_METRICS_5M, range.getStart()));
log.info("Calculated and queued rollup for " + loc + " within range " + range);
} catch (Exception e) {
// I want to be very harsh with exceptions encountered while validating and computing rollups.
// Just stop everything.
log.error("Error encountered while validating and calculating rollups", e);
rollupValidationAndComputeFailed.inc();
RollupGenerator.rollupExecutors.shutdownNow();
OutOFBandRollup.getRollupGeneratorThread().interrupt();
// Stop the monitoring thread
OutOFBandRollup.getMonitoringThread().interrupt();
// Stop the file handler thread pool from sending data to buildstore
FileHandler.handlerThreadPool.shutdownNow();
throw new RuntimeException(e);
} finally {
rollupTimerContext.stop();
}
}
@Override
public Allocation allocate(
ApplicationAttemptId attemptId,
List<ResourceRequest> resourceRequests,
List<ContainerId> containerIds,
List<String> strings,
List<String> strings2) {
if (metricsON) {
final Timer.Context context = schedulerAllocateTimer.time();
Allocation allocation = null;
try {
allocation =
scheduler.allocate(attemptId, resourceRequests, containerIds, strings, strings2);
return allocation;
} finally {
context.stop();
schedulerAllocateCounter.inc();
try {
updateQueueWithAllocateRequest(allocation, attemptId, resourceRequests, containerIds);
} catch (IOException e) {
e.printStackTrace();
}
}
} else {
return scheduler.allocate(attemptId, resourceRequests, containerIds, strings, strings2);
}
}
public ExecuteResult execute(Tuple input, OutputCollector outputCollector) {
if (tuplesReceived != null) {
tuplesReceived.inc();
}
String docId = input.getString(0);
Object docObj = input.getValue(1);
if (docId == null || docObj == null) {
log.warn("Ignored tuple: " + input);
return ExecuteResult.IGNORED; // nothing to index
}
try {
return processInputDoc(docId, docObj);
} catch (Exception exc) {
log.error("Failed to process " + docId + " due to: " + exc);
if (exc instanceof RuntimeException) {
throw (RuntimeException) exc;
} else {
throw new RuntimeException(exc);
}
}
}
@Test
public void testCounter() throws Exception {
final Counter evictions = registry.counter(name("test", "cache-evictions"));
evictions.inc(25);
reportAndRefresh();
SearchResponse searchResponse =
client().prepareSearch(indexWithDate).setTypes("counter").execute().actionGet();
org.assertj.core.api.Assertions.assertThat(searchResponse.getHits().totalHits()).isEqualTo(1L);
Map<String, Object> hit = searchResponse.getHits().getAt(0).sourceAsMap();
assertTimestamp(hit);
assertKey(hit, "count", 25);
assertKey(hit, "name", prefix + ".test.cache-evictions");
assertKey(hit, "host", "localhost");
}
@Override
public void handle(
String path,
Request request,
HttpServletRequest httpRequest,
HttpServletResponse httpResponse)
throws IOException, ServletException {
activeDispatches.inc();
final long start;
final HttpChannelState state = request.getHttpChannelState();
if (state.isInitial()) {
// new request
activeRequests.inc();
start = request.getTimeStamp();
} else {
// resumed request
start = System.currentTimeMillis();
activeSuspended.dec();
if (state.getState() == State.DISPATCHED) {
asyncDispatches.mark();
}
}
try {
super.handle(path, request, httpRequest, httpResponse);
} finally {
final long now = System.currentTimeMillis();
final long dispatched = now - start;
activeDispatches.dec();
dispatches.update(dispatched, TimeUnit.MILLISECONDS);
if (state.isSuspended()) {
if (state.isInitial()) {
state.addListener(listener);
}
activeSuspended.inc();
} else if (state.isInitial()) {
requests.update(dispatched, TimeUnit.MILLISECONDS);
updateResponses(request);
}
// else onCompletion will handle it.
}
}
public void printMetrics() {
logger.info("Metrics");
Metrics metrics = session.getCluster().getMetrics();
Gauge<Integer> gauge = metrics.getConnectedToHosts();
Integer numberOfHosts = gauge.getValue();
logger.info("Number of hosts: " + numberOfHosts);
Metrics.Errors errors = metrics.getErrorMetrics();
Counter counter = errors.getReadTimeouts();
logger.info("Number of read timeouts:" + counter.getCount());
com.codahale.metrics.Timer timer = metrics.getRequestsTimer();
Timer.Context context = timer.time();
try {
long numberUserRequests = timer.getCount();
logger.info("Number of user requests:" + numberUserRequests);
} finally {
context.stop();
}
}
private void updateResponses(Request request) {
final int response = request.getResponse().getStatus() / 100;
if (response >= 1 && response <= 5) {
responses[response - 1].mark();
}
activeRequests.dec();
final long elapsedTime = System.currentTimeMillis() - request.getTimeStamp();
requests.update(elapsedTime, TimeUnit.MILLISECONDS);
requestTimer(request.getMethod()).update(elapsedTime, TimeUnit.MILLISECONDS);
}
public static void main(String[] args) {
ReporterV08 reporter = null;
try {
final MetricRegistry registry = new MetricRegistry();
reporter = getInfluxdbReporter(registry);
reporter.start(3, TimeUnit.SECONDS);
final Counter counter = registry.counter(MetricRegistry.name("test", "counter"));
for (int i = 0; i < 1; ++i) {
counter.inc();
Thread.sleep(Math.round(Math.random()) * 1000);
}
} catch (Exception e) {
e.printStackTrace();
System.exit(1);
} finally {
if (reporter != null) {
reporter.report();
reporter.stop();
}
}
}
/** Tests GetAllMetrics method. */
@Test
public void testGetAllMetrics() {
Counter onosCounter = new Counter();
onosCounter.inc();
Meter onosMeter = new Meter();
onosMeter.mark();
Timer onosTimer = new Timer();
onosTimer.update(1, TimeUnit.MILLISECONDS);
ImmutableMap<String, Metric> metrics =
new ImmutableMap.Builder<String, Metric>()
.put("onosCounter", onosCounter)
.put("onosMeter", onosMeter)
.put("onosTimer", onosTimer)
.build();
expect(mockMetricsService.getMetrics()).andReturn(metrics).anyTimes();
replay(mockMetricsService);
WebTarget wt = target();
String response = wt.path("metrics").request().get(String.class);
assertThat(response, containsString("{\"metrics\":["));
JsonObject result = Json.parse(response).asObject();
assertThat(result, notNullValue());
JsonArray jsonMetrics = result.get("metrics").asArray();
assertThat(jsonMetrics, notNullValue());
assertThat(jsonMetrics.size(), is(3));
assertTrue(
matchesMetric(metrics.get("onosCounter")).matchesSafely(jsonMetrics.get(0).asObject()));
assertTrue(
matchesMetric(metrics.get("onosMeter")).matchesSafely(jsonMetrics.get(1).asObject()));
assertTrue(
matchesMetric(metrics.get("onosTimer")).matchesSafely(jsonMetrics.get(2).asObject()));
}
@Test
public void testCounter() {
System.out.println("******************************* COUNTER *******************************");
counter = registry.counter("counter");
try {
for (int i = 0; i < ITER_COUNT; i++) {
counter.inc(i);
Thread.sleep(SLEEP_MS);
}
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
}
@Test
@InSequence(4)
public void removeCounterFromRegistry() {
assertThat("Counter is not registered correctly", registry.getCounters(), hasKey(COUNTER_NAME));
Counter counter = registry.getCounters().get(COUNTER_NAME);
// Remove the counter from metrics registry
registry.remove(COUNTER_NAME);
try {
// Call the counted method and assert an exception is thrown
bean.countedMethod(
new Callable<Long>() {
@Override
public Long call() throws Exception {
return null;
}
});
} catch (Exception cause) {
assertThat(cause, is(instanceOf(IllegalStateException.class)));
assertThat(
cause.getMessage(),
is(
equalTo(
"No counter with name ["
+ COUNTER_NAME
+ "] found in registry ["
+ registry
+ "]")));
// Make sure that the counter hasn't been called
assertThat(
"Counter count is incorrect", counter.getCount(), is(equalTo(COUNTER_COUNT.get())));
return;
}
fail("No exception has been re-thrown!");
}
protected ExecuteResult flushBufferedDocs(DocBuffer b) {
int numDocsInBatch = b.buffer.size();
if (numDocsInBatch == 0) {
b.reset();
return ExecuteResult.ACK;
}
Timer.Context timer = (sendBatchToSolr != null) ? sendBatchToSolr.time() : null;
try {
sendBatchToSolr(b);
} finally {
if (timer != null) timer.stop();
if (indexedCounter != null) indexedCounter.inc(numDocsInBatch);
b.reset();
}
return ExecuteResult.ACK;
}
public void incMatchesSuccesses() {
matchesSuccess.inc();
}
public void incPredictionsFailures() {
predictionsFailures.inc();
}
public void incPredictionsSuccesses() {
predictionsSuccess.inc();
}
public void incMetadataFailures() {
metadataFailures.inc();
}
public void incMetadataSuccesses() {
metadataSuccess.inc();
}
public void incMatchesFailures() {
matchesFailures.inc();
}
@Override
public boolean matchesSafely(JsonObject jsonObject) {
JsonObject jsonMetric = jsonObject.get("metric").asObject();
JsonObject jsonCounter;
JsonObject jsonMeter;
JsonObject jsonTimer;
Counter counter;
Meter meter;
Timer timer;
// check counter metric
if (jsonMetric.get("counter") != null) {
jsonCounter = jsonMetric.get("counter").asObject();
counter = (Counter) metric;
if (jsonCounter.get("counter").asLong() != counter.getCount()) {
reason = "counter " + counter.getCount();
return false;
}
}
// check meter metric
if (jsonMetric.get("meter") != null) {
jsonMeter = jsonMetric.get("meter").asObject();
meter = (Meter) metric;
if (jsonMeter.get("counter").asLong() != meter.getCount()) {
reason = "counter " + meter.getCount();
return false;
}
if (jsonMeter.get("1_min_rate").asDouble() != meter.getOneMinuteRate()) {
reason = "1 minute rate " + meter.getOneMinuteRate();
return false;
}
if (jsonMeter.get("5_min_rate").asDouble() != meter.getOneMinuteRate()) {
reason = "5 minute rate " + meter.getFiveMinuteRate();
return false;
}
if (jsonMeter.get("15_min_rate").asDouble() != meter.getFifteenMinuteRate()) {
reason = "15 minute rate " + meter.getFifteenMinuteRate();
return false;
}
}
if (jsonMetric.get("timer") != null) {
jsonTimer = jsonMetric.get("timer").asObject();
timer = (Timer) metric;
if (jsonTimer.get("counter").asLong() != timer.getCount()) {
reason = "counter " + timer.getCount();
return false;
}
if (jsonTimer.get("1_min_rate").asDouble() != timer.getOneMinuteRate()) {
reason = "1 minute rate " + timer.getOneMinuteRate();
return false;
}
if (jsonTimer.get("5_min_rate").asDouble() != timer.getOneMinuteRate()) {
reason = "5 minute rate " + timer.getFiveMinuteRate();
return false;
}
if (jsonTimer.get("15_min_rate").asDouble() != timer.getFifteenMinuteRate()) {
reason = "15 minute rate " + timer.getFifteenMinuteRate();
return false;
}
if (jsonTimer.get("mean").asDouble() != nanoToMs(timer.getSnapshot().getMean())) {
reason = "mean " + timer.getSnapshot().getMean();
return false;
}
if (jsonTimer.get("min").asDouble() != nanoToMs(timer.getSnapshot().getMin())) {
reason = "min " + timer.getSnapshot().getMin();
return false;
}
if (jsonTimer.get("max").asDouble() != nanoToMs(timer.getSnapshot().getMax())) {
reason = "max " + timer.getSnapshot().getMax();
return false;
}
if (jsonTimer.get("stddev").asDouble() != nanoToMs(timer.getSnapshot().getStdDev())) {
reason = "stddev " + timer.getSnapshot().getStdDev();
return false;
}
}
return true;
}
