@Override
public Jedis getResource() {
// add a simplistic retry strategy so we don't fail on the occasional pool timeout
Jedis resource = null;
try {
resource = super.getResource();
} catch (RuntimeException e) {
try {
Thread.sleep(500); // give it half a second to recover
} catch (InterruptedException ex) {
throw new IllegalStateException(ex);
}
try {
resource = super.getResource();
poolRetryCounter.inc();
} catch (RuntimeException e2) {
LOG.error(
"redis connect failure after retry once. Host: '"
+ redisHost
+ "' port: '"
+ redisPort
+ "' redis db: '"
+ redisDatabase
+ "' redis password: '******'");
poolFailCounter.inc();
// rethrow and let things escalate
throw e2;
}
}
return (Jedis) Enhancer.create(Jedis.class, new MeasuringJedisHandler(resource));
}
@Override
@Timed("ninja.cache.get")
public Object get(String key) {
Object result = underlyingCache.get(key);
if (result == null) {
missCounter.inc();
} else {
hitCounter.inc();
}
return result;
}
@Override
@Timed("ninja.cache.getMany")
public Map<String, Object> get(String[] keys) {
Map<String, Object> result = underlyingCache.get(keys);
if (result == null || result.isEmpty()) {
missCounter.inc(keys.length);
} else {
hitCounter.inc(result.size());
missCounter.inc(keys.length - result.size());
}
return result;
}
@Override
protected void onException(Throwable exception, Request request, Response response) {
allExceptionsCounter.inc();
String name = getRouteName(request);
if (name == null || name.isEmpty()) return;
Counter exceptionCounter = EXCEPTION_COUNTERS_BY_ROUTE.get(name);
if (exceptionCounter == null) return;
exceptionCounter.inc();
}
@Test
public void testThatPercolationNotificationWorks() throws IOException, InterruptedException {
SimpleNotifier notifier = new SimpleNotifier();
MetricFilter percolationFilter =
new MetricFilter() {
@Override
public boolean matches(String name, Metric metric) {
return name.startsWith(prefix + ".foo");
}
};
elasticsearchReporter =
createElasticsearchReporterBuilder()
.percolationFilter(percolationFilter)
.percolationNotifier(notifier)
.build();
final Counter evictions = registry.counter("foo");
evictions.inc(18);
reportAndRefresh();
QueryBuilder queryBuilder =
QueryBuilders.boolQuery()
.must(QueryBuilders.matchAllQuery())
.filter(
QueryBuilders.boolQuery()
.must(QueryBuilders.rangeQuery("count").gte(20))
.must(QueryBuilders.termQuery("name", prefix + ".foo")));
String json = String.format("{ \"query\" : %s }", queryBuilder.buildAsBytes().toUtf8());
client()
.prepareIndex(indexWithDate, ".percolator", "myName")
.setRefresh(true)
.setSource(json)
.execute()
.actionGet();
evictions.inc(1);
reportAndRefresh();
assertThat(notifier.metrics.size(), is(0));
evictions.inc(2);
reportAndRefresh();
org.assertj.core.api.Assertions.assertThat(notifier.metrics.size()).isEqualTo(1);
org.assertj.core.api.Assertions.assertThat(notifier.metrics).containsKey("myName");
org.assertj.core.api.Assertions.assertThat(notifier.metrics.get("myName").name())
.isEqualTo(prefix + ".foo");
notifier.metrics.clear();
evictions.dec(2);
reportAndRefresh();
org.assertj.core.api.Assertions.assertThat(notifier.metrics.size()).isEqualTo(0);
}
@Override
public void run() {
Timer.Context rollupTimerContext = rollupTimer.time();
try {
Rollup.Type rollupComputer =
RollupRunnable.getRollupComputer(RollupType.BF_BASIC, Granularity.FULL);
Rollup rollup = rollupComputer.compute(points);
writer.enqueueRollupForWrite(
new SingleRollupWriteContext(
rollup, loc, Granularity.MIN_5, CassandraModel.CF_METRICS_5M, range.getStart()));
log.info("Calculated and queued rollup for " + loc + " within range " + range);
} catch (Exception e) {
// I want to be very harsh with exceptions encountered while validating and computing rollups.
// Just stop everything.
log.error("Error encountered while validating and calculating rollups", e);
rollupValidationAndComputeFailed.inc();
RollupGenerator.rollupExecutors.shutdownNow();
OutOFBandRollup.getRollupGeneratorThread().interrupt();
// Stop the monitoring thread
OutOFBandRollup.getMonitoringThread().interrupt();
// Stop the file handler thread pool from sending data to buildstore
FileHandler.handlerThreadPool.shutdownNow();
throw new RuntimeException(e);
} finally {
rollupTimerContext.stop();
}
}
@Override
protected void onComplete(Request request, Response response) {
activeRequestsCounter.dec();
Long duration =
computeDurationMillis(START_TIMES_BY_CORRELATION_ID.remove(request.getCorrelationId()));
if (duration != null && duration.longValue() > 0) {
allTimesTimer.update(duration, TimeUnit.MILLISECONDS);
String name = getRouteName(request);
if (name == null || name.isEmpty()) return;
ROUTE_TIMERS.get(name).update(duration, TimeUnit.MILLISECONDS);
}
Counter responseCounter = COUNTERS_BY_RESPONSE.get(response.getResponseStatus().getCode());
if (responseCounter == null) {
responseCounter = metrics.counter(getResponseCounterName(response.getResponseStatus()));
COUNTERS_BY_RESPONSE.putIfAbsent(response.getResponseStatus().getCode(), responseCounter);
}
responseCounter.inc();
publish(request, response, duration);
}
@Override
public Allocation allocate(
ApplicationAttemptId attemptId,
List<ResourceRequest> resourceRequests,
List<ContainerId> containerIds,
List<String> strings,
List<String> strings2) {
if (metricsON) {
final Timer.Context context = schedulerAllocateTimer.time();
Allocation allocation = null;
try {
allocation =
scheduler.allocate(attemptId, resourceRequests, containerIds, strings, strings2);
return allocation;
} finally {
context.stop();
schedulerAllocateCounter.inc();
try {
updateQueueWithAllocateRequest(allocation, attemptId, resourceRequests, containerIds);
} catch (IOException e) {
e.printStackTrace();
}
}
} else {
return scheduler.allocate(attemptId, resourceRequests, containerIds, strings, strings2);
}
}
public ExecuteResult execute(Tuple input, OutputCollector outputCollector) {
if (tuplesReceived != null) {
tuplesReceived.inc();
}
String docId = input.getString(0);
Object docObj = input.getValue(1);
if (docId == null || docObj == null) {
log.warn("Ignored tuple: " + input);
return ExecuteResult.IGNORED; // nothing to index
}
try {
return processInputDoc(docId, docObj);
} catch (Exception exc) {
log.error("Failed to process " + docId + " due to: " + exc);
if (exc instanceof RuntimeException) {
throw (RuntimeException) exc;
} else {
throw new RuntimeException(exc);
}
}
}
@Test
public void testCounterNoMatch() {
// create timer with id "testMetricAlerts" and register with metric registry, bump up value to
// 4.
Counter c =
MetricsConfigurator.createCounter(metrics, "testCounterNoMatch", PIPELINE_NAME, REVISION);
c.inc(100);
MetricsRuleDefinition metricsRuleDefinition =
new MetricsRuleDefinition(
"testCounterNoMatch",
"testCounterNoMatch",
"testCounterNoMatch",
MetricType.COUNTER,
MetricElement.COUNTER_COUNT,
"${value()>100}",
false,
true);
MetricRuleEvaluator metricRuleEvaluator =
new MetricRuleEvaluator(
metricsRuleDefinition,
metrics,
new AlertManager(
PIPELINE_NAME, REVISION, null, metrics, runtimeInfo, new EventListenerManager()),
Collections.<String>emptyList());
metricRuleEvaluator.checkForAlerts();
// get alert gauge
Gauge<Object> gauge =
MetricsConfigurator.getGauge(
metrics, AlertsUtil.getAlertGaugeName(metricsRuleDefinition.getId()));
Assert.assertNull(gauge);
}
@Override
public void handle(
String path,
Request request,
HttpServletRequest httpRequest,
HttpServletResponse httpResponse)
throws IOException, ServletException {
activeDispatches.inc();
final long start;
final HttpChannelState state = request.getHttpChannelState();
if (state.isInitial()) {
// new request
activeRequests.inc();
start = request.getTimeStamp();
} else {
// resumed request
start = System.currentTimeMillis();
activeSuspended.dec();
if (state.getState() == State.DISPATCHED) {
asyncDispatches.mark();
}
}
try {
super.handle(path, request, httpRequest, httpResponse);
} finally {
final long now = System.currentTimeMillis();
final long dispatched = now - start;
activeDispatches.dec();
dispatches.update(dispatched, TimeUnit.MILLISECONDS);
if (state.isSuspended()) {
if (state.isInitial()) {
state.addListener(listener);
}
activeSuspended.inc();
} else if (state.isInitial()) {
requests.update(dispatched, TimeUnit.MILLISECONDS);
updateResponses(request);
}
// else onCompletion will handle it.
}
}
/**
* Update record-level metrics.
*
* @param recordsWritten number of records written by the writer
* @param branchIndex fork branch index
*/
public void updateRecordMetrics(long recordsWritten, int branchIndex) {
TaskMetrics metrics = TaskMetrics.get(this);
String forkBranchId = ForkOperatorUtils.getForkId(this.taskId, branchIndex);
Counter taskRecordCounter =
metrics.getCounter(gobblin.runtime.util.MetricGroup.TASK.name(), forkBranchId, RECORDS);
long inc = recordsWritten - taskRecordCounter.getCount();
taskRecordCounter.inc(inc);
metrics.getMeter(MetricGroup.TASK.name(), forkBranchId, RECORDS_PER_SECOND).mark(inc);
metrics.getCounter(MetricGroup.JOB.name(), this.jobId, RECORDS).inc(inc);
metrics.getMeter(MetricGroup.JOB.name(), this.jobId, RECORDS_PER_SECOND).mark(inc);
}
@Test
public void testThatBulkIndexingWorks() {
for (int i = 0; i < 2020; i++) {
final Counter evictions = registry.counter(name("foo", "bar", String.valueOf(i)));
evictions.inc(i);
}
reportAndRefresh();
SearchResponse searchResponse =
client().prepareSearch(indexWithDate).setTypes("counter").execute().actionGet();
org.assertj.core.api.Assertions.assertThat(searchResponse.getHits().totalHits())
.isEqualTo(2020L);
}
@Test
public void testCounter() {
System.out.println("******************************* COUNTER *******************************");
counter = registry.counter("counter");
try {
for (int i = 0; i < ITER_COUNT; i++) {
counter.inc(i);
Thread.sleep(SLEEP_MS);
}
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
}
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
throws IOException, ServletException {
final StatusExposingServletResponse wrappedResponse =
new StatusExposingServletResponse((HttpServletResponse) response);
activeRequests.inc();
final Timer.Context context = requestTimer.time();
try {
chain.doFilter(request, wrappedResponse);
} finally {
context.stop();
activeRequests.dec();
markMeterForStatusCode(wrappedResponse.getStatus());
}
}
@Test
public void testCounter() throws Exception {
final Counter evictions = registry.counter(name("test", "cache-evictions"));
evictions.inc(25);
reportAndRefresh();
SearchResponse searchResponse =
client().prepareSearch(indexWithDate).setTypes("counter").execute().actionGet();
org.assertj.core.api.Assertions.assertThat(searchResponse.getHits().totalHits()).isEqualTo(1L);
Map<String, Object> hit = searchResponse.getHits().getAt(0).sourceAsMap();
assertTimestamp(hit);
assertKey(hit, "count", 25);
assertKey(hit, "name", prefix + ".test.cache-evictions");
assertKey(hit, "host", "localhost");
}
protected ExecuteResult flushBufferedDocs(DocBuffer b) {
int numDocsInBatch = b.buffer.size();
if (numDocsInBatch == 0) {
b.reset();
return ExecuteResult.ACK;
}
Timer.Context timer = (sendBatchToSolr != null) ? sendBatchToSolr.time() : null;
try {
sendBatchToSolr(b);
} finally {
if (timer != null) timer.stop();
if (indexedCounter != null) indexedCounter.inc(numDocsInBatch);
b.reset();
}
return ExecuteResult.ACK;
}
public static void main(String[] args) {
ReporterV08 reporter = null;
try {
final MetricRegistry registry = new MetricRegistry();
reporter = getInfluxdbReporter(registry);
reporter.start(3, TimeUnit.SECONDS);
final Counter counter = registry.counter(MetricRegistry.name("test", "counter"));
for (int i = 0; i < 1; ++i) {
counter.inc();
Thread.sleep(Math.round(Math.random()) * 1000);
}
} catch (Exception e) {
e.printStackTrace();
System.exit(1);
} finally {
if (reporter != null) {
reporter.report();
reporter.stop();
}
}
}
/** Tests GetAllMetrics method. */
@Test
public void testGetAllMetrics() {
Counter onosCounter = new Counter();
onosCounter.inc();
Meter onosMeter = new Meter();
onosMeter.mark();
Timer onosTimer = new Timer();
onosTimer.update(1, TimeUnit.MILLISECONDS);
ImmutableMap<String, Metric> metrics =
new ImmutableMap.Builder<String, Metric>()
.put("onosCounter", onosCounter)
.put("onosMeter", onosMeter)
.put("onosTimer", onosTimer)
.build();
expect(mockMetricsService.getMetrics()).andReturn(metrics).anyTimes();
replay(mockMetricsService);
WebTarget wt = target();
String response = wt.path("metrics").request().get(String.class);
assertThat(response, containsString("{\"metrics\":["));
JsonObject result = Json.parse(response).asObject();
assertThat(result, notNullValue());
JsonArray jsonMetrics = result.get("metrics").asArray();
assertThat(jsonMetrics, notNullValue());
assertThat(jsonMetrics.size(), is(3));
assertTrue(
matchesMetric(metrics.get("onosCounter")).matchesSafely(jsonMetrics.get(0).asObject()));
assertTrue(
matchesMetric(metrics.get("onosMeter")).matchesSafely(jsonMetrics.get(1).asObject()));
assertTrue(
matchesMetric(metrics.get("onosTimer")).matchesSafely(jsonMetrics.get(2).asObject()));
}
public void incBytesReadLocal(long n) {
mBytesReadLocal.inc(n);
}
public void incBlocksWrittenLocal(long n) {
mBlocksWrittenLocal.inc(n);
}
public void incPredictionsFailures() {
predictionsFailures.inc();
}
public void incPredictionsSuccesses() {
predictionsSuccess.inc();
}
public void incMetadataFailures() {
metadataFailures.inc();
}
public void incMetadataSuccesses() {
metadataSuccess.inc();
}
public void incMatchesFailures() {
matchesFailures.inc();
}
public void incMatchesSuccesses() {
matchesSuccess.inc();
}
@Override
protected void onReceived(Request request, Response response) {
activeRequestsCounter.inc();
START_TIMES_BY_CORRELATION_ID.put(request.getCorrelationId(), System.nanoTime());
}
@Override
public ActionFuture<SearchResponse> search(SearchRequest request) {
searchRequests.inc();
return super.search(request);
}
private ExtensionResponse tryExecuteGremlinScript(
final RexsterResourceContext rexsterResourceContext,
final Graph graph,
final Vertex vertex,
final Edge edge,
final String script) {
final MetricRegistry metricRegistry = rexsterResourceContext.getMetricRegistry();
final Timer scriptTimer = metricRegistry.timer(MetricRegistry.name("http", "script-engine"));
final Counter successfulExecutions =
metricRegistry.counter(MetricRegistry.name("http", "script-engine", "success"));
final Counter failedExecutions =
metricRegistry.counter(MetricRegistry.name("http", "script-engine", "fail"));
ExtensionResponse extensionResponse;
final JSONObject requestObject = rexsterResourceContext.getRequestObject();
// can't initialize this statically because the configure() method won't get called before it.
// need to think a bit on how to best initialized the controller.
final EngineController engineController = EngineController.getInstance();
final boolean showTypes = RequestObjectHelper.getShowTypes(requestObject);
final long offsetStart = RequestObjectHelper.getStartOffset(requestObject);
final long offsetEnd = RequestObjectHelper.getEndOffset(requestObject);
final GraphSONMode mode = showTypes ? GraphSONMode.EXTENDED : GraphSONMode.NORMAL;
final Set<String> returnKeys = RequestObjectHelper.getReturnKeys(requestObject, WILDCARD);
final String languageToExecuteWith = getLanguageToExecuteWith(requestObject);
final EngineHolder engineHolder;
final ScriptEngine scriptEngine;
try {
if (!engineController.isEngineAvailable(languageToExecuteWith)) {
return ExtensionResponse.error("language requested is not available on the server");
}
engineHolder = engineController.getEngineByLanguageName(languageToExecuteWith);
scriptEngine = engineHolder.getEngine();
} catch (ScriptException se) {
return ExtensionResponse.error("could not get request script engine");
}
final Bindings bindings = createBindings(graph, vertex, edge, scriptEngine);
// add all keys not defined by this request as bindings to the script engine
placeParametersOnBinding(requestObject, bindings, showTypes);
// get the list of "stored procedures" to run
final RexsterApplicationGraph rag = rexsterResourceContext.getRexsterApplicationGraph();
final ExtensionMethod extensionMethod = rexsterResourceContext.getExtensionMethod();
Map configurationMap = null;
Iterator<String> scriptsToRun = null;
try {
final ExtensionConfiguration extensionConfiguration =
rag != null ? rag.findExtensionConfiguration(EXTENSION_NAMESPACE, EXTENSION_NAME) : null;
if (extensionConfiguration != null) {
configurationMap = extensionConfiguration.tryGetMapFromConfiguration();
scriptsToRun = getScriptsToRun(requestObject, configurationMap);
}
} catch (IOException ioe) {
return ExtensionResponse.error(
ioe, generateErrorJson(extensionMethod.getExtensionApiAsJson()));
}
if ((script == null || script.isEmpty()) && scriptsToRun == null) {
return ExtensionResponse.badRequest(
"no scripts provided", generateErrorJson(extensionMethod.getExtensionApiAsJson()));
}
final Timer.Context context = scriptTimer.time();
try {
// result is either the ad-hoc script on the query string or the last "stored procedure"
Object result = null;
if (scriptsToRun != null) {
while (scriptsToRun.hasNext()) {
result = engineHolder.getEngine().eval(scriptsToRun.next(), bindings);
}
}
if (isClientScriptAllowed(configurationMap) && script != null && !script.isEmpty()) {
result = engineHolder.getEngine().eval(script, bindings);
}
final JSONArray results =
new JSONResultConverter(mode, offsetStart, offsetEnd, returnKeys).convert(result);
final HashMap<String, Object> resultMap = new HashMap<String, Object>();
resultMap.put(Tokens.SUCCESS, true);
resultMap.put(Tokens.RESULTS, results);
final JSONObject resultObject = new JSONObject(resultMap);
extensionResponse = ExtensionResponse.ok(resultObject);
successfulExecutions.inc();
} catch (Exception e) {
logger.error(String.format("Gremlin Extension: %s", e.getMessage()), e);
extensionResponse =
ExtensionResponse.error(e, generateErrorJson(extensionMethod.getExtensionApiAsJson()));
failedExecutions.inc();
} finally {
context.stop();
}
return extensionResponse;
}
