@Test
public void queueRejectionWithBlockingQueue() throws Exception {
final int queueMaxSize = 5;
final TenacityConfiguration exampleConfiguration =
new TenacityConfiguration(
new ThreadPoolConfiguration(1, 1, 10, queueMaxSize, 10000, 10),
new CircuitBreakerConfiguration(20, 5000, 50, 10000, 10),
5000);
final TenacityPropertyRegister tenacityPropertyRegister =
new TenacityPropertyRegister(
ImmutableMap.<TenacityPropertyKey, TenacityConfiguration>of(
DependencyKey.SLEEP, exampleConfiguration),
new BreakerboxConfiguration(),
mock(ArchaiusPropertyRegister.class));
tenacityPropertyRegister.register();
final ImmutableList.Builder<Future<Optional<String>>> sleepCommands = ImmutableList.builder();
for (int i = 0; i < queueMaxSize * 2; i++) {
sleepCommands.add(new SleepCommand(DependencyKey.SLEEP).queue());
}
for (Future<Optional<String>> future : sleepCommands.build()) {
assertFalse(future.isCancelled());
final Optional<String> result = future.get();
if (result.isPresent()) {
assertThat(result.get()).isEqualTo("sleep");
} else {
assertThat(result).isEqualTo(Optional.<String>absent());
}
}
final HystrixCommandMetrics sleepCommandMetrics =
new SleepCommand(DependencyKey.SLEEP).getCommandMetrics();
assertThat(
sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED))
.isEqualTo(4);
assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.TIMEOUT))
.isEqualTo(0);
assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS))
.isEqualTo(4);
assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.SHORT_CIRCUITED))
.isEqualTo(0);
final HystrixThreadPoolProperties threadPoolProperties =
new SleepCommand(DependencyKey.SLEEP).getThreadpoolProperties();
final ThreadPoolConfiguration threadPoolConfiguration = exampleConfiguration.getThreadpool();
assertEquals(
threadPoolProperties.queueSizeRejectionThreshold().get().intValue(),
threadPoolConfiguration.getQueueSizeRejectionThreshold());
assertEquals(
threadPoolProperties.maxQueueSize().get().intValue(),
threadPoolConfiguration.getMaxQueueSize());
assertEquals(
TenacityPropertyStore.getTenacityConfiguration(DependencyKey.SLEEP), exampleConfiguration);
}
@Before
public void init() {
for (HystrixCommandMetrics metricsInstance : HystrixCommandMetrics.getInstances()) {
metricsInstance.resetStream();
}
HystrixCommandMetrics.reset();
HystrixCircuitBreaker.Factory.reset();
Hystrix.reset();
}
@Override
public boolean isOpen() {
System.out.println(
"metrics : " + metrics.getCommandKey().name() + " : " + metrics.getHealthCounts());
if (forceShortCircuit) {
return true;
} else {
return metrics.getHealthCounts().getErrorCount() >= 3;
}
}
@Override
public void run() {
try {
for (HystrixCommandMetrics commandMetrics : HystrixCommandMetrics.getInstances()) {
String jsonString = getCommandJson(commandMetrics);
listener.handleJsonMetric(jsonString);
}
for (HystrixThreadPoolMetrics threadPoolMetrics : HystrixThreadPoolMetrics.getInstances()) {
if (hasExecutedCommandsOnThread(threadPoolMetrics)) {
String jsonString = getThreadPoolJson(threadPoolMetrics);
listener.handleJsonMetric(jsonString);
}
}
for (HystrixCollapserMetrics collapserMetrics : HystrixCollapserMetrics.getInstances()) {
String jsonString = getCollapserJson(collapserMetrics);
listener.handleJsonMetric(jsonString);
}
} catch (Exception e) {
logger.warn("Failed to output metrics as JSON", e);
// shutdown
pause();
return;
}
}
/** Utility method for creating {@link HystrixCommandMetrics} for unit tests. */
private static HystrixCommandMetrics getMetrics(HystrixCommandProperties.Setter properties) {
return HystrixCommandMetrics.getInstance(
CommandKeyForUnitTest.KEY_ONE,
CommandOwnerForUnitTest.OWNER_ONE,
ThreadPoolKeyForUnitTest.THREAD_POOL_ONE,
HystrixCommandPropertiesTest.asMock(properties));
}
private <T> void logHC(HystrixInvokable<T> command, T response) {
if (command instanceof HystrixInvokableInfo) {
HystrixInvokableInfo<T> commandInfo = (HystrixInvokableInfo<T>) command;
HystrixCommandMetrics metrics = commandInfo.getMetrics();
System.out.println(
"cb/error-count/%/total: "
+ commandInfo.isCircuitBreakerOpen()
+ " "
+ metrics.getHealthCounts().getErrorCount()
+ " "
+ metrics.getHealthCounts().getErrorPercentage()
+ " "
+ metrics.getHealthCounts().getTotalRequests()
+ " => "
+ response
+ " "
+ commandInfo.getExecutionEvents());
}
}
@Test
public void queueRejectionWithSynchronousQueue() throws Exception {
final ImmutableCollection.Builder<Future<Optional<String>>> futures = ImmutableList.builder();
for (int i = 0; i < 50; i++) {
futures.add(new SleepCommand(DependencyKey.EXAMPLE).queue());
}
for (Future<Optional<String>> future : futures.build()) {
assertFalse(future.isCancelled());
final Optional<String> result = future.get();
if (result.isPresent()) {
assertThat(result.get()).isEqualTo("sleep");
} else {
assertThat(result).isEqualTo(Optional.<String>absent());
}
}
final HystrixCommandMetrics sleepCommandMetrics =
new SleepCommand(DependencyKey.EXAMPLE).getCommandMetrics();
assertThat(
sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED))
.isGreaterThan(15);
assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.TIMEOUT))
.isEqualTo(0);
assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS))
.isGreaterThanOrEqualTo(40);
assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.FALLBACK_FAILURE))
.isEqualTo(0);
assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.FALLBACK_REJECTION))
.isEqualTo(0);
assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.SHORT_CIRCUITED))
.isEqualTo(
sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)
- sleepCommandMetrics.getCumulativeCount(
HystrixRollingNumberEvent.THREAD_POOL_REJECTED));
final HystrixThreadPoolProperties threadPoolProperties =
new SleepCommand(DependencyKey.EXAMPLE).getThreadpoolProperties();
// -1 means no limit on the number of items in the queue, which uses the
// SynchronousBlockingQueue
assertEquals(threadPoolProperties.maxQueueSize().get().intValue(), -1);
assertEquals(
TenacityPropertyStore.getTenacityConfiguration(DependencyKey.EXAMPLE),
new TenacityConfiguration());
}
@Test
public void should_gather_json_metrics() throws Exception {
HystrixCommandKey hystrixCommandKey = HystrixCommandKey.Factory.asKey("commandKey");
HystrixCommandMetrics.getInstance(
hystrixCommandKey,
HystrixCommandGroupKey.Factory.asKey("commandGroupKey"),
new HystrixPropertiesCommandDefault(
hystrixCommandKey, HystrixCommandProperties.defaultSetter()));
given(this.discoveryClient.getLocalServiceInstance()).willReturn(this.serviceInstance);
this.hystrixStreamTask.gatherMetrics();
assertThat(this.hystrixStreamTask.jsonMetrics.isEmpty(), is(false));
}
@Override
public Integer call() {
return metrics.getHealthCounts().getErrorPercentage();
}
@Override
public Long call() {
return metrics.getRollingMaxConcurrentExecutions();
}
@Override
public Integer call() {
return metrics.getCurrentConcurrentExecutionCount();
}
private String getCommandJson(final HystrixCommandMetrics commandMetrics) throws IOException {
HystrixCommandKey key = commandMetrics.getCommandKey();
HystrixCircuitBreaker circuitBreaker = HystrixCircuitBreaker.Factory.getInstance(key);
StringWriter jsonString = new StringWriter();
JsonGenerator json = jsonFactory.createGenerator(jsonString);
json.writeStartObject();
json.writeStringField("type", "HystrixCommand");
json.writeStringField("name", key.name());
json.writeStringField("group", commandMetrics.getCommandGroup().name());
json.writeNumberField("currentTime", System.currentTimeMillis());
// circuit breaker
if (circuitBreaker == null) {
// circuit breaker is disabled and thus never open
json.writeBooleanField("isCircuitBreakerOpen", false);
} else {
json.writeBooleanField("isCircuitBreakerOpen", circuitBreaker.isOpen());
}
HealthCounts healthCounts = commandMetrics.getHealthCounts();
json.writeNumberField("errorPercentage", healthCounts.getErrorPercentage());
json.writeNumberField("errorCount", healthCounts.getErrorCount());
json.writeNumberField("requestCount", healthCounts.getTotalRequests());
// rolling counters
safelyWriteNumberField(
json,
"rollingCountBadRequests",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.BAD_REQUEST);
}
});
safelyWriteNumberField(
json,
"rollingCountCollapsedRequests",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.COLLAPSED);
}
});
safelyWriteNumberField(
json,
"rollingCountEmit",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.EMIT);
}
});
safelyWriteNumberField(
json,
"rollingCountExceptionsThrown",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.EXCEPTION_THROWN);
}
});
safelyWriteNumberField(
json,
"rollingCountFailure",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.FAILURE);
}
});
safelyWriteNumberField(
json,
"rollingCountFallbackEmit",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.FALLBACK_EMIT);
}
});
safelyWriteNumberField(
json,
"rollingCountFallbackFailure",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.FALLBACK_FAILURE);
}
});
safelyWriteNumberField(
json,
"rollingCountFallbackMissing",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.FALLBACK_MISSING);
}
});
safelyWriteNumberField(
json,
"rollingCountFallbackRejection",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.FALLBACK_REJECTION);
}
});
safelyWriteNumberField(
json,
"rollingCountFallbackSuccess",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.FALLBACK_SUCCESS);
}
});
safelyWriteNumberField(
json,
"rollingCountResponsesFromCache",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.RESPONSE_FROM_CACHE);
}
});
safelyWriteNumberField(
json,
"rollingCountSemaphoreRejected",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.SEMAPHORE_REJECTED);
}
});
safelyWriteNumberField(
json,
"rollingCountShortCircuited",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.SHORT_CIRCUITED);
}
});
safelyWriteNumberField(
json,
"rollingCountSuccess",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.SUCCESS);
}
});
safelyWriteNumberField(
json,
"rollingCountThreadPoolRejected",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.THREAD_POOL_REJECTED);
}
});
safelyWriteNumberField(
json,
"rollingCountTimeout",
new Func0<Long>() {
@Override
public Long call() {
return commandMetrics.getRollingCount(HystrixEventType.TIMEOUT);
}
});
json.writeNumberField(
"currentConcurrentExecutionCount", commandMetrics.getCurrentConcurrentExecutionCount());
json.writeNumberField(
"rollingMaxConcurrentExecutionCount", commandMetrics.getRollingMaxConcurrentExecutions());
// latency percentiles
json.writeNumberField("latencyExecute_mean", commandMetrics.getExecutionTimeMean());
json.writeObjectFieldStart("latencyExecute");
json.writeNumberField("0", commandMetrics.getExecutionTimePercentile(0));
json.writeNumberField("25", commandMetrics.getExecutionTimePercentile(25));
json.writeNumberField("50", commandMetrics.getExecutionTimePercentile(50));
json.writeNumberField("75", commandMetrics.getExecutionTimePercentile(75));
json.writeNumberField("90", commandMetrics.getExecutionTimePercentile(90));
json.writeNumberField("95", commandMetrics.getExecutionTimePercentile(95));
json.writeNumberField("99", commandMetrics.getExecutionTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.getExecutionTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.getExecutionTimePercentile(100));
json.writeEndObject();
//
json.writeNumberField("latencyTotal_mean", commandMetrics.getTotalTimeMean());
json.writeObjectFieldStart("latencyTotal");
json.writeNumberField("0", commandMetrics.getTotalTimePercentile(0));
json.writeNumberField("25", commandMetrics.getTotalTimePercentile(25));
json.writeNumberField("50", commandMetrics.getTotalTimePercentile(50));
json.writeNumberField("75", commandMetrics.getTotalTimePercentile(75));
json.writeNumberField("90", commandMetrics.getTotalTimePercentile(90));
json.writeNumberField("95", commandMetrics.getTotalTimePercentile(95));
json.writeNumberField("99", commandMetrics.getTotalTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.getTotalTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.getTotalTimePercentile(100));
json.writeEndObject();
// property values for reporting what is actually seen by the command rather than what was set
// somewhere
HystrixCommandProperties commandProperties = commandMetrics.getProperties();
json.writeNumberField(
"propertyValue_circuitBreakerRequestVolumeThreshold",
commandProperties.circuitBreakerRequestVolumeThreshold().get());
json.writeNumberField(
"propertyValue_circuitBreakerSleepWindowInMilliseconds",
commandProperties.circuitBreakerSleepWindowInMilliseconds().get());
json.writeNumberField(
"propertyValue_circuitBreakerErrorThresholdPercentage",
commandProperties.circuitBreakerErrorThresholdPercentage().get());
json.writeBooleanField(
"propertyValue_circuitBreakerForceOpen",
commandProperties.circuitBreakerForceOpen().get());
json.writeBooleanField(
"propertyValue_circuitBreakerForceClosed",
commandProperties.circuitBreakerForceClosed().get());
json.writeBooleanField(
"propertyValue_circuitBreakerEnabled", commandProperties.circuitBreakerEnabled().get());
json.writeStringField(
"propertyValue_executionIsolationStrategy",
commandProperties.executionIsolationStrategy().get().name());
json.writeNumberField(
"propertyValue_executionIsolationThreadTimeoutInMilliseconds",
commandProperties.executionTimeoutInMilliseconds().get());
json.writeNumberField(
"propertyValue_executionTimeoutInMilliseconds",
commandProperties.executionTimeoutInMilliseconds().get());
json.writeBooleanField(
"propertyValue_executionIsolationThreadInterruptOnTimeout",
commandProperties.executionIsolationThreadInterruptOnTimeout().get());
json.writeStringField(
"propertyValue_executionIsolationThreadPoolKeyOverride",
commandProperties.executionIsolationThreadPoolKeyOverride().get());
json.writeNumberField(
"propertyValue_executionIsolationSemaphoreMaxConcurrentRequests",
commandProperties.executionIsolationSemaphoreMaxConcurrentRequests().get());
json.writeNumberField(
"propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests",
commandProperties.fallbackIsolationSemaphoreMaxConcurrentRequests().get());
/*
* The following are commented out as these rarely change and are verbose for streaming for something people don't change.
* We could perhaps allow a property or request argument to include these.
*/
// json.put("propertyValue_metricsRollingPercentileEnabled",
// commandProperties.metricsRollingPercentileEnabled().get());
// json.put("propertyValue_metricsRollingPercentileBucketSize",
// commandProperties.metricsRollingPercentileBucketSize().get());
// json.put("propertyValue_metricsRollingPercentileWindow",
// commandProperties.metricsRollingPercentileWindowInMilliseconds().get());
// json.put("propertyValue_metricsRollingPercentileWindowBuckets",
// commandProperties.metricsRollingPercentileWindowBuckets().get());
// json.put("propertyValue_metricsRollingStatisticalWindowBuckets",
// commandProperties.metricsRollingStatisticalWindowBuckets().get());
json.writeNumberField(
"propertyValue_metricsRollingStatisticalWindowInMilliseconds",
commandProperties.metricsRollingStatisticalWindowInMilliseconds().get());
json.writeBooleanField(
"propertyValue_requestCacheEnabled", commandProperties.requestCacheEnabled().get());
json.writeBooleanField(
"propertyValue_requestLogEnabled", commandProperties.requestLogEnabled().get());
json.writeNumberField(
"reportingHosts", 1); // this will get summed across all instances in a cluster
json.writeStringField("threadPool", commandMetrics.getThreadPoolKey().name());
json.writeEndObject();
json.close();
return jsonString.getBuffer().toString();
}
@Override
public void run() {
try {
// command metrics
for (HystrixCommandMetrics commandMetrics : HystrixCommandMetrics.getInstances()) {
HystrixCommandKey key = commandMetrics.getCommandKey();
HystrixCircuitBreaker circuitBreaker = HystrixCircuitBreaker.Factory.getInstance(key);
StringWriter jsonString = new StringWriter();
JsonGenerator json = jsonFactory.createJsonGenerator(jsonString);
// Informational and Status
json.writeStartObject();
json.writeStringField(type.value, HystrixCommand.value);
json.writeStringField(name.value, key.name());
json.writeStringField(group.value, commandMetrics.getCommandGroup().name());
json.writeNumberField(currentTime.value, (int) (System.currentTimeMillis() / 1000));
// circuit breaker
json.writeBooleanField(isCircuitBreakerOpen.value, circuitBreaker.isOpen());
HystrixCommandMetrics.HealthCounts healthCounts = commandMetrics.getHealthCounts();
json.writeNumberField(errorPercentage.value, healthCounts.getErrorPercentage());
json.writeNumberField(errorCount.value, healthCounts.getErrorCount());
json.writeNumberField(requestCount.value, healthCounts.getTotalRequests());
// rolling counters Gauge
json.writeNumberField(
rollingCountCollapsedRequests.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.COLLAPSED));
json.writeNumberField(
rollingCountExceptionsThrown.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN));
json.writeNumberField(
rollingCountFailure.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.FAILURE));
json.writeNumberField(
rollingCountFallbackFailure.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE));
json.writeNumberField(
rollingCountFallbackRejection.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION));
json.writeNumberField(
rollingCountFallbackSuccess.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS));
json.writeNumberField(
rollingCountResponsesFromCache.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE));
json.writeNumberField(
rollingCountSemaphoreRejected.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED));
json.writeNumberField(
rollingCountShortCircuited.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED));
json.writeNumberField(
rollingCountSuccess.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS));
json.writeNumberField(
rollingCountThreadPoolRejected.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED));
json.writeNumberField(
rollingCountTimeout.value,
commandMetrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT));
json.writeNumberField(
currentConcurrentExecutionCount.value,
commandMetrics.getCurrentConcurrentExecutionCount());
// latency percentiles
json.writeNumberField(latencyExecute_mean.value, commandMetrics.getExecutionTimeMean());
json.writeObjectFieldStart(latencyExecute.value);
json.writeNumberField("0", commandMetrics.getExecutionTimePercentile(0));
json.writeNumberField("25", commandMetrics.getExecutionTimePercentile(25));
json.writeNumberField("50", commandMetrics.getExecutionTimePercentile(50));
json.writeNumberField("75", commandMetrics.getExecutionTimePercentile(75));
json.writeNumberField("90", commandMetrics.getExecutionTimePercentile(90));
json.writeNumberField("95", commandMetrics.getExecutionTimePercentile(95));
json.writeNumberField("99", commandMetrics.getExecutionTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.getExecutionTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.getExecutionTimePercentile(100));
json.writeEndObject();
//
json.writeNumberField(latencyTotal_mean.value, commandMetrics.getTotalTimeMean());
json.writeObjectFieldStart(latencyTotal.value);
json.writeNumberField("0", commandMetrics.getTotalTimePercentile(0));
json.writeNumberField("25", commandMetrics.getTotalTimePercentile(25));
json.writeNumberField("50", commandMetrics.getTotalTimePercentile(50));
json.writeNumberField("75", commandMetrics.getTotalTimePercentile(75));
json.writeNumberField("90", commandMetrics.getTotalTimePercentile(90));
json.writeNumberField("95", commandMetrics.getTotalTimePercentile(95));
json.writeNumberField("99", commandMetrics.getTotalTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.getTotalTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.getTotalTimePercentile(100));
json.writeEndObject();
// property values for reporting what is actually seen by the command rather than what was
// set somewhere
HystrixCommandProperties commandProperties = commandMetrics.getProperties();
json.writeNumberField(
propertyValue_circuitBreakerRequestVolumeThreshold.value,
commandProperties.circuitBreakerRequestVolumeThreshold().get());
json.writeNumberField(
propertyValue_circuitBreakerSleepWindowInMilliseconds.value,
commandProperties.circuitBreakerSleepWindowInMilliseconds().get());
json.writeNumberField(
propertyValue_circuitBreakerErrorThresholdPercentage.value,
commandProperties.circuitBreakerErrorThresholdPercentage().get());
json.writeBooleanField(
propertyValue_circuitBreakerForceOpen.value,
commandProperties.circuitBreakerForceOpen().get());
json.writeBooleanField(
propertyValue_circuitBreakerForceClosed.value,
commandProperties.circuitBreakerForceClosed().get());
json.writeBooleanField(
propertyValue_circuitBreakerEnabled.value,
commandProperties.circuitBreakerEnabled().get());
json.writeStringField(
propertyValue_executionIsolationStrategy.value,
commandProperties.executionIsolationStrategy().get().name());
json.writeNumberField(
propertyValue_executionIsolationThreadTimeoutInMilliseconds.value,
commandProperties.executionIsolationThreadTimeoutInMilliseconds().get());
json.writeBooleanField(
propertyValue_executionIsolationThreadInterruptOnTimeout.value,
commandProperties.executionIsolationThreadInterruptOnTimeout().get());
json.writeStringField(
propertyValue_executionIsolationThreadPoolKeyOverride.value,
commandProperties.executionIsolationThreadPoolKeyOverride().get());
json.writeNumberField(
propertyValue_executionIsolationSemaphoreMaxConcurrentRequests.value,
commandProperties.executionIsolationSemaphoreMaxConcurrentRequests().get());
json.writeNumberField(
propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests.value,
commandProperties.fallbackIsolationSemaphoreMaxConcurrentRequests().get());
/*
* The following are commented out as these rarely change and are verbose for streaming for something people don't change.
* We could perhaps allow a property or request argument to include these.
*/
// json.put("propertyValue_metricsRollingPercentileEnabled",
// commandProperties.metricsRollingPercentileEnabled().get());
// json.put("propertyValue_metricsRollingPercentileBucketSize",
// commandProperties.metricsRollingPercentileBucketSize().get());
// json.put("propertyValue_metricsRollingPercentileWindow",
// commandProperties.metricsRollingPercentileWindowInMilliseconds().get());
// json.put("propertyValue_metricsRollingPercentileWindowBuckets",
// commandProperties.metricsRollingPercentileWindowBuckets().get());
// json.put("propertyValue_metricsRollingStatisticalWindowBuckets",
// commandProperties.metricsRollingStatisticalWindowBuckets().get());
json.writeNumberField(
propertyValue_metricsRollingStatisticalWindowInMilliseconds.value,
commandProperties.metricsRollingStatisticalWindowInMilliseconds().get());
json.writeBooleanField(
propertyValue_requestCacheEnabled.value, commandProperties.requestCacheEnabled().get());
json.writeBooleanField(
propertyValue_requestLogEnabled.value, commandProperties.requestLogEnabled().get());
json.writeNumberField(reportingHosts.value, 1);
json.writeStringField(Key.ip.value, app_ip);
json.writeEndObject();
json.close();
// System.out.println(ip + ":" + port + "||" +
// jsonString.getBuffer().toString());
UDPClient.send(
socketIp, socketPort, jsonString.getBuffer().toString().getBytes(), new byte[] {});
}
// thread pool metrics
for (HystrixThreadPoolMetrics threadPoolMetrics : HystrixThreadPoolMetrics.getInstances()) {
HystrixThreadPoolKey key = threadPoolMetrics.getThreadPoolKey();
StringWriter jsonString = new StringWriter();
JsonGenerator json = jsonFactory.createJsonGenerator(jsonString);
json.writeStartObject();
json.writeStringField(type.value, HystrixThreadPool.value);
json.writeStringField(name.value, key.name());
json.writeNumberField(currentTime.value, System.currentTimeMillis());
// 101.3 80 154
json.writeNumberField(
currentActiveCount.value, threadPoolMetrics.getCurrentActiveCount().intValue());
json.writeNumberField(
currentCompletedTaskCount.value,
threadPoolMetrics.getCurrentCompletedTaskCount().longValue());
json.writeNumberField(
currentCorePoolSize.value, threadPoolMetrics.getCurrentCorePoolSize().intValue());
json.writeNumberField(
currentLargestPoolSize.value, threadPoolMetrics.getCurrentLargestPoolSize().intValue());
json.writeNumberField(
currentMaximumPoolSize.value, threadPoolMetrics.getCurrentMaximumPoolSize().intValue());
json.writeNumberField(
currentPoolSize.value, threadPoolMetrics.getCurrentPoolSize().intValue());
json.writeNumberField(
currentQueueSize.value, threadPoolMetrics.getCurrentQueueSize().intValue());
json.writeNumberField(
currentTaskCount.value, threadPoolMetrics.getCurrentTaskCount().longValue());
json.writeNumberField(
rollingCountThreadsExecuted.value, threadPoolMetrics.getRollingCountThreadsExecuted());
json.writeNumberField(
rollingMaxActiveThreads.value, threadPoolMetrics.getRollingMaxActiveThreads());
json.writeNumberField(
propertyValue_queueSizeRejectionThreshold.value,
threadPoolMetrics.getProperties().queueSizeRejectionThreshold().get());
json.writeNumberField(
propertyValue_metricsRollingStatisticalWindowInMilliseconds.value,
threadPoolMetrics
.getProperties()
.metricsRollingStatisticalWindowInMilliseconds()
.get());
json.writeNumberField(
reportingHosts.value, 1); // this will get summed across all instances in a cluster
json.writeStringField(Key.ip.value, app_ip);
json.writeEndObject();
json.close();
String str = jsonString.getBuffer().toString();
byte[] ret = str.getBytes();
UDPClient.send(socketIp, socketPort, ret, new byte[] {});
}
} catch (Exception e) {
System.err.println("Failed to output metrics as JSON");
e.printStackTrace();
}
}
@Scheduled(fixedRateString = "${hystrix.stream.amqp.gatherRate:500}")
public void gatherMetrics() {
try {
// command metrics
Collection<HystrixCommandMetrics> instances = HystrixCommandMetrics.getInstances();
if (!instances.isEmpty()) {
log.trace("gathering metrics size: " + instances.size());
}
ServiceInstance localService = this.discoveryClient.getLocalServiceInstance();
for (HystrixCommandMetrics commandMetrics : instances) {
HystrixCommandKey key = commandMetrics.getCommandKey();
HystrixCircuitBreaker circuitBreaker = HystrixCircuitBreaker.Factory.getInstance(key);
StringWriter jsonString = new StringWriter();
JsonGenerator json = this.jsonFactory.createGenerator(jsonString);
json.writeStartObject();
addServiceData(json, localService);
json.writeObjectFieldStart("data");
json.writeStringField("type", "HystrixCommand");
String name = key.name();
if (this.properties.isPrefixMetricName()) {
name = localService.getServiceId() + "." + name;
}
json.writeStringField("name", name);
json.writeStringField("group", commandMetrics.getCommandGroup().name());
json.writeNumberField("currentTime", System.currentTimeMillis());
// circuit breaker
if (circuitBreaker == null) {
// circuit breaker is disabled and thus never open
json.writeBooleanField("isCircuitBreakerOpen", false);
} else {
json.writeBooleanField("isCircuitBreakerOpen", circuitBreaker.isOpen());
}
HystrixCommandMetrics.HealthCounts healthCounts = commandMetrics.getHealthCounts();
json.writeNumberField("errorPercentage", healthCounts.getErrorPercentage());
json.writeNumberField("errorCount", healthCounts.getErrorCount());
json.writeNumberField("requestCount", healthCounts.getTotalRequests());
// rolling counters
json.writeNumberField(
"rollingCountCollapsedRequests",
commandMetrics.getRollingCount(HystrixRollingNumberEvent.COLLAPSED));
json.writeNumberField(
"rollingCountExceptionsThrown",
commandMetrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN));
json.writeNumberField(
"rollingCountFailure",
commandMetrics.getRollingCount(HystrixRollingNumberEvent.FAILURE));
json.writeNumberField(
"rollingCountFallbackFailure",
commandMetrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE));
json.writeNumberField(
"rollingCountFallbackRejection",
commandMetrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION));
json.writeNumberField(
"rollingCountFallbackSuccess",
commandMetrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS));
json.writeNumberField(
"rollingCountResponsesFromCache",
commandMetrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE));
json.writeNumberField(
"rollingCountSemaphoreRejected",
commandMetrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED));
json.writeNumberField(
"rollingCountShortCircuited",
commandMetrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED));
json.writeNumberField(
"rollingCountSuccess",
commandMetrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS));
json.writeNumberField(
"rollingCountThreadPoolRejected",
commandMetrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED));
json.writeNumberField(
"rollingCountTimeout",
commandMetrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT));
json.writeNumberField(
"currentConcurrentExecutionCount", commandMetrics.getCurrentConcurrentExecutionCount());
// latency percentiles
json.writeNumberField("latencyExecute_mean", commandMetrics.getExecutionTimeMean());
json.writeObjectFieldStart("latencyExecute");
json.writeNumberField("0", commandMetrics.getExecutionTimePercentile(0));
json.writeNumberField("25", commandMetrics.getExecutionTimePercentile(25));
json.writeNumberField("50", commandMetrics.getExecutionTimePercentile(50));
json.writeNumberField("75", commandMetrics.getExecutionTimePercentile(75));
json.writeNumberField("90", commandMetrics.getExecutionTimePercentile(90));
json.writeNumberField("95", commandMetrics.getExecutionTimePercentile(95));
json.writeNumberField("99", commandMetrics.getExecutionTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.getExecutionTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.getExecutionTimePercentile(100));
json.writeEndObject();
//
json.writeNumberField("latencyTotal_mean", commandMetrics.getTotalTimeMean());
json.writeObjectFieldStart("latencyTotal");
json.writeNumberField("0", commandMetrics.getTotalTimePercentile(0));
json.writeNumberField("25", commandMetrics.getTotalTimePercentile(25));
json.writeNumberField("50", commandMetrics.getTotalTimePercentile(50));
json.writeNumberField("75", commandMetrics.getTotalTimePercentile(75));
json.writeNumberField("90", commandMetrics.getTotalTimePercentile(90));
json.writeNumberField("95", commandMetrics.getTotalTimePercentile(95));
json.writeNumberField("99", commandMetrics.getTotalTimePercentile(99));
json.writeNumberField("99.5", commandMetrics.getTotalTimePercentile(99.5));
json.writeNumberField("100", commandMetrics.getTotalTimePercentile(100));
json.writeEndObject();
// property values for reporting what is actually seen by the command
// rather than what was set somewhere
HystrixCommandProperties commandProperties = commandMetrics.getProperties();
json.writeNumberField(
"propertyValue_circuitBreakerRequestVolumeThreshold",
commandProperties.circuitBreakerRequestVolumeThreshold().get());
json.writeNumberField(
"propertyValue_circuitBreakerSleepWindowInMilliseconds",
commandProperties.circuitBreakerSleepWindowInMilliseconds().get());
json.writeNumberField(
"propertyValue_circuitBreakerErrorThresholdPercentage",
commandProperties.circuitBreakerErrorThresholdPercentage().get());
json.writeBooleanField(
"propertyValue_circuitBreakerForceOpen",
commandProperties.circuitBreakerForceOpen().get());
json.writeBooleanField(
"propertyValue_circuitBreakerForceClosed",
commandProperties.circuitBreakerForceClosed().get());
json.writeBooleanField(
"propertyValue_circuitBreakerEnabled", commandProperties.circuitBreakerEnabled().get());
json.writeStringField(
"propertyValue_executionIsolationStrategy",
commandProperties.executionIsolationStrategy().get().name());
json.writeNumberField(
"propertyValue_executionIsolationThreadTimeoutInMilliseconds",
commandProperties.executionIsolationThreadTimeoutInMilliseconds().get());
json.writeBooleanField(
"propertyValue_executionIsolationThreadInterruptOnTimeout",
commandProperties.executionIsolationThreadInterruptOnTimeout().get());
json.writeStringField(
"propertyValue_executionIsolationThreadPoolKeyOverride",
commandProperties.executionIsolationThreadPoolKeyOverride().get());
json.writeNumberField(
"propertyValue_executionIsolationSemaphoreMaxConcurrentRequests",
commandProperties.executionIsolationSemaphoreMaxConcurrentRequests().get());
json.writeNumberField(
"propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests",
commandProperties.fallbackIsolationSemaphoreMaxConcurrentRequests().get());
// TODO
/*
* The following are commented out as these rarely change and are verbose
* for streaming for something people don't change. We could perhaps allow
* a property or request argument to include these.
*/
// json.put("propertyValue_metricsRollingPercentileEnabled",
// commandProperties.metricsRollingPercentileEnabled().get());
// json.put("propertyValue_metricsRollingPercentileBucketSize",
// commandProperties.metricsRollingPercentileBucketSize().get());
// json.put("propertyValue_metricsRollingPercentileWindow",
// commandProperties.metricsRollingPercentileWindowInMilliseconds().get());
// json.put("propertyValue_metricsRollingPercentileWindowBuckets",
// commandProperties.metricsRollingPercentileWindowBuckets().get());
// json.put("propertyValue_metricsRollingStatisticalWindowBuckets",
// commandProperties.metricsRollingStatisticalWindowBuckets().get());
json.writeNumberField(
"propertyValue_metricsRollingStatisticalWindowInMilliseconds",
commandProperties.metricsRollingStatisticalWindowInMilliseconds().get());
json.writeBooleanField(
"propertyValue_requestCacheEnabled", commandProperties.requestCacheEnabled().get());
json.writeBooleanField(
"propertyValue_requestLogEnabled", commandProperties.requestLogEnabled().get());
json.writeNumberField("reportingHosts", 1); // this will get summed across
// all instances in a cluster
json.writeEndObject(); // end data attribute
json.writeEndObject();
json.close();
// output
this.jsonMetrics.add(jsonString.getBuffer().toString());
}
// thread pool metrics
for (HystrixThreadPoolMetrics threadPoolMetrics : HystrixThreadPoolMetrics.getInstances()) {
HystrixThreadPoolKey key = threadPoolMetrics.getThreadPoolKey();
StringWriter jsonString = new StringWriter();
JsonGenerator json = this.jsonFactory.createGenerator(jsonString);
json.writeStartObject();
addServiceData(json, localService);
json.writeObjectFieldStart("data");
json.writeStringField("type", "HystrixThreadPool");
json.writeStringField("name", key.name());
json.writeNumberField("currentTime", System.currentTimeMillis());
json.writeNumberField(
"currentActiveCount", threadPoolMetrics.getCurrentActiveCount().intValue());
json.writeNumberField(
"currentCompletedTaskCount",
threadPoolMetrics.getCurrentCompletedTaskCount().longValue());
json.writeNumberField(
"currentCorePoolSize", threadPoolMetrics.getCurrentCorePoolSize().intValue());
json.writeNumberField(
"currentLargestPoolSize", threadPoolMetrics.getCurrentLargestPoolSize().intValue());
json.writeNumberField(
"currentMaximumPoolSize", threadPoolMetrics.getCurrentMaximumPoolSize().intValue());
json.writeNumberField("currentPoolSize", threadPoolMetrics.getCurrentPoolSize().intValue());
json.writeNumberField(
"currentQueueSize", threadPoolMetrics.getCurrentQueueSize().intValue());
json.writeNumberField(
"currentTaskCount", threadPoolMetrics.getCurrentTaskCount().longValue());
json.writeNumberField(
"rollingCountThreadsExecuted", threadPoolMetrics.getRollingCountThreadsExecuted());
json.writeNumberField(
"rollingMaxActiveThreads", threadPoolMetrics.getRollingMaxActiveThreads());
json.writeNumberField(
"propertyValue_queueSizeRejectionThreshold",
threadPoolMetrics.getProperties().queueSizeRejectionThreshold().get());
json.writeNumberField(
"propertyValue_metricsRollingStatisticalWindowInMilliseconds",
threadPoolMetrics
.getProperties()
.metricsRollingStatisticalWindowInMilliseconds()
.get());
json.writeNumberField("reportingHosts", 1); // this will get summed across
// all instances in a cluster
json.writeEndObject(); // end of data object
json.writeEndObject();
json.close();
// output to stream
this.jsonMetrics.add(jsonString.getBuffer().toString());
}
} catch (Exception ex) {
log.error("Error adding metrics to queue", ex);
}
}
