Ejemplo n.º 1
0
  @Test
  public void queueRejectionWithBlockingQueue() throws Exception {
    final int queueMaxSize = 5;
    final TenacityConfiguration exampleConfiguration =
        new TenacityConfiguration(
            new ThreadPoolConfiguration(1, 1, 10, queueMaxSize, 10000, 10),
            new CircuitBreakerConfiguration(20, 5000, 50, 10000, 10),
            5000);

    final TenacityPropertyRegister tenacityPropertyRegister =
        new TenacityPropertyRegister(
            ImmutableMap.<TenacityPropertyKey, TenacityConfiguration>of(
                DependencyKey.SLEEP, exampleConfiguration),
            new BreakerboxConfiguration(),
            mock(ArchaiusPropertyRegister.class));

    tenacityPropertyRegister.register();

    final ImmutableList.Builder<Future<Optional<String>>> sleepCommands = ImmutableList.builder();
    for (int i = 0; i < queueMaxSize * 2; i++) {
      sleepCommands.add(new SleepCommand(DependencyKey.SLEEP).queue());
    }

    for (Future<Optional<String>> future : sleepCommands.build()) {
      assertFalse(future.isCancelled());
      final Optional<String> result = future.get();
      if (result.isPresent()) {
        assertThat(result.get()).isEqualTo("sleep");
      } else {
        assertThat(result).isEqualTo(Optional.<String>absent());
      }
    }

    final HystrixCommandMetrics sleepCommandMetrics =
        new SleepCommand(DependencyKey.SLEEP).getCommandMetrics();
    assertThat(
            sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED))
        .isEqualTo(4);
    assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.TIMEOUT))
        .isEqualTo(0);
    assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS))
        .isEqualTo(4);
    assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.SHORT_CIRCUITED))
        .isEqualTo(0);

    final HystrixThreadPoolProperties threadPoolProperties =
        new SleepCommand(DependencyKey.SLEEP).getThreadpoolProperties();

    final ThreadPoolConfiguration threadPoolConfiguration = exampleConfiguration.getThreadpool();
    assertEquals(
        threadPoolProperties.queueSizeRejectionThreshold().get().intValue(),
        threadPoolConfiguration.getQueueSizeRejectionThreshold());
    assertEquals(
        threadPoolProperties.maxQueueSize().get().intValue(),
        threadPoolConfiguration.getMaxQueueSize());

    assertEquals(
        TenacityPropertyStore.getTenacityConfiguration(DependencyKey.SLEEP), exampleConfiguration);
  }
Ejemplo n.º 2
0
  @Before
  public void init() {
    for (HystrixCommandMetrics metricsInstance : HystrixCommandMetrics.getInstances()) {
      metricsInstance.resetStream();
    }

    HystrixCommandMetrics.reset();
    HystrixCircuitBreaker.Factory.reset();
    Hystrix.reset();
  }
Ejemplo n.º 3
0
 @Override
 public boolean isOpen() {
   System.out.println(
       "metrics : " + metrics.getCommandKey().name() + " : " + metrics.getHealthCounts());
   if (forceShortCircuit) {
     return true;
   } else {
     return metrics.getHealthCounts().getErrorCount() >= 3;
   }
 }
Ejemplo n.º 4
0
    @Override
    public void run() {
      try {
        for (HystrixCommandMetrics commandMetrics : HystrixCommandMetrics.getInstances()) {
          String jsonString = getCommandJson(commandMetrics);
          listener.handleJsonMetric(jsonString);
        }

        for (HystrixThreadPoolMetrics threadPoolMetrics : HystrixThreadPoolMetrics.getInstances()) {
          if (hasExecutedCommandsOnThread(threadPoolMetrics)) {
            String jsonString = getThreadPoolJson(threadPoolMetrics);
            listener.handleJsonMetric(jsonString);
          }
        }

        for (HystrixCollapserMetrics collapserMetrics : HystrixCollapserMetrics.getInstances()) {
          String jsonString = getCollapserJson(collapserMetrics);
          listener.handleJsonMetric(jsonString);
        }

      } catch (Exception e) {
        logger.warn("Failed to output metrics as JSON", e);
        // shutdown
        pause();
        return;
      }
    }
Ejemplo n.º 5
0
 /** Utility method for creating {@link HystrixCommandMetrics} for unit tests. */
 private static HystrixCommandMetrics getMetrics(HystrixCommandProperties.Setter properties) {
   return HystrixCommandMetrics.getInstance(
       CommandKeyForUnitTest.KEY_ONE,
       CommandOwnerForUnitTest.OWNER_ONE,
       ThreadPoolKeyForUnitTest.THREAD_POOL_ONE,
       HystrixCommandPropertiesTest.asMock(properties));
 }
Ejemplo n.º 6
0
    private <T> void logHC(HystrixInvokable<T> command, T response) {

      if (command instanceof HystrixInvokableInfo) {
        HystrixInvokableInfo<T> commandInfo = (HystrixInvokableInfo<T>) command;
        HystrixCommandMetrics metrics = commandInfo.getMetrics();
        System.out.println(
            "cb/error-count/%/total: "
                + commandInfo.isCircuitBreakerOpen()
                + " "
                + metrics.getHealthCounts().getErrorCount()
                + " "
                + metrics.getHealthCounts().getErrorPercentage()
                + " "
                + metrics.getHealthCounts().getTotalRequests()
                + "  => "
                + response
                + "  "
                + commandInfo.getExecutionEvents());
      }
    }
Ejemplo n.º 7
0
  @Test
  public void queueRejectionWithSynchronousQueue() throws Exception {
    final ImmutableCollection.Builder<Future<Optional<String>>> futures = ImmutableList.builder();
    for (int i = 0; i < 50; i++) {
      futures.add(new SleepCommand(DependencyKey.EXAMPLE).queue());
    }

    for (Future<Optional<String>> future : futures.build()) {
      assertFalse(future.isCancelled());
      final Optional<String> result = future.get();
      if (result.isPresent()) {
        assertThat(result.get()).isEqualTo("sleep");
      } else {
        assertThat(result).isEqualTo(Optional.<String>absent());
      }
    }

    final HystrixCommandMetrics sleepCommandMetrics =
        new SleepCommand(DependencyKey.EXAMPLE).getCommandMetrics();
    assertThat(
            sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED))
        .isGreaterThan(15);
    assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.TIMEOUT))
        .isEqualTo(0);
    assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS))
        .isGreaterThanOrEqualTo(40);
    assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.FALLBACK_FAILURE))
        .isEqualTo(0);
    assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.FALLBACK_REJECTION))
        .isEqualTo(0);
    assertThat(sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.SHORT_CIRCUITED))
        .isEqualTo(
            sleepCommandMetrics.getCumulativeCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS)
                - sleepCommandMetrics.getCumulativeCount(
                    HystrixRollingNumberEvent.THREAD_POOL_REJECTED));

    final HystrixThreadPoolProperties threadPoolProperties =
        new SleepCommand(DependencyKey.EXAMPLE).getThreadpoolProperties();

    // -1 means no limit on the number of items in the queue, which uses the
    // SynchronousBlockingQueue
    assertEquals(threadPoolProperties.maxQueueSize().get().intValue(), -1);
    assertEquals(
        TenacityPropertyStore.getTenacityConfiguration(DependencyKey.EXAMPLE),
        new TenacityConfiguration());
  }
  @Test
  public void should_gather_json_metrics() throws Exception {
    HystrixCommandKey hystrixCommandKey = HystrixCommandKey.Factory.asKey("commandKey");
    HystrixCommandMetrics.getInstance(
        hystrixCommandKey,
        HystrixCommandGroupKey.Factory.asKey("commandGroupKey"),
        new HystrixPropertiesCommandDefault(
            hystrixCommandKey, HystrixCommandProperties.defaultSetter()));
    given(this.discoveryClient.getLocalServiceInstance()).willReturn(this.serviceInstance);

    this.hystrixStreamTask.gatherMetrics();

    assertThat(this.hystrixStreamTask.jsonMetrics.isEmpty(), is(false));
  }
 @Override
 public Integer call() {
   return metrics.getHealthCounts().getErrorPercentage();
 }
 @Override
 public Long call() {
   return metrics.getRollingMaxConcurrentExecutions();
 }
 @Override
 public Integer call() {
   return metrics.getCurrentConcurrentExecutionCount();
 }
Ejemplo n.º 12
0
    private String getCommandJson(final HystrixCommandMetrics commandMetrics) throws IOException {
      HystrixCommandKey key = commandMetrics.getCommandKey();
      HystrixCircuitBreaker circuitBreaker = HystrixCircuitBreaker.Factory.getInstance(key);

      StringWriter jsonString = new StringWriter();
      JsonGenerator json = jsonFactory.createGenerator(jsonString);

      json.writeStartObject();
      json.writeStringField("type", "HystrixCommand");
      json.writeStringField("name", key.name());
      json.writeStringField("group", commandMetrics.getCommandGroup().name());
      json.writeNumberField("currentTime", System.currentTimeMillis());

      // circuit breaker
      if (circuitBreaker == null) {
        // circuit breaker is disabled and thus never open
        json.writeBooleanField("isCircuitBreakerOpen", false);
      } else {
        json.writeBooleanField("isCircuitBreakerOpen", circuitBreaker.isOpen());
      }
      HealthCounts healthCounts = commandMetrics.getHealthCounts();
      json.writeNumberField("errorPercentage", healthCounts.getErrorPercentage());
      json.writeNumberField("errorCount", healthCounts.getErrorCount());
      json.writeNumberField("requestCount", healthCounts.getTotalRequests());

      // rolling counters
      safelyWriteNumberField(
          json,
          "rollingCountBadRequests",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.BAD_REQUEST);
            }
          });
      safelyWriteNumberField(
          json,
          "rollingCountCollapsedRequests",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.COLLAPSED);
            }
          });
      safelyWriteNumberField(
          json,
          "rollingCountEmit",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.EMIT);
            }
          });
      safelyWriteNumberField(
          json,
          "rollingCountExceptionsThrown",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.EXCEPTION_THROWN);
            }
          });
      safelyWriteNumberField(
          json,
          "rollingCountFailure",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.FAILURE);
            }
          });
      safelyWriteNumberField(
          json,
          "rollingCountFallbackEmit",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.FALLBACK_EMIT);
            }
          });
      safelyWriteNumberField(
          json,
          "rollingCountFallbackFailure",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.FALLBACK_FAILURE);
            }
          });
      safelyWriteNumberField(
          json,
          "rollingCountFallbackMissing",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.FALLBACK_MISSING);
            }
          });
      safelyWriteNumberField(
          json,
          "rollingCountFallbackRejection",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.FALLBACK_REJECTION);
            }
          });
      safelyWriteNumberField(
          json,
          "rollingCountFallbackSuccess",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.FALLBACK_SUCCESS);
            }
          });
      safelyWriteNumberField(
          json,
          "rollingCountResponsesFromCache",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.RESPONSE_FROM_CACHE);
            }
          });
      safelyWriteNumberField(
          json,
          "rollingCountSemaphoreRejected",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.SEMAPHORE_REJECTED);
            }
          });
      safelyWriteNumberField(
          json,
          "rollingCountShortCircuited",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.SHORT_CIRCUITED);
            }
          });
      safelyWriteNumberField(
          json,
          "rollingCountSuccess",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.SUCCESS);
            }
          });
      safelyWriteNumberField(
          json,
          "rollingCountThreadPoolRejected",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.THREAD_POOL_REJECTED);
            }
          });
      safelyWriteNumberField(
          json,
          "rollingCountTimeout",
          new Func0<Long>() {
            @Override
            public Long call() {
              return commandMetrics.getRollingCount(HystrixEventType.TIMEOUT);
            }
          });

      json.writeNumberField(
          "currentConcurrentExecutionCount", commandMetrics.getCurrentConcurrentExecutionCount());
      json.writeNumberField(
          "rollingMaxConcurrentExecutionCount", commandMetrics.getRollingMaxConcurrentExecutions());

      // latency percentiles
      json.writeNumberField("latencyExecute_mean", commandMetrics.getExecutionTimeMean());
      json.writeObjectFieldStart("latencyExecute");
      json.writeNumberField("0", commandMetrics.getExecutionTimePercentile(0));
      json.writeNumberField("25", commandMetrics.getExecutionTimePercentile(25));
      json.writeNumberField("50", commandMetrics.getExecutionTimePercentile(50));
      json.writeNumberField("75", commandMetrics.getExecutionTimePercentile(75));
      json.writeNumberField("90", commandMetrics.getExecutionTimePercentile(90));
      json.writeNumberField("95", commandMetrics.getExecutionTimePercentile(95));
      json.writeNumberField("99", commandMetrics.getExecutionTimePercentile(99));
      json.writeNumberField("99.5", commandMetrics.getExecutionTimePercentile(99.5));
      json.writeNumberField("100", commandMetrics.getExecutionTimePercentile(100));
      json.writeEndObject();
      //
      json.writeNumberField("latencyTotal_mean", commandMetrics.getTotalTimeMean());
      json.writeObjectFieldStart("latencyTotal");
      json.writeNumberField("0", commandMetrics.getTotalTimePercentile(0));
      json.writeNumberField("25", commandMetrics.getTotalTimePercentile(25));
      json.writeNumberField("50", commandMetrics.getTotalTimePercentile(50));
      json.writeNumberField("75", commandMetrics.getTotalTimePercentile(75));
      json.writeNumberField("90", commandMetrics.getTotalTimePercentile(90));
      json.writeNumberField("95", commandMetrics.getTotalTimePercentile(95));
      json.writeNumberField("99", commandMetrics.getTotalTimePercentile(99));
      json.writeNumberField("99.5", commandMetrics.getTotalTimePercentile(99.5));
      json.writeNumberField("100", commandMetrics.getTotalTimePercentile(100));
      json.writeEndObject();

      // property values for reporting what is actually seen by the command rather than what was set
      // somewhere
      HystrixCommandProperties commandProperties = commandMetrics.getProperties();

      json.writeNumberField(
          "propertyValue_circuitBreakerRequestVolumeThreshold",
          commandProperties.circuitBreakerRequestVolumeThreshold().get());
      json.writeNumberField(
          "propertyValue_circuitBreakerSleepWindowInMilliseconds",
          commandProperties.circuitBreakerSleepWindowInMilliseconds().get());
      json.writeNumberField(
          "propertyValue_circuitBreakerErrorThresholdPercentage",
          commandProperties.circuitBreakerErrorThresholdPercentage().get());
      json.writeBooleanField(
          "propertyValue_circuitBreakerForceOpen",
          commandProperties.circuitBreakerForceOpen().get());
      json.writeBooleanField(
          "propertyValue_circuitBreakerForceClosed",
          commandProperties.circuitBreakerForceClosed().get());
      json.writeBooleanField(
          "propertyValue_circuitBreakerEnabled", commandProperties.circuitBreakerEnabled().get());

      json.writeStringField(
          "propertyValue_executionIsolationStrategy",
          commandProperties.executionIsolationStrategy().get().name());
      json.writeNumberField(
          "propertyValue_executionIsolationThreadTimeoutInMilliseconds",
          commandProperties.executionTimeoutInMilliseconds().get());
      json.writeNumberField(
          "propertyValue_executionTimeoutInMilliseconds",
          commandProperties.executionTimeoutInMilliseconds().get());
      json.writeBooleanField(
          "propertyValue_executionIsolationThreadInterruptOnTimeout",
          commandProperties.executionIsolationThreadInterruptOnTimeout().get());
      json.writeStringField(
          "propertyValue_executionIsolationThreadPoolKeyOverride",
          commandProperties.executionIsolationThreadPoolKeyOverride().get());
      json.writeNumberField(
          "propertyValue_executionIsolationSemaphoreMaxConcurrentRequests",
          commandProperties.executionIsolationSemaphoreMaxConcurrentRequests().get());
      json.writeNumberField(
          "propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests",
          commandProperties.fallbackIsolationSemaphoreMaxConcurrentRequests().get());

      /*
       * The following are commented out as these rarely change and are verbose for streaming for something people don't change.
       * We could perhaps allow a property or request argument to include these.
       */

      //                    json.put("propertyValue_metricsRollingPercentileEnabled",
      // commandProperties.metricsRollingPercentileEnabled().get());
      //                    json.put("propertyValue_metricsRollingPercentileBucketSize",
      // commandProperties.metricsRollingPercentileBucketSize().get());
      //                    json.put("propertyValue_metricsRollingPercentileWindow",
      // commandProperties.metricsRollingPercentileWindowInMilliseconds().get());
      //                    json.put("propertyValue_metricsRollingPercentileWindowBuckets",
      // commandProperties.metricsRollingPercentileWindowBuckets().get());
      //                    json.put("propertyValue_metricsRollingStatisticalWindowBuckets",
      // commandProperties.metricsRollingStatisticalWindowBuckets().get());
      json.writeNumberField(
          "propertyValue_metricsRollingStatisticalWindowInMilliseconds",
          commandProperties.metricsRollingStatisticalWindowInMilliseconds().get());

      json.writeBooleanField(
          "propertyValue_requestCacheEnabled", commandProperties.requestCacheEnabled().get());
      json.writeBooleanField(
          "propertyValue_requestLogEnabled", commandProperties.requestLogEnabled().get());

      json.writeNumberField(
          "reportingHosts", 1); // this will get summed across all instances in a cluster
      json.writeStringField("threadPool", commandMetrics.getThreadPoolKey().name());

      json.writeEndObject();
      json.close();

      return jsonString.getBuffer().toString();
    }
Ejemplo n.º 13
0
  @Override
  public void run() {

    try {
      // command metrics
      for (HystrixCommandMetrics commandMetrics : HystrixCommandMetrics.getInstances()) {
        HystrixCommandKey key = commandMetrics.getCommandKey();
        HystrixCircuitBreaker circuitBreaker = HystrixCircuitBreaker.Factory.getInstance(key);
        StringWriter jsonString = new StringWriter();
        JsonGenerator json = jsonFactory.createJsonGenerator(jsonString);

        // Informational and Status
        json.writeStartObject();
        json.writeStringField(type.value, HystrixCommand.value);
        json.writeStringField(name.value, key.name());
        json.writeStringField(group.value, commandMetrics.getCommandGroup().name());
        json.writeNumberField(currentTime.value, (int) (System.currentTimeMillis() / 1000));

        // circuit breaker
        json.writeBooleanField(isCircuitBreakerOpen.value, circuitBreaker.isOpen());
        HystrixCommandMetrics.HealthCounts healthCounts = commandMetrics.getHealthCounts();
        json.writeNumberField(errorPercentage.value, healthCounts.getErrorPercentage());
        json.writeNumberField(errorCount.value, healthCounts.getErrorCount());
        json.writeNumberField(requestCount.value, healthCounts.getTotalRequests());

        // rolling counters  Gauge
        json.writeNumberField(
            rollingCountCollapsedRequests.value,
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.COLLAPSED));
        json.writeNumberField(
            rollingCountExceptionsThrown.value,
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN));
        json.writeNumberField(
            rollingCountFailure.value,
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.FAILURE));
        json.writeNumberField(
            rollingCountFallbackFailure.value,
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE));
        json.writeNumberField(
            rollingCountFallbackRejection.value,
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION));
        json.writeNumberField(
            rollingCountFallbackSuccess.value,
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS));
        json.writeNumberField(
            rollingCountResponsesFromCache.value,
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE));
        json.writeNumberField(
            rollingCountSemaphoreRejected.value,
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED));
        json.writeNumberField(
            rollingCountShortCircuited.value,
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED));
        json.writeNumberField(
            rollingCountSuccess.value,
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS));
        json.writeNumberField(
            rollingCountThreadPoolRejected.value,
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED));
        json.writeNumberField(
            rollingCountTimeout.value,
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT));

        json.writeNumberField(
            currentConcurrentExecutionCount.value,
            commandMetrics.getCurrentConcurrentExecutionCount());

        // latency percentiles
        json.writeNumberField(latencyExecute_mean.value, commandMetrics.getExecutionTimeMean());
        json.writeObjectFieldStart(latencyExecute.value);
        json.writeNumberField("0", commandMetrics.getExecutionTimePercentile(0));
        json.writeNumberField("25", commandMetrics.getExecutionTimePercentile(25));
        json.writeNumberField("50", commandMetrics.getExecutionTimePercentile(50));
        json.writeNumberField("75", commandMetrics.getExecutionTimePercentile(75));
        json.writeNumberField("90", commandMetrics.getExecutionTimePercentile(90));
        json.writeNumberField("95", commandMetrics.getExecutionTimePercentile(95));
        json.writeNumberField("99", commandMetrics.getExecutionTimePercentile(99));
        json.writeNumberField("99.5", commandMetrics.getExecutionTimePercentile(99.5));
        json.writeNumberField("100", commandMetrics.getExecutionTimePercentile(100));
        json.writeEndObject();
        //
        json.writeNumberField(latencyTotal_mean.value, commandMetrics.getTotalTimeMean());
        json.writeObjectFieldStart(latencyTotal.value);
        json.writeNumberField("0", commandMetrics.getTotalTimePercentile(0));
        json.writeNumberField("25", commandMetrics.getTotalTimePercentile(25));
        json.writeNumberField("50", commandMetrics.getTotalTimePercentile(50));
        json.writeNumberField("75", commandMetrics.getTotalTimePercentile(75));
        json.writeNumberField("90", commandMetrics.getTotalTimePercentile(90));
        json.writeNumberField("95", commandMetrics.getTotalTimePercentile(95));
        json.writeNumberField("99", commandMetrics.getTotalTimePercentile(99));
        json.writeNumberField("99.5", commandMetrics.getTotalTimePercentile(99.5));
        json.writeNumberField("100", commandMetrics.getTotalTimePercentile(100));
        json.writeEndObject();

        // property values for reporting what is actually seen by the command rather than what was
        // set somewhere
        HystrixCommandProperties commandProperties = commandMetrics.getProperties();

        json.writeNumberField(
            propertyValue_circuitBreakerRequestVolumeThreshold.value,
            commandProperties.circuitBreakerRequestVolumeThreshold().get());
        json.writeNumberField(
            propertyValue_circuitBreakerSleepWindowInMilliseconds.value,
            commandProperties.circuitBreakerSleepWindowInMilliseconds().get());
        json.writeNumberField(
            propertyValue_circuitBreakerErrorThresholdPercentage.value,
            commandProperties.circuitBreakerErrorThresholdPercentage().get());
        json.writeBooleanField(
            propertyValue_circuitBreakerForceOpen.value,
            commandProperties.circuitBreakerForceOpen().get());
        json.writeBooleanField(
            propertyValue_circuitBreakerForceClosed.value,
            commandProperties.circuitBreakerForceClosed().get());
        json.writeBooleanField(
            propertyValue_circuitBreakerEnabled.value,
            commandProperties.circuitBreakerEnabled().get());

        json.writeStringField(
            propertyValue_executionIsolationStrategy.value,
            commandProperties.executionIsolationStrategy().get().name());
        json.writeNumberField(
            propertyValue_executionIsolationThreadTimeoutInMilliseconds.value,
            commandProperties.executionIsolationThreadTimeoutInMilliseconds().get());
        json.writeBooleanField(
            propertyValue_executionIsolationThreadInterruptOnTimeout.value,
            commandProperties.executionIsolationThreadInterruptOnTimeout().get());
        json.writeStringField(
            propertyValue_executionIsolationThreadPoolKeyOverride.value,
            commandProperties.executionIsolationThreadPoolKeyOverride().get());
        json.writeNumberField(
            propertyValue_executionIsolationSemaphoreMaxConcurrentRequests.value,
            commandProperties.executionIsolationSemaphoreMaxConcurrentRequests().get());
        json.writeNumberField(
            propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests.value,
            commandProperties.fallbackIsolationSemaphoreMaxConcurrentRequests().get());

        /*
         * The following are commented out as these rarely change and are verbose for streaming for something people don't change.
         * We could perhaps allow a property or request argument to include these.
         */

        //                    json.put("propertyValue_metricsRollingPercentileEnabled",
        // commandProperties.metricsRollingPercentileEnabled().get());
        //                    json.put("propertyValue_metricsRollingPercentileBucketSize",
        // commandProperties.metricsRollingPercentileBucketSize().get());
        //                    json.put("propertyValue_metricsRollingPercentileWindow",
        // commandProperties.metricsRollingPercentileWindowInMilliseconds().get());
        //                    json.put("propertyValue_metricsRollingPercentileWindowBuckets",
        // commandProperties.metricsRollingPercentileWindowBuckets().get());
        //                    json.put("propertyValue_metricsRollingStatisticalWindowBuckets",
        // commandProperties.metricsRollingStatisticalWindowBuckets().get());
        json.writeNumberField(
            propertyValue_metricsRollingStatisticalWindowInMilliseconds.value,
            commandProperties.metricsRollingStatisticalWindowInMilliseconds().get());
        json.writeBooleanField(
            propertyValue_requestCacheEnabled.value, commandProperties.requestCacheEnabled().get());
        json.writeBooleanField(
            propertyValue_requestLogEnabled.value, commandProperties.requestLogEnabled().get());
        json.writeNumberField(reportingHosts.value, 1);

        json.writeStringField(Key.ip.value, app_ip);

        json.writeEndObject();
        json.close();
        //                System.out.println(ip + ":" + port + "||" +
        // jsonString.getBuffer().toString());
        UDPClient.send(
            socketIp, socketPort, jsonString.getBuffer().toString().getBytes(), new byte[] {});
      }

      // thread pool metrics
      for (HystrixThreadPoolMetrics threadPoolMetrics : HystrixThreadPoolMetrics.getInstances()) {
        HystrixThreadPoolKey key = threadPoolMetrics.getThreadPoolKey();

        StringWriter jsonString = new StringWriter();
        JsonGenerator json = jsonFactory.createJsonGenerator(jsonString);
        json.writeStartObject();

        json.writeStringField(type.value, HystrixThreadPool.value);
        json.writeStringField(name.value, key.name());
        json.writeNumberField(currentTime.value, System.currentTimeMillis());

        // 101.3 80 154

        json.writeNumberField(
            currentActiveCount.value, threadPoolMetrics.getCurrentActiveCount().intValue());
        json.writeNumberField(
            currentCompletedTaskCount.value,
            threadPoolMetrics.getCurrentCompletedTaskCount().longValue());
        json.writeNumberField(
            currentCorePoolSize.value, threadPoolMetrics.getCurrentCorePoolSize().intValue());
        json.writeNumberField(
            currentLargestPoolSize.value, threadPoolMetrics.getCurrentLargestPoolSize().intValue());
        json.writeNumberField(
            currentMaximumPoolSize.value, threadPoolMetrics.getCurrentMaximumPoolSize().intValue());
        json.writeNumberField(
            currentPoolSize.value, threadPoolMetrics.getCurrentPoolSize().intValue());
        json.writeNumberField(
            currentQueueSize.value, threadPoolMetrics.getCurrentQueueSize().intValue());
        json.writeNumberField(
            currentTaskCount.value, threadPoolMetrics.getCurrentTaskCount().longValue());
        json.writeNumberField(
            rollingCountThreadsExecuted.value, threadPoolMetrics.getRollingCountThreadsExecuted());
        json.writeNumberField(
            rollingMaxActiveThreads.value, threadPoolMetrics.getRollingMaxActiveThreads());

        json.writeNumberField(
            propertyValue_queueSizeRejectionThreshold.value,
            threadPoolMetrics.getProperties().queueSizeRejectionThreshold().get());
        json.writeNumberField(
            propertyValue_metricsRollingStatisticalWindowInMilliseconds.value,
            threadPoolMetrics
                .getProperties()
                .metricsRollingStatisticalWindowInMilliseconds()
                .get());

        json.writeNumberField(
            reportingHosts.value, 1); // this will get summed across all instances in a cluster
        json.writeStringField(Key.ip.value, app_ip);
        json.writeEndObject();
        json.close();

        String str = jsonString.getBuffer().toString();
        byte[] ret = str.getBytes();
        UDPClient.send(socketIp, socketPort, ret, new byte[] {});
      }
    } catch (Exception e) {
      System.err.println("Failed to output metrics as JSON");
      e.printStackTrace();
    }
  }
  @Scheduled(fixedRateString = "${hystrix.stream.amqp.gatherRate:500}")
  public void gatherMetrics() {
    try {
      // command metrics
      Collection<HystrixCommandMetrics> instances = HystrixCommandMetrics.getInstances();
      if (!instances.isEmpty()) {
        log.trace("gathering metrics size: " + instances.size());
      }

      ServiceInstance localService = this.discoveryClient.getLocalServiceInstance();

      for (HystrixCommandMetrics commandMetrics : instances) {
        HystrixCommandKey key = commandMetrics.getCommandKey();
        HystrixCircuitBreaker circuitBreaker = HystrixCircuitBreaker.Factory.getInstance(key);

        StringWriter jsonString = new StringWriter();
        JsonGenerator json = this.jsonFactory.createGenerator(jsonString);

        json.writeStartObject();

        addServiceData(json, localService);
        json.writeObjectFieldStart("data");
        json.writeStringField("type", "HystrixCommand");
        String name = key.name();

        if (this.properties.isPrefixMetricName()) {
          name = localService.getServiceId() + "." + name;
        }

        json.writeStringField("name", name);
        json.writeStringField("group", commandMetrics.getCommandGroup().name());
        json.writeNumberField("currentTime", System.currentTimeMillis());

        // circuit breaker
        if (circuitBreaker == null) {
          // circuit breaker is disabled and thus never open
          json.writeBooleanField("isCircuitBreakerOpen", false);
        } else {
          json.writeBooleanField("isCircuitBreakerOpen", circuitBreaker.isOpen());
        }
        HystrixCommandMetrics.HealthCounts healthCounts = commandMetrics.getHealthCounts();
        json.writeNumberField("errorPercentage", healthCounts.getErrorPercentage());
        json.writeNumberField("errorCount", healthCounts.getErrorCount());
        json.writeNumberField("requestCount", healthCounts.getTotalRequests());

        // rolling counters
        json.writeNumberField(
            "rollingCountCollapsedRequests",
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.COLLAPSED));
        json.writeNumberField(
            "rollingCountExceptionsThrown",
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.EXCEPTION_THROWN));
        json.writeNumberField(
            "rollingCountFailure",
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.FAILURE));
        json.writeNumberField(
            "rollingCountFallbackFailure",
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_FAILURE));
        json.writeNumberField(
            "rollingCountFallbackRejection",
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_REJECTION));
        json.writeNumberField(
            "rollingCountFallbackSuccess",
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.FALLBACK_SUCCESS));
        json.writeNumberField(
            "rollingCountResponsesFromCache",
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE));
        json.writeNumberField(
            "rollingCountSemaphoreRejected",
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.SEMAPHORE_REJECTED));
        json.writeNumberField(
            "rollingCountShortCircuited",
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.SHORT_CIRCUITED));
        json.writeNumberField(
            "rollingCountSuccess",
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.SUCCESS));
        json.writeNumberField(
            "rollingCountThreadPoolRejected",
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.THREAD_POOL_REJECTED));
        json.writeNumberField(
            "rollingCountTimeout",
            commandMetrics.getRollingCount(HystrixRollingNumberEvent.TIMEOUT));

        json.writeNumberField(
            "currentConcurrentExecutionCount", commandMetrics.getCurrentConcurrentExecutionCount());

        // latency percentiles
        json.writeNumberField("latencyExecute_mean", commandMetrics.getExecutionTimeMean());
        json.writeObjectFieldStart("latencyExecute");
        json.writeNumberField("0", commandMetrics.getExecutionTimePercentile(0));
        json.writeNumberField("25", commandMetrics.getExecutionTimePercentile(25));
        json.writeNumberField("50", commandMetrics.getExecutionTimePercentile(50));
        json.writeNumberField("75", commandMetrics.getExecutionTimePercentile(75));
        json.writeNumberField("90", commandMetrics.getExecutionTimePercentile(90));
        json.writeNumberField("95", commandMetrics.getExecutionTimePercentile(95));
        json.writeNumberField("99", commandMetrics.getExecutionTimePercentile(99));
        json.writeNumberField("99.5", commandMetrics.getExecutionTimePercentile(99.5));
        json.writeNumberField("100", commandMetrics.getExecutionTimePercentile(100));
        json.writeEndObject();
        //
        json.writeNumberField("latencyTotal_mean", commandMetrics.getTotalTimeMean());
        json.writeObjectFieldStart("latencyTotal");
        json.writeNumberField("0", commandMetrics.getTotalTimePercentile(0));
        json.writeNumberField("25", commandMetrics.getTotalTimePercentile(25));
        json.writeNumberField("50", commandMetrics.getTotalTimePercentile(50));
        json.writeNumberField("75", commandMetrics.getTotalTimePercentile(75));
        json.writeNumberField("90", commandMetrics.getTotalTimePercentile(90));
        json.writeNumberField("95", commandMetrics.getTotalTimePercentile(95));
        json.writeNumberField("99", commandMetrics.getTotalTimePercentile(99));
        json.writeNumberField("99.5", commandMetrics.getTotalTimePercentile(99.5));
        json.writeNumberField("100", commandMetrics.getTotalTimePercentile(100));
        json.writeEndObject();

        // property values for reporting what is actually seen by the command
        // rather than what was set somewhere
        HystrixCommandProperties commandProperties = commandMetrics.getProperties();

        json.writeNumberField(
            "propertyValue_circuitBreakerRequestVolumeThreshold",
            commandProperties.circuitBreakerRequestVolumeThreshold().get());
        json.writeNumberField(
            "propertyValue_circuitBreakerSleepWindowInMilliseconds",
            commandProperties.circuitBreakerSleepWindowInMilliseconds().get());
        json.writeNumberField(
            "propertyValue_circuitBreakerErrorThresholdPercentage",
            commandProperties.circuitBreakerErrorThresholdPercentage().get());
        json.writeBooleanField(
            "propertyValue_circuitBreakerForceOpen",
            commandProperties.circuitBreakerForceOpen().get());
        json.writeBooleanField(
            "propertyValue_circuitBreakerForceClosed",
            commandProperties.circuitBreakerForceClosed().get());
        json.writeBooleanField(
            "propertyValue_circuitBreakerEnabled", commandProperties.circuitBreakerEnabled().get());

        json.writeStringField(
            "propertyValue_executionIsolationStrategy",
            commandProperties.executionIsolationStrategy().get().name());
        json.writeNumberField(
            "propertyValue_executionIsolationThreadTimeoutInMilliseconds",
            commandProperties.executionIsolationThreadTimeoutInMilliseconds().get());
        json.writeBooleanField(
            "propertyValue_executionIsolationThreadInterruptOnTimeout",
            commandProperties.executionIsolationThreadInterruptOnTimeout().get());
        json.writeStringField(
            "propertyValue_executionIsolationThreadPoolKeyOverride",
            commandProperties.executionIsolationThreadPoolKeyOverride().get());
        json.writeNumberField(
            "propertyValue_executionIsolationSemaphoreMaxConcurrentRequests",
            commandProperties.executionIsolationSemaphoreMaxConcurrentRequests().get());
        json.writeNumberField(
            "propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests",
            commandProperties.fallbackIsolationSemaphoreMaxConcurrentRequests().get());

        // TODO
        /*
         * The following are commented out as these rarely change and are verbose
         * for streaming for something people don't change. We could perhaps allow
         * a property or request argument to include these.
         */

        // json.put("propertyValue_metricsRollingPercentileEnabled",
        // commandProperties.metricsRollingPercentileEnabled().get());
        // json.put("propertyValue_metricsRollingPercentileBucketSize",
        // commandProperties.metricsRollingPercentileBucketSize().get());
        // json.put("propertyValue_metricsRollingPercentileWindow",
        // commandProperties.metricsRollingPercentileWindowInMilliseconds().get());
        // json.put("propertyValue_metricsRollingPercentileWindowBuckets",
        // commandProperties.metricsRollingPercentileWindowBuckets().get());
        // json.put("propertyValue_metricsRollingStatisticalWindowBuckets",
        // commandProperties.metricsRollingStatisticalWindowBuckets().get());
        json.writeNumberField(
            "propertyValue_metricsRollingStatisticalWindowInMilliseconds",
            commandProperties.metricsRollingStatisticalWindowInMilliseconds().get());

        json.writeBooleanField(
            "propertyValue_requestCacheEnabled", commandProperties.requestCacheEnabled().get());
        json.writeBooleanField(
            "propertyValue_requestLogEnabled", commandProperties.requestLogEnabled().get());

        json.writeNumberField("reportingHosts", 1); // this will get summed across
        // all instances in a cluster

        json.writeEndObject(); // end data attribute
        json.writeEndObject();
        json.close();

        // output
        this.jsonMetrics.add(jsonString.getBuffer().toString());
      }

      // thread pool metrics
      for (HystrixThreadPoolMetrics threadPoolMetrics : HystrixThreadPoolMetrics.getInstances()) {
        HystrixThreadPoolKey key = threadPoolMetrics.getThreadPoolKey();

        StringWriter jsonString = new StringWriter();
        JsonGenerator json = this.jsonFactory.createGenerator(jsonString);
        json.writeStartObject();

        addServiceData(json, localService);
        json.writeObjectFieldStart("data");

        json.writeStringField("type", "HystrixThreadPool");
        json.writeStringField("name", key.name());
        json.writeNumberField("currentTime", System.currentTimeMillis());

        json.writeNumberField(
            "currentActiveCount", threadPoolMetrics.getCurrentActiveCount().intValue());
        json.writeNumberField(
            "currentCompletedTaskCount",
            threadPoolMetrics.getCurrentCompletedTaskCount().longValue());
        json.writeNumberField(
            "currentCorePoolSize", threadPoolMetrics.getCurrentCorePoolSize().intValue());
        json.writeNumberField(
            "currentLargestPoolSize", threadPoolMetrics.getCurrentLargestPoolSize().intValue());
        json.writeNumberField(
            "currentMaximumPoolSize", threadPoolMetrics.getCurrentMaximumPoolSize().intValue());
        json.writeNumberField("currentPoolSize", threadPoolMetrics.getCurrentPoolSize().intValue());
        json.writeNumberField(
            "currentQueueSize", threadPoolMetrics.getCurrentQueueSize().intValue());
        json.writeNumberField(
            "currentTaskCount", threadPoolMetrics.getCurrentTaskCount().longValue());
        json.writeNumberField(
            "rollingCountThreadsExecuted", threadPoolMetrics.getRollingCountThreadsExecuted());
        json.writeNumberField(
            "rollingMaxActiveThreads", threadPoolMetrics.getRollingMaxActiveThreads());

        json.writeNumberField(
            "propertyValue_queueSizeRejectionThreshold",
            threadPoolMetrics.getProperties().queueSizeRejectionThreshold().get());
        json.writeNumberField(
            "propertyValue_metricsRollingStatisticalWindowInMilliseconds",
            threadPoolMetrics
                .getProperties()
                .metricsRollingStatisticalWindowInMilliseconds()
                .get());

        json.writeNumberField("reportingHosts", 1); // this will get summed across
        // all instances in a cluster

        json.writeEndObject(); // end of data object
        json.writeEndObject();
        json.close();
        // output to stream
        this.jsonMetrics.add(jsonString.getBuffer().toString());
      }
    } catch (Exception ex) {
      log.error("Error adding metrics to queue", ex);
    }
  }