Ejemplo n.º 1
0
 public static boolean shouldUseESForUnits() {
   return Configuration.getInstance().getBooleanProperty(CoreConfig.SHOULD_STORE_UNITS)
       && Configuration.getInstance().getBooleanProperty(CoreConfig.USE_ES_FOR_UNITS)
       && Configuration.getInstance()
           .getListProperty(CoreConfig.DISCOVERY_MODULES)
           .contains(ElasticIOPath);
 }
public class InternalApiIntegrationTest {
  private final String clusterString =
      Configuration.getInstance().getStringProperty(CoreConfig.INTERNAL_API_CLUSTER);

  @Test
  public void testRegularWorks() throws IOException {
    InternalAPI api = InternalAPIFactory.create(5, clusterString);
    try {
      Account acct = api.fetchAccount("acDUSBABEK");
      Assert.assertNotNull(acct);
    } catch (IOException ex) {
      if (ex instanceof ClusterException) {
        ClusterException clusterEx = (ClusterException) ex;
        for (Throwable err : clusterEx.getExceptions()) err.printStackTrace();
      } else ex.printStackTrace();
      Assert.fail(ex.getMessage());
    } finally {
      shutdown(api);
    }
  }

  @Test(expected = HttpResponseException.class)
  public void testRegularHttpError() throws IOException {
    InternalAPI api = InternalAPIFactory.create(1, clusterString);
    Account acct = api.fetchAccount("acBOGUS___");
    shutdown(api);
  }

  @Test
  public void testConcurrentRequests() throws InterruptedException {
    final int threads = 150;
    final InternalAPI api = InternalAPIFactory.create(5, clusterString);
    final AtomicInteger errors = new AtomicInteger(0);
    final CountDownLatch latch = new CountDownLatch(threads);
    for (int i = 0; i < threads; i++) {
      new Thread("request " + i) {
        public void run() {
          try {
            Account acct = api.fetchAccount("acDUSBABEK");
            Assert.assertNotNull(acct);
          } catch (IOException ex) {
            errors.incrementAndGet();
            ex.printStackTrace();
          } finally {
            latch.countDown();
          }
        }
      }.start();
    }
    latch.await(5000, TimeUnit.MILLISECONDS);
    shutdown(api);
    if (errors.get() > 0) Assert.fail("There were IOExceptions on some requests.");
  }

  private static void shutdown(InternalAPI api) {
    ClientConnectionManager connectionManager =
        (ClientConnectionManager) Whitebox.getInternalState(api, "connectionManager");
    connectionManager.shutdown();
  }
}
 public HttpMultiRollupsQueryHandler() {
   Configuration config = Configuration.getInstance();
   int maxThreadsToUse = config.getIntegerProperty(HttpConfig.MAX_READ_WORKER_THREADS);
   int maxQueueSize = config.getIntegerProperty(HttpConfig.MAX_BATCH_READ_REQUESTS_TO_QUEUE);
   this.maxMetricsPerRequest = config.getIntegerProperty(HttpConfig.MAX_METRICS_PER_BATCH_QUERY);
   this.serializer = new BatchedMetricsJSONOutputSerializer();
   this.gson = new GsonBuilder().setPrettyPrinting().serializeNulls().create();
   this.parser = new JsonParser();
   this.executor =
       new ThreadPoolBuilder()
           .withCorePoolSize(maxThreadsToUse)
           .withMaxPoolSize(maxThreadsToUse)
           .withName("HTTP-BatchMetricsFetch")
           .withBoundedQueue(maxQueueSize)
           .build();
 }
 @BeforeClass
 public static void setUpHttp() {
   queryPort =
       Configuration.getInstance().getIntegerProperty(HttpConfig.HTTP_METRIC_DATA_QUERY_PORT);
   httpQueryService = new HttpQueryService();
   httpQueryService.startService();
   vendor = new HttpClientVendor();
   client = vendor.getClient();
 }
Ejemplo n.º 5
0
  static {
    Configuration config = Configuration.getInstance();

    // register jvm metrics
    MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
    if (!System.getProperty("java.version").split("\\.")[1].equals("6")) {
      // if not running 1.6
      registry.registerAll(
          new PrefixedMetricSet(new BufferPoolMetricSet(mbs), JVM_PREFIX, "buffer-pool"));
    }
    registry.registerAll(new PrefixedMetricSet(new GarbageCollectorMetricSet(), JVM_PREFIX, "gc"));
    registry.registerAll(new PrefixedMetricSet(new MemoryUsageGaugeSet(), JVM_PREFIX, "memory"));
    registry.registerAll(
        new PrefixedMetricSet(new ThreadStatesGaugeSet(), JVM_PREFIX, "thread-states"));

    // instrument log4j
    InstrumentedAppender appender = new InstrumentedAppender(registry);
    appender.activateOptions();
    LogManager.getRootLogger().addAppender(appender);

    if (!config.getStringProperty(CoreConfig.RIEMANN_HOST).equals("")) {
      RiemannReporter tmpreporter;
      try {
        Riemann riemann =
            new Riemann(
                config.getStringProperty(CoreConfig.RIEMANN_HOST),
                config.getIntegerProperty(CoreConfig.RIEMANN_PORT));

        RiemannReporter.Builder builder =
            RiemannReporter.forRegistry(registry)
                .convertDurationsTo(TimeUnit.MILLISECONDS)
                .convertRatesTo(TimeUnit.SECONDS);
        if (!config.getStringProperty(CoreConfig.RIEMANN_SEPARATOR).isEmpty()) {
          builder.useSeparator(config.getStringProperty(CoreConfig.RIEMANN_SEPARATOR));
        }
        if (!config.getStringProperty(CoreConfig.RIEMANN_TTL).isEmpty()) {
          builder.withTtl(config.getFloatProperty(CoreConfig.RIEMANN_TTL));
        }
        if (!config.getStringProperty(CoreConfig.RIEMANN_LOCALHOST).isEmpty()) {
          builder.localHost(config.getStringProperty(CoreConfig.RIEMANN_LOCALHOST));
        }
        if (!config.getStringProperty(CoreConfig.RIEMANN_PREFIX).isEmpty()) {
          builder.prefixedWith(config.getStringProperty(CoreConfig.RIEMANN_PREFIX));
        }
        if (!config.getStringProperty(CoreConfig.RIEMANN_TAGS).isEmpty()) {
          builder.tags(config.getListProperty(CoreConfig.RIEMANN_TAGS));
        }
        tmpreporter = builder.build(riemann);

        tmpreporter.start(30l, TimeUnit.SECONDS);
      } catch (IOException e) {
        tmpreporter = null;
      }
      reporter1 = tmpreporter;
    } else {
      reporter1 = null;
    }

    if (!config.getStringProperty(CoreConfig.GRAPHITE_HOST).equals("")) {
      Graphite graphite =
          new Graphite(
              new InetSocketAddress(
                  config.getStringProperty(CoreConfig.GRAPHITE_HOST),
                  config.getIntegerProperty(CoreConfig.GRAPHITE_PORT)));

      reporter =
          GraphiteReporter.forRegistry(registry)
              .convertDurationsTo(TimeUnit.MILLISECONDS)
              .convertRatesTo(TimeUnit.SECONDS)
              .prefixedWith(config.getStringProperty(CoreConfig.GRAPHITE_PREFIX))
              .build(graphite);

      reporter.start(30l, TimeUnit.SECONDS);
    } else {
      reporter = null;
    }

    reporter2 =
        JmxReporter.forRegistry(registry)
            .convertDurationsTo(TimeUnit.MILLISECONDS)
            .convertRatesTo(TimeUnit.SECONDS)
            .build();
    reporter2.start();
  }
/**
 * Base class for getting BasicMetricsRW implementations for writing/reading basic metrics
 * (SimpleNumber). This class mostly creates:
 *
 * <ul>
 *   <li>the sample data which is read and written using the MetricsRW implementations
 *   <li>helper methods for creating {@link
 *       com.rackspacecloud.blueflood.service.SingleRollupWriteContext}
 * </ul>
 */
public class BasicMetricsRWIntegrationTest extends IntegrationTestBase {

  protected static final double EPSILON = .5;

  private static final String TENANT1 = "123456";
  private static final String TENANT2 = "987654";
  private static final String TENANT3 = "123789";

  protected LocatorIO locatorIO = new DLocatorIO();
  protected DelayedLocatorIO delayedLocatorIO = new DDelayedLocatorIO();

  protected MetricsRW datastaxMetricsRW =
      new DBasicMetricsRW(locatorIO, delayedLocatorIO, false, new DefaultClockImpl());
  protected MetricsRW astyanaxMetricsRW = new ABasicMetricsRW(false, new DefaultClockImpl());

  protected Map<Locator, IMetric> numericMap = new HashMap<Locator, IMetric>();
  protected Map<Locator, IMetric> stringMap = new HashMap<Locator, IMetric>();
  protected Map<Locator, IMetric> boolMap = new HashMap<Locator, IMetric>();

  protected static final long MAX_AGE_ALLOWED =
      Configuration.getInstance().getLongProperty(CoreConfig.ROLLUP_DELAY_MILLIS);

  protected static Granularity DELAYED_METRICS_STORAGE_GRANULARITY =
      Granularity.getRollupGranularity(
          Configuration.getInstance()
              .getStringProperty(CoreConfig.DELAYED_METRICS_STORAGE_GRANULARITY));

  /**
   * Generate numeric metrics to be used by the tests.
   *
   * @throws CacheException
   */
  @Before
  public void generateMetrics() throws CacheException {

    String className = getClass().getSimpleName();

    for (String tid : Arrays.asList(TENANT1, TENANT2, TENANT3)) {

      // Numeric
      Locator locator =
          Locator.createLocatorFromPathComponents(
              tid, className + ".numeric.metric." + System.currentTimeMillis());
      Metric metric =
          new Metric(
              locator,
              new Long(System.currentTimeMillis() % 100),
              System.currentTimeMillis(),
              new TimeValue(1, TimeUnit.DAYS),
              "unit");
      numericMap.put(locator, metric);
    }
  }

  /**
   * This method is to supply the granularity parameter to some test methods below
   *
   * @return
   */
  protected Object getGranularitiesToTest() {
    return Arrays.copyOfRange(
        Granularity.granularities(), 1, Granularity.granularities().length - 1);
  }

  /**
   * Converts the input metrics from a map of locator -> IMetric to a list of {@link
   * com.rackspacecloud.blueflood.service.SingleRollupWriteContext} objects
   *
   * @param inputMetrics
   * @return
   */
  protected List<SingleRollupWriteContext> toWriteContext(
      Collection<IMetric> inputMetrics, Granularity destGran) throws IOException {

    List<SingleRollupWriteContext> resultList = new ArrayList<SingleRollupWriteContext>();
    for (IMetric metric : inputMetrics) {

      SingleRollupWriteContext writeContext = createSingleRollupWriteContext(destGran, metric);
      resultList.add(writeContext);
    }
    return resultList;
  }

  /**
   * Convert a list of {@link com.rackspacecloud.blueflood.types.Points} into a list of {@link
   * com.rackspacecloud.blueflood.service.SingleRollupWriteContext} for the given Granularity and
   * Locator.
   *
   * @param locator
   * @param points
   * @param gran
   * @return
   */
  protected List<SingleRollupWriteContext> toWriteContext(
      Locator locator, Points<Rollup> points, Granularity gran) {

    List<SingleRollupWriteContext> resultList = new ArrayList<SingleRollupWriteContext>();

    for (Map.Entry<Long, Points.Point<Rollup>> entry : points.getPoints().entrySet()) {

      resultList.add(
          new SingleRollupWriteContext(
              entry.getValue().getData(),
              locator,
              gran,
              CassandraModel.getBasicColumnFamily(gran),
              entry.getKey()));
    }

    return resultList;
  }

  /**
   * Create a single {@link com.rackspacecloud.blueflood.service.SingleRollupWriteContext} from the
   * given {@link com.rackspacecloud.blueflood.types.IMetric} and Granularity.
   *
   * @param destGran
   * @param metric
   * @return
   * @throws IOException
   */
  protected SingleRollupWriteContext createSingleRollupWriteContext(
      Granularity destGran, IMetric metric) throws IOException {

    Locator locator = metric.getLocator();

    Points<SimpleNumber> points = new Points<SimpleNumber>();
    points.add(
        new Points.Point<SimpleNumber>(
            metric.getCollectionTime(), new SimpleNumber(metric.getMetricValue())));

    BasicRollup rollup = BasicRollup.buildRollupFromRawSamples(points);

    return new SingleRollupWriteContext(
        rollup,
        locator,
        destGran,
        CassandraModel.getBasicColumnFamily(destGran),
        metric.getCollectionTime());
  }

  /**
   * For a given list of locators, figure the shard they belong to and for all those shards get all
   * the locators in metric_locator column family
   *
   * @param ingestedLocators
   * @return
   * @throws IOException
   */
  protected Set<Locator> retrieveLocators(Set<Locator> ingestedLocators) throws IOException {

    Set<Long> shards = new HashSet<Long>();
    for (Locator locator : ingestedLocators) {
      long shard = (long) Util.getShard(locator.toString());
      shards.add(shard);
    }

    Set<Locator> locatorsFromDB = new HashSet<Locator>();
    for (Long shard : shards) {
      locatorsFromDB.addAll(locatorIO.getLocators(shard));
    }

    return locatorsFromDB;
  }

  /**
   * For a given list of metrics, figure out the shard and slot they belong to and for those shard
   * and slot combinations, get all the locators from metrics_delayed_locator column family.
   *
   * @param metrics
   * @return
   * @throws IOException
   */
  protected Set<Locator> retrieveLocatorsByShardAndSlot(List<IMetric> metrics) throws IOException {
    Set<String> slotKeys = new HashSet<String>();

    for (IMetric metric : metrics) {
      int shard = Util.getShard(metric.getLocator().toString());
      int slot = DELAYED_METRICS_STORAGE_GRANULARITY.slot(metric.getCollectionTime());
      SlotKey slotKey = SlotKey.of(DELAYED_METRICS_STORAGE_GRANULARITY, slot, shard);

      slotKeys.add(slotKey.toString());
    }

    Set<Locator> locatorsFromDB = new HashSet<Locator>();
    for (String slotKeyStr : slotKeys) {
      locatorsFromDB.addAll(delayedLocatorIO.getLocators(SlotKey.parse(slotKeyStr)));
    }

    return locatorsFromDB;
  }
}