public static boolean shouldUseESForUnits() {
return Configuration.getInstance().getBooleanProperty(CoreConfig.SHOULD_STORE_UNITS)
&& Configuration.getInstance().getBooleanProperty(CoreConfig.USE_ES_FOR_UNITS)
&& Configuration.getInstance()
.getListProperty(CoreConfig.DISCOVERY_MODULES)
.contains(ElasticIOPath);
}
public class InternalApiIntegrationTest {
private final String clusterString =
Configuration.getInstance().getStringProperty(CoreConfig.INTERNAL_API_CLUSTER);
@Test
public void testRegularWorks() throws IOException {
InternalAPI api = InternalAPIFactory.create(5, clusterString);
try {
Account acct = api.fetchAccount("acDUSBABEK");
Assert.assertNotNull(acct);
} catch (IOException ex) {
if (ex instanceof ClusterException) {
ClusterException clusterEx = (ClusterException) ex;
for (Throwable err : clusterEx.getExceptions()) err.printStackTrace();
} else ex.printStackTrace();
Assert.fail(ex.getMessage());
} finally {
shutdown(api);
}
}
@Test(expected = HttpResponseException.class)
public void testRegularHttpError() throws IOException {
InternalAPI api = InternalAPIFactory.create(1, clusterString);
Account acct = api.fetchAccount("acBOGUS___");
shutdown(api);
}
@Test
public void testConcurrentRequests() throws InterruptedException {
final int threads = 150;
final InternalAPI api = InternalAPIFactory.create(5, clusterString);
final AtomicInteger errors = new AtomicInteger(0);
final CountDownLatch latch = new CountDownLatch(threads);
for (int i = 0; i < threads; i++) {
new Thread("request " + i) {
public void run() {
try {
Account acct = api.fetchAccount("acDUSBABEK");
Assert.assertNotNull(acct);
} catch (IOException ex) {
errors.incrementAndGet();
ex.printStackTrace();
} finally {
latch.countDown();
}
}
}.start();
}
latch.await(5000, TimeUnit.MILLISECONDS);
shutdown(api);
if (errors.get() > 0) Assert.fail("There were IOExceptions on some requests.");
}
private static void shutdown(InternalAPI api) {
ClientConnectionManager connectionManager =
(ClientConnectionManager) Whitebox.getInternalState(api, "connectionManager");
connectionManager.shutdown();
}
}
public HttpMultiRollupsQueryHandler() {
Configuration config = Configuration.getInstance();
int maxThreadsToUse = config.getIntegerProperty(HttpConfig.MAX_READ_WORKER_THREADS);
int maxQueueSize = config.getIntegerProperty(HttpConfig.MAX_BATCH_READ_REQUESTS_TO_QUEUE);
this.maxMetricsPerRequest = config.getIntegerProperty(HttpConfig.MAX_METRICS_PER_BATCH_QUERY);
this.serializer = new BatchedMetricsJSONOutputSerializer();
this.gson = new GsonBuilder().setPrettyPrinting().serializeNulls().create();
this.parser = new JsonParser();
this.executor =
new ThreadPoolBuilder()
.withCorePoolSize(maxThreadsToUse)
.withMaxPoolSize(maxThreadsToUse)
.withName("HTTP-BatchMetricsFetch")
.withBoundedQueue(maxQueueSize)
.build();
}
@BeforeClass
public static void setUpHttp() {
queryPort =
Configuration.getInstance().getIntegerProperty(HttpConfig.HTTP_METRIC_DATA_QUERY_PORT);
httpQueryService = new HttpQueryService();
httpQueryService.startService();
vendor = new HttpClientVendor();
client = vendor.getClient();
}
static {
Configuration config = Configuration.getInstance();
// register jvm metrics
MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
if (!System.getProperty("java.version").split("\\.")[1].equals("6")) {
// if not running 1.6
registry.registerAll(
new PrefixedMetricSet(new BufferPoolMetricSet(mbs), JVM_PREFIX, "buffer-pool"));
}
registry.registerAll(new PrefixedMetricSet(new GarbageCollectorMetricSet(), JVM_PREFIX, "gc"));
registry.registerAll(new PrefixedMetricSet(new MemoryUsageGaugeSet(), JVM_PREFIX, "memory"));
registry.registerAll(
new PrefixedMetricSet(new ThreadStatesGaugeSet(), JVM_PREFIX, "thread-states"));
// instrument log4j
InstrumentedAppender appender = new InstrumentedAppender(registry);
appender.activateOptions();
LogManager.getRootLogger().addAppender(appender);
if (!config.getStringProperty(CoreConfig.RIEMANN_HOST).equals("")) {
RiemannReporter tmpreporter;
try {
Riemann riemann =
new Riemann(
config.getStringProperty(CoreConfig.RIEMANN_HOST),
config.getIntegerProperty(CoreConfig.RIEMANN_PORT));
RiemannReporter.Builder builder =
RiemannReporter.forRegistry(registry)
.convertDurationsTo(TimeUnit.MILLISECONDS)
.convertRatesTo(TimeUnit.SECONDS);
if (!config.getStringProperty(CoreConfig.RIEMANN_SEPARATOR).isEmpty()) {
builder.useSeparator(config.getStringProperty(CoreConfig.RIEMANN_SEPARATOR));
}
if (!config.getStringProperty(CoreConfig.RIEMANN_TTL).isEmpty()) {
builder.withTtl(config.getFloatProperty(CoreConfig.RIEMANN_TTL));
}
if (!config.getStringProperty(CoreConfig.RIEMANN_LOCALHOST).isEmpty()) {
builder.localHost(config.getStringProperty(CoreConfig.RIEMANN_LOCALHOST));
}
if (!config.getStringProperty(CoreConfig.RIEMANN_PREFIX).isEmpty()) {
builder.prefixedWith(config.getStringProperty(CoreConfig.RIEMANN_PREFIX));
}
if (!config.getStringProperty(CoreConfig.RIEMANN_TAGS).isEmpty()) {
builder.tags(config.getListProperty(CoreConfig.RIEMANN_TAGS));
}
tmpreporter = builder.build(riemann);
tmpreporter.start(30l, TimeUnit.SECONDS);
} catch (IOException e) {
tmpreporter = null;
}
reporter1 = tmpreporter;
} else {
reporter1 = null;
}
if (!config.getStringProperty(CoreConfig.GRAPHITE_HOST).equals("")) {
Graphite graphite =
new Graphite(
new InetSocketAddress(
config.getStringProperty(CoreConfig.GRAPHITE_HOST),
config.getIntegerProperty(CoreConfig.GRAPHITE_PORT)));
reporter =
GraphiteReporter.forRegistry(registry)
.convertDurationsTo(TimeUnit.MILLISECONDS)
.convertRatesTo(TimeUnit.SECONDS)
.prefixedWith(config.getStringProperty(CoreConfig.GRAPHITE_PREFIX))
.build(graphite);
reporter.start(30l, TimeUnit.SECONDS);
} else {
reporter = null;
}
reporter2 =
JmxReporter.forRegistry(registry)
.convertDurationsTo(TimeUnit.MILLISECONDS)
.convertRatesTo(TimeUnit.SECONDS)
.build();
reporter2.start();
}
/**
* Base class for getting BasicMetricsRW implementations for writing/reading basic metrics
* (SimpleNumber). This class mostly creates:
*
* <ul>
* <li>the sample data which is read and written using the MetricsRW implementations
* <li>helper methods for creating {@link
* com.rackspacecloud.blueflood.service.SingleRollupWriteContext}
* </ul>
*/
public class BasicMetricsRWIntegrationTest extends IntegrationTestBase {
protected static final double EPSILON = .5;
private static final String TENANT1 = "123456";
private static final String TENANT2 = "987654";
private static final String TENANT3 = "123789";
protected LocatorIO locatorIO = new DLocatorIO();
protected DelayedLocatorIO delayedLocatorIO = new DDelayedLocatorIO();
protected MetricsRW datastaxMetricsRW =
new DBasicMetricsRW(locatorIO, delayedLocatorIO, false, new DefaultClockImpl());
protected MetricsRW astyanaxMetricsRW = new ABasicMetricsRW(false, new DefaultClockImpl());
protected Map<Locator, IMetric> numericMap = new HashMap<Locator, IMetric>();
protected Map<Locator, IMetric> stringMap = new HashMap<Locator, IMetric>();
protected Map<Locator, IMetric> boolMap = new HashMap<Locator, IMetric>();
protected static final long MAX_AGE_ALLOWED =
Configuration.getInstance().getLongProperty(CoreConfig.ROLLUP_DELAY_MILLIS);
protected static Granularity DELAYED_METRICS_STORAGE_GRANULARITY =
Granularity.getRollupGranularity(
Configuration.getInstance()
.getStringProperty(CoreConfig.DELAYED_METRICS_STORAGE_GRANULARITY));
/**
* Generate numeric metrics to be used by the tests.
*
* @throws CacheException
*/
@Before
public void generateMetrics() throws CacheException {
String className = getClass().getSimpleName();
for (String tid : Arrays.asList(TENANT1, TENANT2, TENANT3)) {
// Numeric
Locator locator =
Locator.createLocatorFromPathComponents(
tid, className + ".numeric.metric." + System.currentTimeMillis());
Metric metric =
new Metric(
locator,
new Long(System.currentTimeMillis() % 100),
System.currentTimeMillis(),
new TimeValue(1, TimeUnit.DAYS),
"unit");
numericMap.put(locator, metric);
}
}
/**
* This method is to supply the granularity parameter to some test methods below
*
* @return
*/
protected Object getGranularitiesToTest() {
return Arrays.copyOfRange(
Granularity.granularities(), 1, Granularity.granularities().length - 1);
}
/**
* Converts the input metrics from a map of locator -> IMetric to a list of {@link
* com.rackspacecloud.blueflood.service.SingleRollupWriteContext} objects
*
* @param inputMetrics
* @return
*/
protected List<SingleRollupWriteContext> toWriteContext(
Collection<IMetric> inputMetrics, Granularity destGran) throws IOException {
List<SingleRollupWriteContext> resultList = new ArrayList<SingleRollupWriteContext>();
for (IMetric metric : inputMetrics) {
SingleRollupWriteContext writeContext = createSingleRollupWriteContext(destGran, metric);
resultList.add(writeContext);
}
return resultList;
}
/**
* Convert a list of {@link com.rackspacecloud.blueflood.types.Points} into a list of {@link
* com.rackspacecloud.blueflood.service.SingleRollupWriteContext} for the given Granularity and
* Locator.
*
* @param locator
* @param points
* @param gran
* @return
*/
protected List<SingleRollupWriteContext> toWriteContext(
Locator locator, Points<Rollup> points, Granularity gran) {
List<SingleRollupWriteContext> resultList = new ArrayList<SingleRollupWriteContext>();
for (Map.Entry<Long, Points.Point<Rollup>> entry : points.getPoints().entrySet()) {
resultList.add(
new SingleRollupWriteContext(
entry.getValue().getData(),
locator,
gran,
CassandraModel.getBasicColumnFamily(gran),
entry.getKey()));
}
return resultList;
}
/**
* Create a single {@link com.rackspacecloud.blueflood.service.SingleRollupWriteContext} from the
* given {@link com.rackspacecloud.blueflood.types.IMetric} and Granularity.
*
* @param destGran
* @param metric
* @return
* @throws IOException
*/
protected SingleRollupWriteContext createSingleRollupWriteContext(
Granularity destGran, IMetric metric) throws IOException {
Locator locator = metric.getLocator();
Points<SimpleNumber> points = new Points<SimpleNumber>();
points.add(
new Points.Point<SimpleNumber>(
metric.getCollectionTime(), new SimpleNumber(metric.getMetricValue())));
BasicRollup rollup = BasicRollup.buildRollupFromRawSamples(points);
return new SingleRollupWriteContext(
rollup,
locator,
destGran,
CassandraModel.getBasicColumnFamily(destGran),
metric.getCollectionTime());
}
/**
* For a given list of locators, figure the shard they belong to and for all those shards get all
* the locators in metric_locator column family
*
* @param ingestedLocators
* @return
* @throws IOException
*/
protected Set<Locator> retrieveLocators(Set<Locator> ingestedLocators) throws IOException {
Set<Long> shards = new HashSet<Long>();
for (Locator locator : ingestedLocators) {
long shard = (long) Util.getShard(locator.toString());
shards.add(shard);
}
Set<Locator> locatorsFromDB = new HashSet<Locator>();
for (Long shard : shards) {
locatorsFromDB.addAll(locatorIO.getLocators(shard));
}
return locatorsFromDB;
}
/**
* For a given list of metrics, figure out the shard and slot they belong to and for those shard
* and slot combinations, get all the locators from metrics_delayed_locator column family.
*
* @param metrics
* @return
* @throws IOException
*/
protected Set<Locator> retrieveLocatorsByShardAndSlot(List<IMetric> metrics) throws IOException {
Set<String> slotKeys = new HashSet<String>();
for (IMetric metric : metrics) {
int shard = Util.getShard(metric.getLocator().toString());
int slot = DELAYED_METRICS_STORAGE_GRANULARITY.slot(metric.getCollectionTime());
SlotKey slotKey = SlotKey.of(DELAYED_METRICS_STORAGE_GRANULARITY, slot, shard);
slotKeys.add(slotKey.toString());
}
Set<Locator> locatorsFromDB = new HashSet<Locator>();
for (String slotKeyStr : slotKeys) {
locatorsFromDB.addAll(delayedLocatorIO.getLocators(SlotKey.parse(slotKeyStr)));
}
return locatorsFromDB;
}
}
