private int getNumberOfPointsViaHTTPHandler(
HttpRollupsQueryHandler handler, Locator locator, long from, long to, Resolution resolution)
throws Exception {
final MetricData values =
handler.GetDataByResolution(
locator.getTenantId(), locator.getMetricName(), from, to, resolution);
return values.getData().getPoints().size();
}
private void testHappyCaseMultiFetchHTTPRequest() throws Exception {
HttpPost post = new HttpPost(getBatchMetricsQueryURI());
JSONArray metricsToGet = new JSONArray();
for (Locator locator : locators) {
metricsToGet.add(locator.toString());
}
HttpEntity entity = new StringEntity(metricsToGet.toString(), ContentType.APPLICATION_JSON);
post.setEntity(entity);
HttpResponse response = client.execute(post);
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
}
/**
* For a given list of locators, figure the shard they belong to and for all those shards get all
* the locators in metric_locator column family
*
* @param ingestedLocators
* @return
* @throws IOException
*/
protected Set<Locator> retrieveLocators(Set<Locator> ingestedLocators) throws IOException {
Set<Long> shards = new HashSet<Long>();
for (Locator locator : ingestedLocators) {
long shard = (long) Util.getShard(locator.toString());
shards.add(shard);
}
Set<Locator> locatorsFromDB = new HashSet<Locator>();
for (Long shard : shards) {
locatorsFromDB.addAll(locatorIO.getLocators(shard));
}
return locatorsFromDB;
}
private void testHTTPRollupHandlerGetByPoints(
Map<Locator, Map<Granularity, Integer>> answers,
Map<Granularity, Integer> points,
long from,
long to)
throws Exception {
for (Locator locator : locators) {
for (Granularity g2 : Granularity.granularities()) {
MetricData data =
httpHandler.GetDataByPoints(
locator.getTenantId(),
locator.getMetricName(),
baseMillis,
baseMillis + 86400000,
points.get(g2));
Assert.assertEquals((int) answers.get(locator).get(g2), data.getData().getPoints().size());
Assert.assertEquals(locatorToUnitMap.get(locator), data.getUnit());
}
}
}
/**
* Generate numeric metrics to be used by the tests.
*
* @throws CacheException
*/
@Before
public void generateMetrics() throws CacheException {
String className = getClass().getSimpleName();
for (String tid : Arrays.asList(TENANT1, TENANT2, TENANT3)) {
// Numeric
Locator locator =
Locator.createLocatorFromPathComponents(
tid, className + ".numeric.metric." + System.currentTimeMillis());
Metric metric =
new Metric(
locator,
new Long(System.currentTimeMillis() % 100),
System.currentTimeMillis(),
new TimeValue(1, TimeUnit.DAYS),
"unit");
numericMap.put(locator, metric);
}
}
@Override
public Metric deserialize(JsonParser jp, DeserializationContext ctxt)
throws IOException, JsonProcessingException {
Object metricValue = null;
Long collectionTime = null;
Long ttlInSeconds = null;
String unit = null;
String tenantId = null;
String metricName = null;
String fieldName = "";
while (!jp.getCurrentToken().equals(JsonToken.END_OBJECT)) {
if (jp.getCurrentToken().equals(JsonToken.FIELD_NAME)) {
fieldName = jp.getText();
}
jp.nextToken();
if (validFields.contains(fieldName)) {
if (fieldName.equals("class")) {
if (!jp.getText().equals("raw")) {
throw new IOException(
"Metric.class deserialization requires field 'class' to be 'raw'");
}
} else if (fieldName.equals("metricValue")) {
if (jp.getCurrentToken().isNumeric()) {
metricValue = jp.getNumberValue();
} else if (jp.getCurrentToken().equals(JsonToken.VALUE_FALSE)) {
metricValue = false;
} else if (jp.getCurrentToken().equals(JsonToken.VALUE_TRUE)) {
metricValue = true;
} else {
metricValue = jp.getText();
}
} else if (fieldName.equals("collectionTime")) {
collectionTime = jp.getLongValue();
} else if (fieldName.equals("ttlInSeconds")) {
ttlInSeconds = jp.getLongValue();
} else if (fieldName.equals("unit")) {
unit = jp.getText();
} else if (fieldName.equals("tenantId")) {
tenantId = jp.getText();
} else if (fieldName.equals("metricName")) {
metricName = jp.getText();
}
}
jp.nextToken();
}
if (metricValue == null
|| collectionTime == null
|| ttlInSeconds == null
|| unit == null
|| tenantId == null
|| metricName == null) {
throw new IOException("A required field was not found."); // TODO: specificity
}
Locator locator = Locator.createLocatorFromPathComponents(tenantId, metricName);
Metric m =
new Metric(
locator,
metricValue,
collectionTime,
new TimeValue(ttlInSeconds, TimeUnit.SECONDS),
unit);
return m;
}
public class HttpRollupHandlerIntegrationTest extends IntegrationTestBase {
private final long baseMillis = 1335820166000L;
private final String tenantId = "ac" + IntegrationTestBase.randString(8);
private final String metricName = "met_" + IntegrationTestBase.randString(8);
private final String strMetricName = "strMet_" + IntegrationTestBase.randString(8);
final Locator[] locators =
new Locator[] {
Locator.createLocatorFromPathComponents(tenantId, metricName),
Locator.createLocatorFromPathComponents(tenantId, strMetricName)
};
private static int queryPort = 20000;
private static HttpQueryService httpQueryService;
private static HttpClientVendor vendor;
private static DefaultHttpClient client;
private HttpRollupsQueryHandler httpHandler;
private final Map<Locator, Map<Granularity, Integer>> locatorToPoints =
new HashMap<Locator, Map<Granularity, Integer>>();
@BeforeClass
public static void setUpHttp() {
queryPort =
Configuration.getInstance().getIntegerProperty(HttpConfig.HTTP_METRIC_DATA_QUERY_PORT);
httpQueryService = new HttpQueryService();
httpQueryService.startService();
vendor = new HttpClientVendor();
client = vendor.getClient();
}
@Before
public void setUp() throws Exception {
super.setUp();
AstyanaxWriter writer = AstyanaxWriter.getInstance();
IncomingMetricMetadataAnalyzer analyzer =
new IncomingMetricMetadataAnalyzer(MetadataCache.getInstance());
// insert something every 1m for 24h
for (int i = 0; i < 1440; i++) {
final long curMillis = baseMillis + i * 60000;
final List<Metric> metrics = new ArrayList<Metric>();
final Metric metric = getRandomIntMetric(locators[0], curMillis);
final Metric stringMetric = getRandomStringmetric(locators[1], curMillis);
metrics.add(metric);
metrics.add(stringMetric);
analyzer.scanMetrics(new ArrayList<IMetric>(metrics));
writer.insertFull(metrics);
}
httpHandler = new HttpRollupsQueryHandler();
// generate every level of rollup for the raw data
Granularity g = Granularity.FULL;
while (g != Granularity.MIN_1440) {
g = g.coarser();
for (Locator locator : locators) {
generateRollups(locator, baseMillis, baseMillis + 86400000, g);
}
}
final Map<Granularity, Integer> answerForNumericMetric = new HashMap<Granularity, Integer>();
answerForNumericMetric.put(Granularity.FULL, 1440);
answerForNumericMetric.put(Granularity.MIN_5, 289);
answerForNumericMetric.put(Granularity.MIN_20, 73);
answerForNumericMetric.put(Granularity.MIN_60, 25);
answerForNumericMetric.put(Granularity.MIN_240, 7);
answerForNumericMetric.put(Granularity.MIN_1440, 2);
final Map<Granularity, Integer> answerForStringMetric = new HashMap<Granularity, Integer>();
answerForStringMetric.put(Granularity.FULL, 1440);
answerForStringMetric.put(Granularity.MIN_5, 1440);
answerForStringMetric.put(Granularity.MIN_20, 1440);
answerForStringMetric.put(Granularity.MIN_60, 1440);
answerForStringMetric.put(Granularity.MIN_240, 1440);
answerForStringMetric.put(Granularity.MIN_1440, 1440);
locatorToPoints.put(locators[0], answerForNumericMetric);
locatorToPoints.put(locators[1], answerForStringMetric);
}
@Test
public void testGetPoints() throws Exception {
testGetRollupByPoints();
testGetRollupByResolution();
testHttpRequestForPoints();
testHttpRequestForHistograms();
}
private void testGetRollupByPoints() throws Exception {
final Map<Granularity, Integer> points = new HashMap<Granularity, Integer>();
points.put(Granularity.FULL, 1600);
points.put(Granularity.MIN_5, 287);
points.put(Granularity.MIN_20, 71);
points.put(Granularity.MIN_60, 23);
points.put(Granularity.MIN_240, 5);
points.put(Granularity.MIN_1440, 1);
testHTTPRollupHandlerGetByPoints(locatorToPoints, points, baseMillis, baseMillis + 86400000);
}
private void testGetRollupByResolution() throws Exception {
for (Locator locator : locators) {
for (Resolution resolution : Resolution.values()) {
Granularity g = Granularity.granularities()[resolution.getValue()];
testHTTPHandlersGetByResolution(
locator,
resolution,
baseMillis,
baseMillis + 86400000,
locatorToPoints.get(locator).get(g));
}
}
}
private void testHTTPRollupHandlerGetByPoints(
Map<Locator, Map<Granularity, Integer>> answers,
Map<Granularity, Integer> points,
long from,
long to)
throws Exception {
for (Locator locator : locators) {
for (Granularity g2 : Granularity.granularities()) {
MetricData data =
httpHandler.GetDataByPoints(
locator.getTenantId(),
locator.getMetricName(),
baseMillis,
baseMillis + 86400000,
points.get(g2));
Assert.assertEquals((int) answers.get(locator).get(g2), data.getData().getPoints().size());
Assert.assertEquals(locatorToUnitMap.get(locator), data.getUnit());
}
}
}
private void testHTTPHandlersGetByResolution(
Locator locator, Resolution resolution, long from, long to, int expectedPoints)
throws Exception {
Assert.assertEquals(
expectedPoints,
getNumberOfPointsViaHTTPHandler(httpHandler, locator, from, to, resolution));
}
private int getNumberOfPointsViaHTTPHandler(
HttpRollupsQueryHandler handler, Locator locator, long from, long to, Resolution resolution)
throws Exception {
final MetricData values =
handler.GetDataByResolution(
locator.getTenantId(), locator.getMetricName(), from, to, resolution);
return values.getData().getPoints().size();
}
private void testHttpRequestForPoints() throws Exception {
testHappyCaseHTTPRequest();
testBadRequest();
testBadMethod();
testHappyCaseMultiFetchHTTPRequest();
}
private void testHappyCaseHTTPRequest() throws Exception {
HttpGet get = new HttpGet(getMetricsQueryURI());
HttpResponse response = client.execute(get);
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
}
private void testHttpRequestForHistograms() throws Exception {
HttpGet get = new HttpGet(getHistQueryURI());
HttpResponse response = client.execute(get);
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
}
private void testBadRequest() throws Exception {
HttpGet get = new HttpGet(getInvalidMetricsQueryURI());
HttpResponse response = client.execute(get);
Assert.assertEquals(400, response.getStatusLine().getStatusCode());
}
private void testBadMethod() throws Exception {
HttpPost post = new HttpPost(getMetricsQueryURI());
HttpResponse response = client.execute(post);
Assert.assertEquals(405, response.getStatusLine().getStatusCode());
}
private void testHappyCaseMultiFetchHTTPRequest() throws Exception {
HttpPost post = new HttpPost(getBatchMetricsQueryURI());
JSONArray metricsToGet = new JSONArray();
for (Locator locator : locators) {
metricsToGet.add(locator.toString());
}
HttpEntity entity = new StringEntity(metricsToGet.toString(), ContentType.APPLICATION_JSON);
post.setEntity(entity);
HttpResponse response = client.execute(post);
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
}
private URI getMetricsQueryURI() throws URISyntaxException {
URIBuilder builder =
new URIBuilder()
.setScheme("http")
.setHost("127.0.0.1")
.setPort(queryPort)
.setPath("/v2.0/" + tenantId + "/views/" + metricName)
.setParameter("from", String.valueOf(baseMillis))
.setParameter("to", String.valueOf(baseMillis + 86400000))
.setParameter("resolution", "full");
return builder.build();
}
private URI getHistQueryURI() throws URISyntaxException {
URIBuilder builder =
new URIBuilder()
.setScheme("http")
.setHost("127.0.0.1")
.setPort(queryPort)
.setPath("/v2.0/" + tenantId + "/views/histograms/" + metricName)
.setParameter("from", String.valueOf(baseMillis))
.setParameter("to", String.valueOf(baseMillis + 86400000))
.setParameter("resolution", "full");
return builder.build();
}
private URI getBatchMetricsQueryURI() throws Exception {
URIBuilder builder =
new URIBuilder()
.setScheme("http")
.setHost("127.0.0.1")
.setPort(queryPort)
.setPath("/v2.0/" + tenantId + "/views")
.setParameter("from", String.valueOf(baseMillis))
.setParameter("to", String.valueOf(baseMillis + 86400000))
.setParameter("resolution", "full");
return builder.build();
}
private URI getInvalidMetricsQueryURI() throws URISyntaxException {
URIBuilder builder =
new URIBuilder()
.setScheme("http")
.setHost("127.0.0.1")
.setPort(queryPort)
.setPath("/v2.0/" + tenantId + "/views/" + metricName)
.setParameter("from", String.valueOf(baseMillis))
.setParameter("resolution", "full"); // Misses parameter 'to'
return builder.build();
}
@AfterClass
public static void shutdown() {
vendor.shutdown();
httpQueryService.stopService();
}
}