private int getNumberOfPointsViaHTTPHandler(
     HttpRollupsQueryHandler handler, Locator locator, long from, long to, Resolution resolution)
     throws Exception {
   final MetricData values =
       handler.GetDataByResolution(
           locator.getTenantId(), locator.getMetricName(), from, to, resolution);
   return values.getData().getPoints().size();
 }
 private void testHappyCaseMultiFetchHTTPRequest() throws Exception {
   HttpPost post = new HttpPost(getBatchMetricsQueryURI());
   JSONArray metricsToGet = new JSONArray();
   for (Locator locator : locators) {
     metricsToGet.add(locator.toString());
   }
   HttpEntity entity = new StringEntity(metricsToGet.toString(), ContentType.APPLICATION_JSON);
   post.setEntity(entity);
   HttpResponse response = client.execute(post);
   Assert.assertEquals(200, response.getStatusLine().getStatusCode());
 }
  /**
   * For a given list of locators, figure the shard they belong to and for all those shards get all
   * the locators in metric_locator column family
   *
   * @param ingestedLocators
   * @return
   * @throws IOException
   */
  protected Set<Locator> retrieveLocators(Set<Locator> ingestedLocators) throws IOException {

    Set<Long> shards = new HashSet<Long>();
    for (Locator locator : ingestedLocators) {
      long shard = (long) Util.getShard(locator.toString());
      shards.add(shard);
    }

    Set<Locator> locatorsFromDB = new HashSet<Locator>();
    for (Long shard : shards) {
      locatorsFromDB.addAll(locatorIO.getLocators(shard));
    }

    return locatorsFromDB;
  }
 private void testHTTPRollupHandlerGetByPoints(
     Map<Locator, Map<Granularity, Integer>> answers,
     Map<Granularity, Integer> points,
     long from,
     long to)
     throws Exception {
   for (Locator locator : locators) {
     for (Granularity g2 : Granularity.granularities()) {
       MetricData data =
           httpHandler.GetDataByPoints(
               locator.getTenantId(),
               locator.getMetricName(),
               baseMillis,
               baseMillis + 86400000,
               points.get(g2));
       Assert.assertEquals((int) answers.get(locator).get(g2), data.getData().getPoints().size());
       Assert.assertEquals(locatorToUnitMap.get(locator), data.getUnit());
     }
   }
 }
  /**
   * Generate numeric metrics to be used by the tests.
   *
   * @throws CacheException
   */
  @Before
  public void generateMetrics() throws CacheException {

    String className = getClass().getSimpleName();

    for (String tid : Arrays.asList(TENANT1, TENANT2, TENANT3)) {

      // Numeric
      Locator locator =
          Locator.createLocatorFromPathComponents(
              tid, className + ".numeric.metric." + System.currentTimeMillis());
      Metric metric =
          new Metric(
              locator,
              new Long(System.currentTimeMillis() % 100),
              System.currentTimeMillis(),
              new TimeValue(1, TimeUnit.DAYS),
              "unit");
      numericMap.put(locator, metric);
    }
  }
Пример #6
0
  @Override
  public Metric deserialize(JsonParser jp, DeserializationContext ctxt)
      throws IOException, JsonProcessingException {
    Object metricValue = null;
    Long collectionTime = null;
    Long ttlInSeconds = null;
    String unit = null;
    String tenantId = null;
    String metricName = null;

    String fieldName = "";
    while (!jp.getCurrentToken().equals(JsonToken.END_OBJECT)) {

      if (jp.getCurrentToken().equals(JsonToken.FIELD_NAME)) {
        fieldName = jp.getText();
      }
      jp.nextToken();

      if (validFields.contains(fieldName)) {
        if (fieldName.equals("class")) {
          if (!jp.getText().equals("raw")) {
            throw new IOException(
                "Metric.class deserialization requires field 'class' to be 'raw'");
          }
        } else if (fieldName.equals("metricValue")) {
          if (jp.getCurrentToken().isNumeric()) {
            metricValue = jp.getNumberValue();
          } else if (jp.getCurrentToken().equals(JsonToken.VALUE_FALSE)) {
            metricValue = false;
          } else if (jp.getCurrentToken().equals(JsonToken.VALUE_TRUE)) {
            metricValue = true;
          } else {
            metricValue = jp.getText();
          }
        } else if (fieldName.equals("collectionTime")) {
          collectionTime = jp.getLongValue();
        } else if (fieldName.equals("ttlInSeconds")) {
          ttlInSeconds = jp.getLongValue();
        } else if (fieldName.equals("unit")) {
          unit = jp.getText();
        } else if (fieldName.equals("tenantId")) {
          tenantId = jp.getText();
        } else if (fieldName.equals("metricName")) {
          metricName = jp.getText();
        }
      }

      jp.nextToken();
    }

    if (metricValue == null
        || collectionTime == null
        || ttlInSeconds == null
        || unit == null
        || tenantId == null
        || metricName == null) {
      throw new IOException("A required field was not found."); // TODO: specificity
    }

    Locator locator = Locator.createLocatorFromPathComponents(tenantId, metricName);
    Metric m =
        new Metric(
            locator,
            metricValue,
            collectionTime,
            new TimeValue(ttlInSeconds, TimeUnit.SECONDS),
            unit);
    return m;
  }
public class HttpRollupHandlerIntegrationTest extends IntegrationTestBase {
  private final long baseMillis = 1335820166000L;
  private final String tenantId = "ac" + IntegrationTestBase.randString(8);
  private final String metricName = "met_" + IntegrationTestBase.randString(8);
  private final String strMetricName = "strMet_" + IntegrationTestBase.randString(8);
  final Locator[] locators =
      new Locator[] {
        Locator.createLocatorFromPathComponents(tenantId, metricName),
        Locator.createLocatorFromPathComponents(tenantId, strMetricName)
      };
  private static int queryPort = 20000;
  private static HttpQueryService httpQueryService;
  private static HttpClientVendor vendor;
  private static DefaultHttpClient client;

  private HttpRollupsQueryHandler httpHandler;
  private final Map<Locator, Map<Granularity, Integer>> locatorToPoints =
      new HashMap<Locator, Map<Granularity, Integer>>();

  @BeforeClass
  public static void setUpHttp() {
    queryPort =
        Configuration.getInstance().getIntegerProperty(HttpConfig.HTTP_METRIC_DATA_QUERY_PORT);
    httpQueryService = new HttpQueryService();
    httpQueryService.startService();
    vendor = new HttpClientVendor();
    client = vendor.getClient();
  }

  @Before
  public void setUp() throws Exception {
    super.setUp();
    AstyanaxWriter writer = AstyanaxWriter.getInstance();
    IncomingMetricMetadataAnalyzer analyzer =
        new IncomingMetricMetadataAnalyzer(MetadataCache.getInstance());

    // insert something every 1m for 24h
    for (int i = 0; i < 1440; i++) {
      final long curMillis = baseMillis + i * 60000;
      final List<Metric> metrics = new ArrayList<Metric>();
      final Metric metric = getRandomIntMetric(locators[0], curMillis);
      final Metric stringMetric = getRandomStringmetric(locators[1], curMillis);
      metrics.add(metric);
      metrics.add(stringMetric);

      analyzer.scanMetrics(new ArrayList<IMetric>(metrics));
      writer.insertFull(metrics);
    }

    httpHandler = new HttpRollupsQueryHandler();

    // generate every level of rollup for the raw data
    Granularity g = Granularity.FULL;
    while (g != Granularity.MIN_1440) {
      g = g.coarser();
      for (Locator locator : locators) {
        generateRollups(locator, baseMillis, baseMillis + 86400000, g);
      }
    }

    final Map<Granularity, Integer> answerForNumericMetric = new HashMap<Granularity, Integer>();
    answerForNumericMetric.put(Granularity.FULL, 1440);
    answerForNumericMetric.put(Granularity.MIN_5, 289);
    answerForNumericMetric.put(Granularity.MIN_20, 73);
    answerForNumericMetric.put(Granularity.MIN_60, 25);
    answerForNumericMetric.put(Granularity.MIN_240, 7);
    answerForNumericMetric.put(Granularity.MIN_1440, 2);

    final Map<Granularity, Integer> answerForStringMetric = new HashMap<Granularity, Integer>();
    answerForStringMetric.put(Granularity.FULL, 1440);
    answerForStringMetric.put(Granularity.MIN_5, 1440);
    answerForStringMetric.put(Granularity.MIN_20, 1440);
    answerForStringMetric.put(Granularity.MIN_60, 1440);
    answerForStringMetric.put(Granularity.MIN_240, 1440);
    answerForStringMetric.put(Granularity.MIN_1440, 1440);

    locatorToPoints.put(locators[0], answerForNumericMetric);
    locatorToPoints.put(locators[1], answerForStringMetric);
  }

  @Test
  public void testGetPoints() throws Exception {
    testGetRollupByPoints();
    testGetRollupByResolution();
    testHttpRequestForPoints();
    testHttpRequestForHistograms();
  }

  private void testGetRollupByPoints() throws Exception {
    final Map<Granularity, Integer> points = new HashMap<Granularity, Integer>();
    points.put(Granularity.FULL, 1600);
    points.put(Granularity.MIN_5, 287);
    points.put(Granularity.MIN_20, 71);
    points.put(Granularity.MIN_60, 23);
    points.put(Granularity.MIN_240, 5);
    points.put(Granularity.MIN_1440, 1);

    testHTTPRollupHandlerGetByPoints(locatorToPoints, points, baseMillis, baseMillis + 86400000);
  }

  private void testGetRollupByResolution() throws Exception {
    for (Locator locator : locators) {
      for (Resolution resolution : Resolution.values()) {
        Granularity g = Granularity.granularities()[resolution.getValue()];
        testHTTPHandlersGetByResolution(
            locator,
            resolution,
            baseMillis,
            baseMillis + 86400000,
            locatorToPoints.get(locator).get(g));
      }
    }
  }

  private void testHTTPRollupHandlerGetByPoints(
      Map<Locator, Map<Granularity, Integer>> answers,
      Map<Granularity, Integer> points,
      long from,
      long to)
      throws Exception {
    for (Locator locator : locators) {
      for (Granularity g2 : Granularity.granularities()) {
        MetricData data =
            httpHandler.GetDataByPoints(
                locator.getTenantId(),
                locator.getMetricName(),
                baseMillis,
                baseMillis + 86400000,
                points.get(g2));
        Assert.assertEquals((int) answers.get(locator).get(g2), data.getData().getPoints().size());
        Assert.assertEquals(locatorToUnitMap.get(locator), data.getUnit());
      }
    }
  }

  private void testHTTPHandlersGetByResolution(
      Locator locator, Resolution resolution, long from, long to, int expectedPoints)
      throws Exception {
    Assert.assertEquals(
        expectedPoints,
        getNumberOfPointsViaHTTPHandler(httpHandler, locator, from, to, resolution));
  }

  private int getNumberOfPointsViaHTTPHandler(
      HttpRollupsQueryHandler handler, Locator locator, long from, long to, Resolution resolution)
      throws Exception {
    final MetricData values =
        handler.GetDataByResolution(
            locator.getTenantId(), locator.getMetricName(), from, to, resolution);
    return values.getData().getPoints().size();
  }

  private void testHttpRequestForPoints() throws Exception {
    testHappyCaseHTTPRequest();
    testBadRequest();
    testBadMethod();
    testHappyCaseMultiFetchHTTPRequest();
  }

  private void testHappyCaseHTTPRequest() throws Exception {
    HttpGet get = new HttpGet(getMetricsQueryURI());
    HttpResponse response = client.execute(get);
    Assert.assertEquals(200, response.getStatusLine().getStatusCode());
  }

  private void testHttpRequestForHistograms() throws Exception {
    HttpGet get = new HttpGet(getHistQueryURI());
    HttpResponse response = client.execute(get);
    Assert.assertEquals(200, response.getStatusLine().getStatusCode());
  }

  private void testBadRequest() throws Exception {
    HttpGet get = new HttpGet(getInvalidMetricsQueryURI());
    HttpResponse response = client.execute(get);
    Assert.assertEquals(400, response.getStatusLine().getStatusCode());
  }

  private void testBadMethod() throws Exception {
    HttpPost post = new HttpPost(getMetricsQueryURI());
    HttpResponse response = client.execute(post);
    Assert.assertEquals(405, response.getStatusLine().getStatusCode());
  }

  private void testHappyCaseMultiFetchHTTPRequest() throws Exception {
    HttpPost post = new HttpPost(getBatchMetricsQueryURI());
    JSONArray metricsToGet = new JSONArray();
    for (Locator locator : locators) {
      metricsToGet.add(locator.toString());
    }
    HttpEntity entity = new StringEntity(metricsToGet.toString(), ContentType.APPLICATION_JSON);
    post.setEntity(entity);
    HttpResponse response = client.execute(post);
    Assert.assertEquals(200, response.getStatusLine().getStatusCode());
  }

  private URI getMetricsQueryURI() throws URISyntaxException {
    URIBuilder builder =
        new URIBuilder()
            .setScheme("http")
            .setHost("127.0.0.1")
            .setPort(queryPort)
            .setPath("/v2.0/" + tenantId + "/views/" + metricName)
            .setParameter("from", String.valueOf(baseMillis))
            .setParameter("to", String.valueOf(baseMillis + 86400000))
            .setParameter("resolution", "full");
    return builder.build();
  }

  private URI getHistQueryURI() throws URISyntaxException {
    URIBuilder builder =
        new URIBuilder()
            .setScheme("http")
            .setHost("127.0.0.1")
            .setPort(queryPort)
            .setPath("/v2.0/" + tenantId + "/views/histograms/" + metricName)
            .setParameter("from", String.valueOf(baseMillis))
            .setParameter("to", String.valueOf(baseMillis + 86400000))
            .setParameter("resolution", "full");
    return builder.build();
  }

  private URI getBatchMetricsQueryURI() throws Exception {
    URIBuilder builder =
        new URIBuilder()
            .setScheme("http")
            .setHost("127.0.0.1")
            .setPort(queryPort)
            .setPath("/v2.0/" + tenantId + "/views")
            .setParameter("from", String.valueOf(baseMillis))
            .setParameter("to", String.valueOf(baseMillis + 86400000))
            .setParameter("resolution", "full");
    return builder.build();
  }

  private URI getInvalidMetricsQueryURI() throws URISyntaxException {
    URIBuilder builder =
        new URIBuilder()
            .setScheme("http")
            .setHost("127.0.0.1")
            .setPort(queryPort)
            .setPath("/v2.0/" + tenantId + "/views/" + metricName)
            .setParameter("from", String.valueOf(baseMillis))
            .setParameter("resolution", "full"); // Misses parameter 'to'
    return builder.build();
  }

  @AfterClass
  public static void shutdown() {
    vendor.shutdown();
    httpQueryService.stopService();
  }
}