private void assertBrokerResponse(BrokerResponse brokerResponse, int groupSize)
      throws JSONException {
    Assert.assertEquals(_numAggregations, brokerResponse.getAggregationResults().size());
    for (int i = 0; i < _numAggregations; ++i) {
      Assert.assertEquals(
          "[\"column11\",\"column10\"]",
          brokerResponse.getAggregationResults().get(i).getJSONArray("groupByColumns").toString());
      Assert.assertEquals(
          groupSize,
          brokerResponse.getAggregationResults().get(i).getJSONArray("groupByResult").length());
    }

    // Assertion on Count
    Assert.assertEquals(
        "count_star",
        brokerResponse.getAggregationResults().get(0).getString("function").toString());
    Assert.assertEquals(
        "sum_met_impressionCount",
        brokerResponse.getAggregationResults().get(1).getString("function").toString());
    Assert.assertEquals(
        "max_met_impressionCount",
        brokerResponse.getAggregationResults().get(2).getString("function").toString());
    Assert.assertEquals(
        "min_met_impressionCount",
        brokerResponse.getAggregationResults().get(3).getString("function").toString());
    Assert.assertEquals(
        "avg_met_impressionCount",
        brokerResponse.getAggregationResults().get(4).getString("function").toString());
    Assert.assertEquals(
        "distinctCount_column12",
        brokerResponse.getAggregationResults().get(5).getString("function").toString());
  }
  private void assertEmptyBrokerResponse(BrokerResponse brokerResponse) throws JSONException {
    Assert.assertEquals(0, brokerResponse.getNumDocsScanned());
    Assert.assertEquals(_numAggregations, brokerResponse.getAggregationResults().size());
    for (int i = 0; i < _numAggregations; ++i) {
      Assert.assertEquals(
          "[\"column11\",\"column10\"]",
          brokerResponse.getAggregationResults().get(i).getJSONArray("groupByColumns").toString());
      Assert.assertEquals(
          0, brokerResponse.getAggregationResults().get(i).getJSONArray("groupByResult").length());
    }

    // Assertion on Count
    assertionOnCount(brokerResponse);
  }
  private void runApproximationQueries(
      List<? extends AvroQueryGenerator.TestAggreationQuery> queries, double precision)
      throws Exception {
    boolean isAccurate = true;
    Object accurateValue = null;

    int counter = 0;
    final Map<ServerInstance, DataTable> instanceResponseMap =
        new HashMap<ServerInstance, DataTable>();
    for (final AvroQueryGenerator.TestAggreationQuery query : queries) {
      LOGGER.info("**************************");
      LOGGER.info("running " + counter + " : " + query.getPql());
      final BrokerRequest brokerRequest =
          RequestConverter.fromJSON(REQUEST_COMPILER.compile(query.getPql()));
      InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest);
      instanceRequest.setSearchSegments(new ArrayList<String>());
      instanceRequest.getSearchSegments().add(segmentName);
      final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
      instanceResponseMap.clear();
      instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
      final BrokerResponse brokerResponse =
          REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
      LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));

      // compute value
      Object val;
      if (query instanceof AvroQueryGenerator.TestSimpleAggreationQuery) {
        val = Double.parseDouble(brokerResponse.getAggregationResults().get(0).getString("value"));
      } else {
        val = brokerResponse.getAggregationResults().get(0).getJSONArray("groupByResult");
      }

      if (isAccurate) {
        // store accurate value
        accurateValue = val;
        isAccurate = false;
      } else {
        // compare value with accurate value
        // it's estimation so we need to test its result within error bound
        if (query instanceof AvroQueryGenerator.TestSimpleAggreationQuery) {
          TestUtils.assertApproximation((Double) val, (Double) accurateValue, precision);
        } else {
          TestUtils.assertJSONArrayApproximation(
              (JSONArray) val, (JSONArray) accurateValue, precision);
        }
        isAccurate = true;
      }
    }
  }
  private void assertBrokerResponse(int numSegments, BrokerResponse brokerResponse)
      throws JSONException {
    Assert.assertEquals(10001 * numSegments, brokerResponse.getNumDocsScanned());
    Assert.assertEquals(_numAggregations, brokerResponse.getAggregationResults().size());
    for (int i = 0; i < _numAggregations; ++i) {
      Assert.assertEquals(
          "[\"column11\",\"column10\"]",
          brokerResponse.getAggregationResults().get(i).getJSONArray("groupByColumns").toString());
      Assert.assertEquals(
          15, brokerResponse.getAggregationResults().get(i).getJSONArray("groupByResult").length());
    }

    // Assertion on Count
    assertionOnCount(brokerResponse);

    // Assertion on Aggregation Results
    LOGGER.info("brokerResponse = {}", brokerResponse);
    final List<double[]> aggregationResult = getAggregationResult(numSegments);
    final List<String[]> groupByResult = getGroupResult();
    for (int j = 0; j < _numAggregations; ++j) {
      final double[] aggResult = aggregationResult.get(j);
      final String[] groupResult = groupByResult.get(j);
      for (int i = 0; i < 15; ++i) {
        Assert.assertEquals(
            0,
            DoubleComparisonUtil.defaultDoubleCompare(
                aggResult[i],
                brokerResponse
                    .getAggregationResults()
                    .get(j)
                    .getJSONArray("groupByResult")
                    .getJSONObject(i)
                    .getDouble("value")));
        if ((i < 14 && aggResult[i] == aggResult[i + 1])
            || (i > 0 && aggResult[i] == aggResult[i - 1])) {
          // do nothing, as we have multiple groups within same value.
        } else {
          Assert.assertEquals(
              groupResult[i],
              brokerResponse
                  .getAggregationResults()
                  .get(j)
                  .getJSONArray("groupByResult")
                  .getJSONObject(i)
                  .getString("group"));
        }
      }
    }
  }
 @Test
 public void testRangeQuery() throws RecognitionException, Exception {
   String query =
       "select count(*) from testTable where column1 in ('999983251', '510705831', '1000720716', '1001058817', '1001099410')";
   LOGGER.info("running  : " + query);
   final Map<ServerInstance, DataTable> instanceResponseMap =
       new HashMap<ServerInstance, DataTable>();
   final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query));
   InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest);
   instanceRequest.setSearchSegments(new ArrayList<String>());
   instanceRequest.getSearchSegments().add(segmentName);
   final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
   instanceResponseMap.clear();
   instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
   final BrokerResponse brokerResponse =
       REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
   LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
   Assert.assertEquals(brokerResponse.getAggregationResults().get(0).getInt("value"), 14);
   Assert.assertEquals(brokerResponse.getNumDocsScanned(), 14);
 }
  @Test
  public void testAggregationGroupBy() throws Exception {
    final List<TestGroupByAggreationQuery> groupByCalls =
        AVRO_QUERY_GENERATOR.giveMeNGroupByAggregationQueries(10000);
    int counter = 0;
    final Map<ServerInstance, DataTable> instanceResponseMap =
        new HashMap<ServerInstance, DataTable>();
    for (final TestGroupByAggreationQuery groupBy : groupByCalls) {
      LOGGER.info("running " + counter + " : " + groupBy.pql);
      final BrokerRequest brokerRequest =
          RequestConverter.fromJSON(REQUEST_COMPILER.compile(groupBy.pql));
      InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest);
      instanceRequest.setSearchSegments(new ArrayList<String>());
      instanceRequest.getSearchSegments().add(segmentName);
      final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
      instanceResponseMap.clear();
      instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
      final BrokerResponse brokerResponse =
          REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
      LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
      LOGGER.info("Result from avro is : " + groupBy.groupResults);

      try {
        assertGroupByResults(
            brokerResponse.getAggregationResults().get(0).getJSONArray("groupByResult"),
            groupBy.groupResults);
      } catch (AssertionError e) {
        System.out.println(groupBy.pql);
        System.out.println(
            "from broker : "
                + brokerResponse
                    .getAggregationResults()
                    .get(0)
                    .getJSONArray("groupByResult")
                    .toString());
        System.out.println("from precomp : " + groupBy.groupResults);
        throw new AssertionError(e);
      }
    }
  }
  private void assertBrokerResponse(int numSegments, BrokerResponse brokerResponse)
      throws JSONException {
    Assert.assertEquals(10001 * numSegments, brokerResponse.getNumDocsScanned());
    final int groupSize = 15;
    assertBrokerResponse(brokerResponse, groupSize);

    // Assertion on Aggregation Results
    final List<double[]> aggregationResult = getAggregationResult(numSegments);
    final List<String[]> groupByResult = getGroupResult();
    for (int j = 0; j < _numAggregations; ++j) {
      final double[] aggResult = aggregationResult.get(j);
      final String[] groupResult = groupByResult.get(j);
      for (int i = 0; i < 15; ++i) {
        Assert.assertEquals(
            0,
            DoubleComparisonUtil.defaultDoubleCompare(
                aggResult[i],
                brokerResponse
                    .getAggregationResults()
                    .get(j)
                    .getJSONArray("groupByResult")
                    .getJSONObject(i)
                    .getDouble("value")));
        if ((i < 14 && aggResult[i] == aggResult[i + 1])
            || (i > 0 && aggResult[i] == aggResult[i - 1])) {
          // do nothing, as we have multiple groups within same value.
        } else {
          Assert.assertEquals(
              groupResult[i],
              brokerResponse
                  .getAggregationResults()
                  .get(j)
                  .getJSONArray("groupByResult")
                  .getJSONObject(i)
                  .getString("group"));
        }
      }
    }
  }
  @Test
  public void testMatchAllQuery() throws RecognitionException, Exception {
    String query = "select count(*),sum(count) from testTable  ";
    LOGGER.info("running  : " + query);
    final Map<ServerInstance, DataTable> instanceResponseMap =
        new HashMap<ServerInstance, DataTable>();
    final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query));
    InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest);
    instanceRequest.setSearchSegments(new ArrayList<String>());
    instanceRequest.getSearchSegments().add(segmentName);
    final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
    instanceResponseMap.clear();
    instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
    final BrokerResponse brokerResponse =
        REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
    LOGGER.info("BrokerResponse is " + brokerResponse);
    LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
    LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(1));

    Assert.assertEquals(brokerResponse.getAggregationResults().get(0).getInt("value"), 100000);
    Assert.assertEquals(
        brokerResponse.getAggregationResults().get(1).getDouble("value"), 8.90662862E13);
    Assert.assertEquals(brokerResponse.getNumDocsScanned(), 100000);
  }
 @Test
 public void testAggregation() throws Exception {
   int counter = 0;
   final Map<ServerInstance, DataTable> instanceResponseMap =
       new HashMap<ServerInstance, DataTable>();
   final List<TestSimpleAggreationQuery> aggCalls =
       AVRO_QUERY_GENERATOR.giveMeNSimpleAggregationQueries(10000);
   for (final TestSimpleAggreationQuery aggCall : aggCalls) {
     LOGGER.info("running " + counter + " : " + aggCall.pql);
     final BrokerRequest brokerRequest =
         RequestConverter.fromJSON(REQUEST_COMPILER.compile(aggCall.pql));
     InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest);
     instanceRequest.setSearchSegments(new ArrayList<String>());
     instanceRequest.getSearchSegments().add(segmentName);
     final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
     instanceResponseMap.clear();
     instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
     final BrokerResponse brokerResponse =
         REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
     LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
     LOGGER.info("Result from avro is : " + aggCall.result);
     try {
       Assert.assertEquals(
           Double.parseDouble(brokerResponse.getAggregationResults().get(0).getString("value")),
           aggCall.result);
     } catch (AssertionError e) {
       System.out.println(aggCall.pql);
       System.out.println(
           "from broker : "
               + Double.parseDouble(
                   brokerResponse.getAggregationResults().get(0).getString("value")));
       System.out.println("from precomp : " + aggCall.result);
       throw new AssertionError(e);
     }
   }
 }
 @Test
 public void testSingleQuery() throws RecognitionException, Exception {
   String query;
   query = "select count(*) from testTable where column5='kCMyNVGCASKYDdQbftOPaqVMWc'";
   // query= "select sum('count') from testTable where column1='660156454'";
   LOGGER.info("running  : " + query);
   final Map<ServerInstance, DataTable> instanceResponseMap =
       new HashMap<ServerInstance, DataTable>();
   final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query));
   InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest);
   instanceRequest.setSearchSegments(new ArrayList<String>());
   instanceRequest.getSearchSegments().add(segmentName);
   final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
   instanceResponseMap.clear();
   instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
   final BrokerResponse brokerResponse =
       REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
   LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
 }
 @Test
 public void testTrace() throws RecognitionException, Exception {
   String query = "select count(*) from testTable where column1='186154188'";
   LOGGER.info("running  : " + query);
   final Map<ServerInstance, DataTable> instanceResponseMap =
       new HashMap<ServerInstance, DataTable>();
   final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query));
   brokerRequest.setEnableTrace(true); //
   InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest);
   instanceRequest.setEnableTrace(true); // TODO: add trace settings consistency
   instanceRequest.setSearchSegments(new ArrayList<String>());
   instanceRequest.getSearchSegments().add(segmentName);
   final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
   instanceResponseMap.clear();
   instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
   final BrokerResponse brokerResponse =
       REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
   LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
   LOGGER.info("TraceInfo is " + brokerResponse.getTraceInfo()); //
 }
  private BrokerResponse getBrokerResponse(
      PlanMaker instancePlanMaker, BrokerRequest brokerRequest) {
    final ExecutorService executorService =
        Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker"));
    final Plan globalPlan =
        instancePlanMaker.makeInterSegmentPlan(
            _indexSegmentList, brokerRequest, executorService, 150000);
    globalPlan.print();
    globalPlan.execute();
    final DataTable instanceResponse = globalPlan.getInstanceResponse();
    LOGGER.debug("Instance Response: {}", instanceResponse);

    final DefaultReduceService defaultReduceService = new DefaultReduceService();
    final Map<ServerInstance, DataTable> instanceResponseMap =
        new HashMap<ServerInstance, DataTable>();
    instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
    final BrokerResponse brokerResponse =
        defaultReduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
    LOGGER.debug("Broker Response: {}", new JSONArray(brokerResponse.getAggregationResults()));
    LOGGER.debug("Time used : {}", brokerResponse.getTimeUsedMs());
    return brokerResponse;
  }
 private void assertionOnCount(BrokerResponse brokerResponse) throws JSONException {
   Assert.assertEquals(
       "count_star",
       brokerResponse.getAggregationResults().get(0).getString("function").toString());
   Assert.assertEquals(
       "sum_met_impressionCount",
       brokerResponse.getAggregationResults().get(1).getString("function").toString());
   Assert.assertEquals(
       "max_met_impressionCount",
       brokerResponse.getAggregationResults().get(2).getString("function").toString());
   Assert.assertEquals(
       "min_met_impressionCount",
       brokerResponse.getAggregationResults().get(3).getString("function").toString());
   Assert.assertEquals(
       "avg_met_impressionCount",
       brokerResponse.getAggregationResults().get(4).getString("function").toString());
   Assert.assertEquals(
       "distinctCount_column12",
       brokerResponse.getAggregationResults().get(5).getString("function").toString());
 }