private void logJsonResult(BrokerRequest brokerRequest, IntermediateResultsBlock resultBlock)
      throws Exception {
    final AggregationGroupByOperatorService aggregationGroupByOperatorService =
        new AggregationGroupByOperatorService(_aggregationInfos, brokerRequest.getGroupBy());

    final Map<ServerInstance, DataTable> instanceResponseMap =
        new HashMap<ServerInstance, DataTable>();
    instanceResponseMap.put(
        new ServerInstance("localhost:0000"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:1111"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:2222"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:3333"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:4444"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:5555"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:6666"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:7777"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:8888"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:9999"), resultBlock.getAggregationGroupByResultDataTable());
    final List<Map<String, Serializable>> reducedResults =
        aggregationGroupByOperatorService.reduceGroupByOperators(instanceResponseMap);

    final List<JSONObject> jsonResult =
        aggregationGroupByOperatorService.renderGroupByOperators(reducedResults);
    LOGGER.info("Result: {}", jsonResult);
  }
  @Test
  public void testInnerSegmentPlanMakerForAggregationGroupByOperatorWithFilter() throws Exception {
    final BrokerRequest brokerRequest = getAggregationGroupByWithFilterBrokerRequest();
    final PlanMaker instancePlanMaker = new InstancePlanMakerImplV1();
    final PlanNode rootPlanNode =
        instancePlanMaker.makeInnerSegmentPlan(_indexSegment, brokerRequest);
    rootPlanNode.showTree("");
    final MAggregationGroupByOperator operator = (MAggregationGroupByOperator) rootPlanNode.run();
    final IntermediateResultsBlock resultBlock = (IntermediateResultsBlock) operator.nextBlock();
    LOGGER.debug("RunningTime : {}", resultBlock.getTimeUsedMs());
    LOGGER.debug("NumDocsScanned : {}", resultBlock.getNumDocsScanned());
    LOGGER.debug("TotalDocs : {}", resultBlock.getTotalRawDocs());
    LOGGER.debug("Result: {}", resultBlock.getAggregationGroupByResultDataTable());
    Assert.assertEquals(resultBlock.getNumDocsScanned(), 582);
    Assert.assertEquals(resultBlock.getTotalRawDocs(), 10001);

    final AggregationGroupByOperatorService aggregationGroupByOperatorService =
        new AggregationGroupByOperatorService(_aggregationInfos, brokerRequest.getGroupBy());

    final Map<ServerInstance, DataTable> instanceResponseMap =
        new HashMap<ServerInstance, DataTable>();
    instanceResponseMap.put(
        new ServerInstance("localhost:0000"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:1111"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:2222"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:3333"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:4444"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:5555"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:6666"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:7777"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:8888"), resultBlock.getAggregationGroupByResultDataTable());
    instanceResponseMap.put(
        new ServerInstance("localhost:9999"), resultBlock.getAggregationGroupByResultDataTable());
    final List<Map<String, Serializable>> reducedResults =
        aggregationGroupByOperatorService.reduceGroupByOperators(instanceResponseMap);

    final List<JSONObject> jsonResult =
        aggregationGroupByOperatorService.renderGroupByOperators(reducedResults);
    LOGGER.debug("Result: {}", jsonResult);
  }
  @Test
  public void testAggregationGroupBysWithDataTableEncodeAndDecode() throws Exception {
    final MAggregationGroupByOperator aggregationGroupByOperator = getmAggregationGroupByOperator();

    final IntermediateResultsBlock block =
        (IntermediateResultsBlock) aggregationGroupByOperator.nextBlock();

    for (int i = 0; i < _numAggregations; ++i) {
      LOGGER.debug("Result 1: {}", block.getAggregationGroupByOperatorResult().get(i));
    }

    ////////////////////////////////////////////////////////////////////////////////
    final MAggregationGroupByOperator aggregationGroupByOperator1 =
        getmAggregationGroupByOperator();

    final IntermediateResultsBlock block1 =
        (IntermediateResultsBlock) aggregationGroupByOperator1.nextBlock();

    for (int i = 0; i < _numAggregations; ++i) {
      LOGGER.debug("Result 2: {}", block1.getAggregationGroupByOperatorResult().get(i));
    }

    CombineService.mergeTwoBlocks(getAggregationGroupByNoFilterBrokerRequest(), block, block1);

    for (int i = 0; i < _numAggregations; ++i) {
      LOGGER.debug("Combined Result: {}", block.getAggregationGroupByOperatorResult().get(i));
    }

    final DataTable dataTable = block.getAggregationGroupByResultDataTable();

    final List<Map<String, Serializable>> results =
        AggregationGroupByOperatorService.transformDataTableToGroupByResult(dataTable);
    for (int i = 0; i < _numAggregations; ++i) {
      LOGGER.debug("Decoded Result: {}", results.get(i));
    }
  }
Example #4
0
  @Test
  public void testInterSegmentAggregationGroupByPlanMakerAndRun() {
    PlanMaker instancePlanMaker = new InstancePlanMakerImplV2();
    BrokerRequest brokerRequest = getAggregationGroupByNoFilterBrokerRequest();
    ExecutorService executorService =
        Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker"));
    Plan globalPlan =
        instancePlanMaker.makeInterSegmentPlan(
            makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000);
    globalPlan.print();
    globalPlan.execute();
    DataTable instanceResponse = globalPlan.getInstanceResponse();

    LOGGER.debug(instanceResponse.toString());
    List<DataTable> instanceResponseList = new ArrayList<DataTable>();
    instanceResponseList.add(instanceResponse);

    List<Map<String, Serializable>> combinedGroupByResult =
        AggregationGroupByOperatorService.transformDataTableToGroupByResult(instanceResponse);

    Map<String, Serializable> singleGroupByResult =
        combinedGroupByResult.get(COUNT_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      if (keyString.equals("0")) {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(400020, ((Number) resultList).longValue());
      } else {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(400000, ((Number) resultList).longValue());
      }
    }

    singleGroupByResult = combinedGroupByResult.get(SUM_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      if (keyString.equals("0")) {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        double expectedSumValue =
            (((Double.parseDouble(keyString) + 200000 + Double.parseDouble(keyString)) * 20001) / 2)
                * 20;
        assertEquals(expectedSumValue, ((Double) resultList).doubleValue());
      } else {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        double expectedSumValue =
            (((Double.parseDouble(keyString) + 200000) - 10) + Double.parseDouble(keyString))
                * 10000
                * 20;

        assertEquals(expectedSumValue, ((Double) resultList).doubleValue());
      }
    }

    singleGroupByResult = combinedGroupByResult.get(MAX_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      if (keyString.equals("0")) {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(200000 + Double.parseDouble(keyString), ((Double) resultList).doubleValue());
      } else {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(
            (200000 - 10) + Double.parseDouble(keyString), ((Double) resultList).doubleValue());
      }
    }

    singleGroupByResult = combinedGroupByResult.get(MIN_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      if (keyString.equals("0")) {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(Double.parseDouble(keyString), ((Double) resultList).doubleValue());
      } else {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(Double.parseDouble(keyString), ((Double) resultList).doubleValue());
      }
    }

    singleGroupByResult = combinedGroupByResult.get(AVG_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      if (keyString.equals("0")) {
        Serializable resultList = singleGroupByResult.get(keyString);

        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        double expectedAvgValue =
            ((((Double.parseDouble(keyString) + 200000 + Double.parseDouble(keyString)) * 20001)
                        / 2)
                    * 20)
                / 400020;
        assertEquals(expectedAvgValue, Double.parseDouble((resultList.toString())));
      } else {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        double expectedAvgValue =
            ((((Double.parseDouble(keyString) + 200000) - 10) + Double.parseDouble(keyString))
                    * 10000
                    * 20)
                / 400000;
        assertEquals(expectedAvgValue, Double.parseDouble((resultList.toString())));
      }
    }

    singleGroupByResult = combinedGroupByResult.get(DISTINCT_DIM0_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      Serializable resultList = singleGroupByResult.get(keyString);
      LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
      int expectedAvgValue = 1;
      assertEquals(expectedAvgValue, ((IntOpenHashSet) resultList).size());
    }

    singleGroupByResult = combinedGroupByResult.get(DISTINCT_DIM1_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      Serializable resultList = singleGroupByResult.get(keyString);
      LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
      int expectedAvgValue = 10;
      assertEquals(expectedAvgValue, ((IntOpenHashSet) resultList).size());
    }

    DefaultReduceService defaultReduceService = new DefaultReduceService();
    Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
    instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
    BrokerResponseJSON brokerResponse =
        defaultReduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
    LOGGER.debug(new JSONArray(brokerResponse.getAggregationResults()).toString());
    LOGGER.debug("Time used: {}", brokerResponse.getTimeUsedMs());
  }
  @Test
  public void testAggregationGroupBysWithDataTableEncodeAndDecode() throws Exception {
    final List<AggregationFunctionGroupByOperator> aggregationFunctionGroupByOperatorList =
        new ArrayList<AggregationFunctionGroupByOperator>();
    final BReusableFilteredDocIdSetOperator docIdSetOperator =
        new BReusableFilteredDocIdSetOperator(null, _indexSegment.getTotalDocs(), 5000);
    final Map<String, DataSource> dataSourceMap = getDataSourceMap();
    final MProjectionOperator projectionOperator =
        new MProjectionOperator(dataSourceMap, docIdSetOperator);

    for (int i = 0; i < _numAggregations; ++i) {
      final MDefaultAggregationFunctionGroupByOperator aggregationFunctionGroupByOperator =
          new MDefaultAggregationFunctionGroupByOperator(
              _aggregationInfos.get(i),
              _groupBy,
              new UReplicatedProjectionOperator(projectionOperator),
              true);
      aggregationFunctionGroupByOperatorList.add(aggregationFunctionGroupByOperator);
    }
    final MAggregationGroupByOperator aggregationGroupByOperator =
        new MAggregationGroupByOperator(
            _indexSegment,
            _aggregationInfos,
            _groupBy,
            projectionOperator,
            aggregationFunctionGroupByOperatorList);

    final IntermediateResultsBlock block =
        (IntermediateResultsBlock) aggregationGroupByOperator.nextBlock();

    for (int i = 0; i < _numAggregations; ++i) {
      LOGGER.info("Result 1: {}", block.getAggregationGroupByOperatorResult().get(i));
    }
    /////////////////////////////////////////////////////////////////////////
    final List<AggregationFunctionGroupByOperator> aggregationFunctionGroupByOperatorList1 =
        new ArrayList<AggregationFunctionGroupByOperator>();
    final BReusableFilteredDocIdSetOperator docIdSetOperator1 =
        new BReusableFilteredDocIdSetOperator(null, _indexSegment.getTotalDocs(), 5000);
    final Map<String, DataSource> dataSourceMap1 = getDataSourceMap();
    final MProjectionOperator projectionOperator1 =
        new MProjectionOperator(dataSourceMap1, docIdSetOperator1);

    for (int i = 0; i < _numAggregations; ++i) {
      final MDefaultAggregationFunctionGroupByOperator aggregationFunctionGroupByOperator1 =
          new MDefaultAggregationFunctionGroupByOperator(
              _aggregationInfos.get(i),
              _groupBy,
              new UReplicatedProjectionOperator(projectionOperator1),
              true);
      aggregationFunctionGroupByOperatorList1.add(aggregationFunctionGroupByOperator1);
    }
    final MAggregationGroupByOperator aggregationGroupByOperator1 =
        new MAggregationGroupByOperator(
            _indexSegment,
            _aggregationInfos,
            _groupBy,
            projectionOperator1,
            aggregationFunctionGroupByOperatorList1);

    final IntermediateResultsBlock block1 =
        (IntermediateResultsBlock) aggregationGroupByOperator1.nextBlock();

    for (int i = 0; i < _numAggregations; ++i) {
      LOGGER.info("Result 2: {}", block1.getAggregationGroupByOperatorResult().get(i));
    }
    CombineService.mergeTwoBlocks(getAggregationGroupByNoFilterBrokerRequest(), block, block1);

    for (int i = 0; i < _numAggregations; ++i) {
      LOGGER.info("Combined Result: {}", block.getAggregationGroupByOperatorResult().get(i));
    }

    final DataTable dataTable = block.getAggregationGroupByResultDataTable();

    final List<Map<String, Serializable>> results =
        AggregationGroupByOperatorService.transformDataTableToGroupByResult(dataTable);
    for (int i = 0; i < _numAggregations; ++i) {
      LOGGER.info("Decode AggregationResult from DataTable: {}", results.get(i));
    }
  }