private void logJsonResult(BrokerRequest brokerRequest, IntermediateResultsBlock resultBlock) throws Exception { final AggregationGroupByOperatorService aggregationGroupByOperatorService = new AggregationGroupByOperatorService(_aggregationInfos, brokerRequest.getGroupBy()); final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); instanceResponseMap.put( new ServerInstance("localhost:0000"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:1111"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:2222"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:3333"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:4444"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:5555"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:6666"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:7777"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:8888"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:9999"), resultBlock.getAggregationGroupByResultDataTable()); final List<Map<String, Serializable>> reducedResults = aggregationGroupByOperatorService.reduceGroupByOperators(instanceResponseMap); final List<JSONObject> jsonResult = aggregationGroupByOperatorService.renderGroupByOperators(reducedResults); LOGGER.info("Result: {}", jsonResult); }
@Test public void testInnerSegmentPlanMakerForAggregationGroupByOperatorWithFilter() throws Exception { final BrokerRequest brokerRequest = getAggregationGroupByWithFilterBrokerRequest(); final PlanMaker instancePlanMaker = new InstancePlanMakerImplV1(); final PlanNode rootPlanNode = instancePlanMaker.makeInnerSegmentPlan(_indexSegment, brokerRequest); rootPlanNode.showTree(""); final MAggregationGroupByOperator operator = (MAggregationGroupByOperator) rootPlanNode.run(); final IntermediateResultsBlock resultBlock = (IntermediateResultsBlock) operator.nextBlock(); LOGGER.debug("RunningTime : {}", resultBlock.getTimeUsedMs()); LOGGER.debug("NumDocsScanned : {}", resultBlock.getNumDocsScanned()); LOGGER.debug("TotalDocs : {}", resultBlock.getTotalRawDocs()); LOGGER.debug("Result: {}", resultBlock.getAggregationGroupByResultDataTable()); Assert.assertEquals(resultBlock.getNumDocsScanned(), 582); Assert.assertEquals(resultBlock.getTotalRawDocs(), 10001); final AggregationGroupByOperatorService aggregationGroupByOperatorService = new AggregationGroupByOperatorService(_aggregationInfos, brokerRequest.getGroupBy()); final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); instanceResponseMap.put( new ServerInstance("localhost:0000"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:1111"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:2222"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:3333"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:4444"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:5555"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:6666"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:7777"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:8888"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:9999"), resultBlock.getAggregationGroupByResultDataTable()); final List<Map<String, Serializable>> reducedResults = aggregationGroupByOperatorService.reduceGroupByOperators(instanceResponseMap); final List<JSONObject> jsonResult = aggregationGroupByOperatorService.renderGroupByOperators(reducedResults); LOGGER.debug("Result: {}", jsonResult); }
@Test public void testAggregationGroupBysWithDataTableEncodeAndDecode() throws Exception { final MAggregationGroupByOperator aggregationGroupByOperator = getmAggregationGroupByOperator(); final IntermediateResultsBlock block = (IntermediateResultsBlock) aggregationGroupByOperator.nextBlock(); for (int i = 0; i < _numAggregations; ++i) { LOGGER.debug("Result 1: {}", block.getAggregationGroupByOperatorResult().get(i)); } //////////////////////////////////////////////////////////////////////////////// final MAggregationGroupByOperator aggregationGroupByOperator1 = getmAggregationGroupByOperator(); final IntermediateResultsBlock block1 = (IntermediateResultsBlock) aggregationGroupByOperator1.nextBlock(); for (int i = 0; i < _numAggregations; ++i) { LOGGER.debug("Result 2: {}", block1.getAggregationGroupByOperatorResult().get(i)); } CombineService.mergeTwoBlocks(getAggregationGroupByNoFilterBrokerRequest(), block, block1); for (int i = 0; i < _numAggregations; ++i) { LOGGER.debug("Combined Result: {}", block.getAggregationGroupByOperatorResult().get(i)); } final DataTable dataTable = block.getAggregationGroupByResultDataTable(); final List<Map<String, Serializable>> results = AggregationGroupByOperatorService.transformDataTableToGroupByResult(dataTable); for (int i = 0; i < _numAggregations; ++i) { LOGGER.debug("Decoded Result: {}", results.get(i)); } }
@Test public void testInterSegmentAggregationGroupByPlanMakerAndRun() { PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); BrokerRequest brokerRequest = getAggregationGroupByNoFilterBrokerRequest(); ExecutorService executorService = Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker")); Plan globalPlan = instancePlanMaker.makeInterSegmentPlan( makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000); globalPlan.print(); globalPlan.execute(); DataTable instanceResponse = globalPlan.getInstanceResponse(); LOGGER.debug(instanceResponse.toString()); List<DataTable> instanceResponseList = new ArrayList<DataTable>(); instanceResponseList.add(instanceResponse); List<Map<String, Serializable>> combinedGroupByResult = AggregationGroupByOperatorService.transformDataTableToGroupByResult(instanceResponse); Map<String, Serializable> singleGroupByResult = combinedGroupByResult.get(COUNT_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { if (keyString.equals("0")) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(400020, ((Number) resultList).longValue()); } else { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(400000, ((Number) resultList).longValue()); } } singleGroupByResult = combinedGroupByResult.get(SUM_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { if (keyString.equals("0")) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); double expectedSumValue = (((Double.parseDouble(keyString) + 200000 + Double.parseDouble(keyString)) * 20001) / 2) * 20; assertEquals(expectedSumValue, ((Double) resultList).doubleValue()); } else { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); double expectedSumValue = (((Double.parseDouble(keyString) + 200000) - 10) + Double.parseDouble(keyString)) * 10000 * 20; assertEquals(expectedSumValue, ((Double) resultList).doubleValue()); } } singleGroupByResult = combinedGroupByResult.get(MAX_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { if (keyString.equals("0")) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(200000 + Double.parseDouble(keyString), ((Double) resultList).doubleValue()); } else { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals( (200000 - 10) + Double.parseDouble(keyString), ((Double) resultList).doubleValue()); } } singleGroupByResult = combinedGroupByResult.get(MIN_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { if (keyString.equals("0")) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(Double.parseDouble(keyString), ((Double) resultList).doubleValue()); } else { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); assertEquals(Double.parseDouble(keyString), ((Double) resultList).doubleValue()); } } singleGroupByResult = combinedGroupByResult.get(AVG_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { if (keyString.equals("0")) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); double expectedAvgValue = ((((Double.parseDouble(keyString) + 200000 + Double.parseDouble(keyString)) * 20001) / 2) * 20) / 400020; assertEquals(expectedAvgValue, Double.parseDouble((resultList.toString()))); } else { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); double expectedAvgValue = ((((Double.parseDouble(keyString) + 200000) - 10) + Double.parseDouble(keyString)) * 10000 * 20) / 400000; assertEquals(expectedAvgValue, Double.parseDouble((resultList.toString()))); } } singleGroupByResult = combinedGroupByResult.get(DISTINCT_DIM0_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); int expectedAvgValue = 1; assertEquals(expectedAvgValue, ((IntOpenHashSet) resultList).size()); } singleGroupByResult = combinedGroupByResult.get(DISTINCT_DIM1_AGGREGATION_INDEX); for (String keyString : singleGroupByResult.keySet()) { Serializable resultList = singleGroupByResult.get(keyString); LOGGER.debug("grouped key: {}, value: {}", keyString, resultList); int expectedAvgValue = 10; assertEquals(expectedAvgValue, ((IntOpenHashSet) resultList).size()); } DefaultReduceService defaultReduceService = new DefaultReduceService(); Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse); BrokerResponseJSON brokerResponse = defaultReduceService.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.debug(new JSONArray(brokerResponse.getAggregationResults()).toString()); LOGGER.debug("Time used: {}", brokerResponse.getTimeUsedMs()); }
@Test public void testAggregationGroupBysWithDataTableEncodeAndDecode() throws Exception { final List<AggregationFunctionGroupByOperator> aggregationFunctionGroupByOperatorList = new ArrayList<AggregationFunctionGroupByOperator>(); final BReusableFilteredDocIdSetOperator docIdSetOperator = new BReusableFilteredDocIdSetOperator(null, _indexSegment.getTotalDocs(), 5000); final Map<String, DataSource> dataSourceMap = getDataSourceMap(); final MProjectionOperator projectionOperator = new MProjectionOperator(dataSourceMap, docIdSetOperator); for (int i = 0; i < _numAggregations; ++i) { final MDefaultAggregationFunctionGroupByOperator aggregationFunctionGroupByOperator = new MDefaultAggregationFunctionGroupByOperator( _aggregationInfos.get(i), _groupBy, new UReplicatedProjectionOperator(projectionOperator), true); aggregationFunctionGroupByOperatorList.add(aggregationFunctionGroupByOperator); } final MAggregationGroupByOperator aggregationGroupByOperator = new MAggregationGroupByOperator( _indexSegment, _aggregationInfos, _groupBy, projectionOperator, aggregationFunctionGroupByOperatorList); final IntermediateResultsBlock block = (IntermediateResultsBlock) aggregationGroupByOperator.nextBlock(); for (int i = 0; i < _numAggregations; ++i) { LOGGER.info("Result 1: {}", block.getAggregationGroupByOperatorResult().get(i)); } ///////////////////////////////////////////////////////////////////////// final List<AggregationFunctionGroupByOperator> aggregationFunctionGroupByOperatorList1 = new ArrayList<AggregationFunctionGroupByOperator>(); final BReusableFilteredDocIdSetOperator docIdSetOperator1 = new BReusableFilteredDocIdSetOperator(null, _indexSegment.getTotalDocs(), 5000); final Map<String, DataSource> dataSourceMap1 = getDataSourceMap(); final MProjectionOperator projectionOperator1 = new MProjectionOperator(dataSourceMap1, docIdSetOperator1); for (int i = 0; i < _numAggregations; ++i) { final MDefaultAggregationFunctionGroupByOperator aggregationFunctionGroupByOperator1 = new MDefaultAggregationFunctionGroupByOperator( _aggregationInfos.get(i), _groupBy, new UReplicatedProjectionOperator(projectionOperator1), true); aggregationFunctionGroupByOperatorList1.add(aggregationFunctionGroupByOperator1); } final MAggregationGroupByOperator aggregationGroupByOperator1 = new MAggregationGroupByOperator( _indexSegment, _aggregationInfos, _groupBy, projectionOperator1, aggregationFunctionGroupByOperatorList1); final IntermediateResultsBlock block1 = (IntermediateResultsBlock) aggregationGroupByOperator1.nextBlock(); for (int i = 0; i < _numAggregations; ++i) { LOGGER.info("Result 2: {}", block1.getAggregationGroupByOperatorResult().get(i)); } CombineService.mergeTwoBlocks(getAggregationGroupByNoFilterBrokerRequest(), block, block1); for (int i = 0; i < _numAggregations; ++i) { LOGGER.info("Combined Result: {}", block.getAggregationGroupByOperatorResult().get(i)); } final DataTable dataTable = block.getAggregationGroupByResultDataTable(); final List<Map<String, Serializable>> results = AggregationGroupByOperatorService.transformDataTableToGroupByResult(dataTable); for (int i = 0; i < _numAggregations; ++i) { LOGGER.info("Decode AggregationResult from DataTable: {}", results.get(i)); } }