private void logJsonResult(BrokerRequest brokerRequest, IntermediateResultsBlock resultBlock)
throws Exception {
final AggregationGroupByOperatorService aggregationGroupByOperatorService =
new AggregationGroupByOperatorService(_aggregationInfos, brokerRequest.getGroupBy());
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
instanceResponseMap.put(
new ServerInstance("localhost:0000"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:1111"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:2222"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:3333"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:4444"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:5555"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:6666"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:7777"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:8888"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:9999"), resultBlock.getAggregationGroupByResultDataTable());
final List<Map<String, Serializable>> reducedResults =
aggregationGroupByOperatorService.reduceGroupByOperators(instanceResponseMap);
final List<JSONObject> jsonResult =
aggregationGroupByOperatorService.renderGroupByOperators(reducedResults);
LOGGER.info("Result: {}", jsonResult);
}
@Test
public void testInnerSegmentPlanMakerForAggregationGroupByOperatorWithFilter() throws Exception {
final BrokerRequest brokerRequest = getAggregationGroupByWithFilterBrokerRequest();
final PlanMaker instancePlanMaker = new InstancePlanMakerImplV1();
final PlanNode rootPlanNode =
instancePlanMaker.makeInnerSegmentPlan(_indexSegment, brokerRequest);
rootPlanNode.showTree("");
final MAggregationGroupByOperator operator = (MAggregationGroupByOperator) rootPlanNode.run();
final IntermediateResultsBlock resultBlock = (IntermediateResultsBlock) operator.nextBlock();
LOGGER.debug("RunningTime : {}", resultBlock.getTimeUsedMs());
LOGGER.debug("NumDocsScanned : {}", resultBlock.getNumDocsScanned());
LOGGER.debug("TotalDocs : {}", resultBlock.getTotalRawDocs());
LOGGER.debug("Result: {}", resultBlock.getAggregationGroupByResultDataTable());
Assert.assertEquals(resultBlock.getNumDocsScanned(), 582);
Assert.assertEquals(resultBlock.getTotalRawDocs(), 10001);
final AggregationGroupByOperatorService aggregationGroupByOperatorService =
new AggregationGroupByOperatorService(_aggregationInfos, brokerRequest.getGroupBy());
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
instanceResponseMap.put(
new ServerInstance("localhost:0000"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:1111"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:2222"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:3333"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:4444"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:5555"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:6666"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:7777"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:8888"), resultBlock.getAggregationGroupByResultDataTable());
instanceResponseMap.put(
new ServerInstance("localhost:9999"), resultBlock.getAggregationGroupByResultDataTable());
final List<Map<String, Serializable>> reducedResults =
aggregationGroupByOperatorService.reduceGroupByOperators(instanceResponseMap);
final List<JSONObject> jsonResult =
aggregationGroupByOperatorService.renderGroupByOperators(reducedResults);
LOGGER.debug("Result: {}", jsonResult);
}
@Test
public void testAggregationGroupBysWithDataTableEncodeAndDecode() throws Exception {
final MAggregationGroupByOperator aggregationGroupByOperator = getmAggregationGroupByOperator();
final IntermediateResultsBlock block =
(IntermediateResultsBlock) aggregationGroupByOperator.nextBlock();
for (int i = 0; i < _numAggregations; ++i) {
LOGGER.debug("Result 1: {}", block.getAggregationGroupByOperatorResult().get(i));
}
////////////////////////////////////////////////////////////////////////////////
final MAggregationGroupByOperator aggregationGroupByOperator1 =
getmAggregationGroupByOperator();
final IntermediateResultsBlock block1 =
(IntermediateResultsBlock) aggregationGroupByOperator1.nextBlock();
for (int i = 0; i < _numAggregations; ++i) {
LOGGER.debug("Result 2: {}", block1.getAggregationGroupByOperatorResult().get(i));
}
CombineService.mergeTwoBlocks(getAggregationGroupByNoFilterBrokerRequest(), block, block1);
for (int i = 0; i < _numAggregations; ++i) {
LOGGER.debug("Combined Result: {}", block.getAggregationGroupByOperatorResult().get(i));
}
final DataTable dataTable = block.getAggregationGroupByResultDataTable();
final List<Map<String, Serializable>> results =
AggregationGroupByOperatorService.transformDataTableToGroupByResult(dataTable);
for (int i = 0; i < _numAggregations; ++i) {
LOGGER.debug("Decoded Result: {}", results.get(i));
}
}
@Test
public void testInterSegmentAggregationGroupByPlanMakerAndRun() {
PlanMaker instancePlanMaker = new InstancePlanMakerImplV2();
BrokerRequest brokerRequest = getAggregationGroupByNoFilterBrokerRequest();
ExecutorService executorService =
Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker"));
Plan globalPlan =
instancePlanMaker.makeInterSegmentPlan(
makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000);
globalPlan.print();
globalPlan.execute();
DataTable instanceResponse = globalPlan.getInstanceResponse();
LOGGER.debug(instanceResponse.toString());
List<DataTable> instanceResponseList = new ArrayList<DataTable>();
instanceResponseList.add(instanceResponse);
List<Map<String, Serializable>> combinedGroupByResult =
AggregationGroupByOperatorService.transformDataTableToGroupByResult(instanceResponse);
Map<String, Serializable> singleGroupByResult =
combinedGroupByResult.get(COUNT_AGGREGATION_INDEX);
for (String keyString : singleGroupByResult.keySet()) {
if (keyString.equals("0")) {
Serializable resultList = singleGroupByResult.get(keyString);
LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
assertEquals(400020, ((Number) resultList).longValue());
} else {
Serializable resultList = singleGroupByResult.get(keyString);
LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
assertEquals(400000, ((Number) resultList).longValue());
}
}
singleGroupByResult = combinedGroupByResult.get(SUM_AGGREGATION_INDEX);
for (String keyString : singleGroupByResult.keySet()) {
if (keyString.equals("0")) {
Serializable resultList = singleGroupByResult.get(keyString);
LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
double expectedSumValue =
(((Double.parseDouble(keyString) + 200000 + Double.parseDouble(keyString)) * 20001) / 2)
* 20;
assertEquals(expectedSumValue, ((Double) resultList).doubleValue());
} else {
Serializable resultList = singleGroupByResult.get(keyString);
LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
double expectedSumValue =
(((Double.parseDouble(keyString) + 200000) - 10) + Double.parseDouble(keyString))
* 10000
* 20;
assertEquals(expectedSumValue, ((Double) resultList).doubleValue());
}
}
singleGroupByResult = combinedGroupByResult.get(MAX_AGGREGATION_INDEX);
for (String keyString : singleGroupByResult.keySet()) {
if (keyString.equals("0")) {
Serializable resultList = singleGroupByResult.get(keyString);
LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
assertEquals(200000 + Double.parseDouble(keyString), ((Double) resultList).doubleValue());
} else {
Serializable resultList = singleGroupByResult.get(keyString);
LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
assertEquals(
(200000 - 10) + Double.parseDouble(keyString), ((Double) resultList).doubleValue());
}
}
singleGroupByResult = combinedGroupByResult.get(MIN_AGGREGATION_INDEX);
for (String keyString : singleGroupByResult.keySet()) {
if (keyString.equals("0")) {
Serializable resultList = singleGroupByResult.get(keyString);
LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
assertEquals(Double.parseDouble(keyString), ((Double) resultList).doubleValue());
} else {
Serializable resultList = singleGroupByResult.get(keyString);
LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
assertEquals(Double.parseDouble(keyString), ((Double) resultList).doubleValue());
}
}
singleGroupByResult = combinedGroupByResult.get(AVG_AGGREGATION_INDEX);
for (String keyString : singleGroupByResult.keySet()) {
if (keyString.equals("0")) {
Serializable resultList = singleGroupByResult.get(keyString);
LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
double expectedAvgValue =
((((Double.parseDouble(keyString) + 200000 + Double.parseDouble(keyString)) * 20001)
/ 2)
* 20)
/ 400020;
assertEquals(expectedAvgValue, Double.parseDouble((resultList.toString())));
} else {
Serializable resultList = singleGroupByResult.get(keyString);
LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
double expectedAvgValue =
((((Double.parseDouble(keyString) + 200000) - 10) + Double.parseDouble(keyString))
* 10000
* 20)
/ 400000;
assertEquals(expectedAvgValue, Double.parseDouble((resultList.toString())));
}
}
singleGroupByResult = combinedGroupByResult.get(DISTINCT_DIM0_AGGREGATION_INDEX);
for (String keyString : singleGroupByResult.keySet()) {
Serializable resultList = singleGroupByResult.get(keyString);
LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
int expectedAvgValue = 1;
assertEquals(expectedAvgValue, ((IntOpenHashSet) resultList).size());
}
singleGroupByResult = combinedGroupByResult.get(DISTINCT_DIM1_AGGREGATION_INDEX);
for (String keyString : singleGroupByResult.keySet()) {
Serializable resultList = singleGroupByResult.get(keyString);
LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
int expectedAvgValue = 10;
assertEquals(expectedAvgValue, ((IntOpenHashSet) resultList).size());
}
DefaultReduceService defaultReduceService = new DefaultReduceService();
Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
BrokerResponseJSON brokerResponse =
defaultReduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.debug(new JSONArray(brokerResponse.getAggregationResults()).toString());
LOGGER.debug("Time used: {}", brokerResponse.getTimeUsedMs());
}
@Test
public void testAggregationGroupBysWithDataTableEncodeAndDecode() throws Exception {
final List<AggregationFunctionGroupByOperator> aggregationFunctionGroupByOperatorList =
new ArrayList<AggregationFunctionGroupByOperator>();
final BReusableFilteredDocIdSetOperator docIdSetOperator =
new BReusableFilteredDocIdSetOperator(null, _indexSegment.getTotalDocs(), 5000);
final Map<String, DataSource> dataSourceMap = getDataSourceMap();
final MProjectionOperator projectionOperator =
new MProjectionOperator(dataSourceMap, docIdSetOperator);
for (int i = 0; i < _numAggregations; ++i) {
final MDefaultAggregationFunctionGroupByOperator aggregationFunctionGroupByOperator =
new MDefaultAggregationFunctionGroupByOperator(
_aggregationInfos.get(i),
_groupBy,
new UReplicatedProjectionOperator(projectionOperator),
true);
aggregationFunctionGroupByOperatorList.add(aggregationFunctionGroupByOperator);
}
final MAggregationGroupByOperator aggregationGroupByOperator =
new MAggregationGroupByOperator(
_indexSegment,
_aggregationInfos,
_groupBy,
projectionOperator,
aggregationFunctionGroupByOperatorList);
final IntermediateResultsBlock block =
(IntermediateResultsBlock) aggregationGroupByOperator.nextBlock();
for (int i = 0; i < _numAggregations; ++i) {
LOGGER.info("Result 1: {}", block.getAggregationGroupByOperatorResult().get(i));
}
/////////////////////////////////////////////////////////////////////////
final List<AggregationFunctionGroupByOperator> aggregationFunctionGroupByOperatorList1 =
new ArrayList<AggregationFunctionGroupByOperator>();
final BReusableFilteredDocIdSetOperator docIdSetOperator1 =
new BReusableFilteredDocIdSetOperator(null, _indexSegment.getTotalDocs(), 5000);
final Map<String, DataSource> dataSourceMap1 = getDataSourceMap();
final MProjectionOperator projectionOperator1 =
new MProjectionOperator(dataSourceMap1, docIdSetOperator1);
for (int i = 0; i < _numAggregations; ++i) {
final MDefaultAggregationFunctionGroupByOperator aggregationFunctionGroupByOperator1 =
new MDefaultAggregationFunctionGroupByOperator(
_aggregationInfos.get(i),
_groupBy,
new UReplicatedProjectionOperator(projectionOperator1),
true);
aggregationFunctionGroupByOperatorList1.add(aggregationFunctionGroupByOperator1);
}
final MAggregationGroupByOperator aggregationGroupByOperator1 =
new MAggregationGroupByOperator(
_indexSegment,
_aggregationInfos,
_groupBy,
projectionOperator1,
aggregationFunctionGroupByOperatorList1);
final IntermediateResultsBlock block1 =
(IntermediateResultsBlock) aggregationGroupByOperator1.nextBlock();
for (int i = 0; i < _numAggregations; ++i) {
LOGGER.info("Result 2: {}", block1.getAggregationGroupByOperatorResult().get(i));
}
CombineService.mergeTwoBlocks(getAggregationGroupByNoFilterBrokerRequest(), block, block1);
for (int i = 0; i < _numAggregations; ++i) {
LOGGER.info("Combined Result: {}", block.getAggregationGroupByOperatorResult().get(i));
}
final DataTable dataTable = block.getAggregationGroupByResultDataTable();
final List<Map<String, Serializable>> results =
AggregationGroupByOperatorService.transformDataTableToGroupByResult(dataTable);
for (int i = 0; i < _numAggregations; ++i) {
LOGGER.info("Decode AggregationResult from DataTable: {}", results.get(i));
}
}