public AggregationGroupByOperatorPlanNode( IndexSegment indexSegment, BrokerRequest query, AggregationGroupByImplementationType aggregationGroupByImplementationType) { _indexSegment = indexSegment; _brokerRequest = query; _aggregationGroupByImplementationType = aggregationGroupByImplementationType; _projectionPlanNode = new ProjectionPlanNode( _indexSegment, getAggregationGroupByRelatedColumns(), new DocIdSetPlanNode(_indexSegment, _brokerRequest)); for (int i = 0; i < _brokerRequest.getAggregationsInfo().size(); ++i) { AggregationInfo aggregationInfo = _brokerRequest.getAggregationsInfo().get(i); AggregationFunctionUtils.ensureAggregationColumnsAreSingleValued( aggregationInfo, _indexSegment); boolean hasDictionary = AggregationFunctionUtils.isAggregationFunctionWithDictionary( aggregationInfo, _indexSegment); _aggregationFunctionGroupByPlanNodes.add( new AggregationFunctionGroupByPlanNode( aggregationInfo, _brokerRequest.getGroupBy(), _projectionPlanNode, _aggregationGroupByImplementationType, hasDictionary)); } }
@Test public void testMaxQuery() { BrokerRequest brokerRequest = getMaxQuery(); QuerySource querySource = new QuerySource(); querySource.setTableName("midas"); brokerRequest.setQuerySource(querySource); InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest); instanceRequest.setSearchSegments(new ArrayList<String>()); for (IndexSegment segment : _indexSegmentList) { instanceRequest.getSearchSegments().add(segment.getSegmentName()); } try { DataTable instanceResponse = _queryExecutor.processQuery(instanceRequest); LOGGER.info("InstanceResponse is " + instanceResponse.getDouble(0, 0)); Assert.assertEquals(instanceResponse.getDouble(0, 0), 200000.0); LOGGER.info( "Time used for instanceResponse is " + instanceResponse.getMetadata().get("timeUsedMs")); } catch (Exception e) { e.printStackTrace(); // Should never happen Assert.assertEquals(true, false); } }
private BrokerRequest getMinQuery() { BrokerRequest query = new BrokerRequest(); AggregationInfo aggregationInfo = getMinAggregationInfo(); List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>(); aggregationsInfo.add(aggregationInfo); query.setAggregationsInfo(aggregationsInfo); FilterQuery filterQuery = getFilterQuery(); query.setFilterQuery(filterQuery); return query; }
private static void setAggregationsInfo(BrokerRequest brokerRequest) { final List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>(); aggregationsInfo.add(getCountAggregationInfo()); aggregationsInfo.add(getSumAggregationInfo()); aggregationsInfo.add(getMaxAggregationInfo()); aggregationsInfo.add(getMinAggregationInfo()); aggregationsInfo.add(getAvgAggregationInfo()); aggregationsInfo.add(getDistinctCountAggregationInfo("column12")); brokerRequest.setAggregationsInfo(aggregationsInfo); brokerRequest.setGroupBy(getGroupBy()); }
private String[] getAggregationGroupByRelatedColumns() { Set<String> aggregationGroupByRelatedColumns = new HashSet<String>(); for (AggregationInfo aggregationInfo : _brokerRequest.getAggregationsInfo()) { if (aggregationInfo.getAggregationType().equalsIgnoreCase("count")) { continue; } String columns = aggregationInfo.getAggregationParams().get("column").trim(); aggregationGroupByRelatedColumns.addAll(Arrays.asList(columns.split(","))); } aggregationGroupByRelatedColumns.addAll(_brokerRequest.getGroupBy().getColumns()); return aggregationGroupByRelatedColumns.toArray(new String[0]); }
private static BrokerRequest getAggregationNoFilterBrokerRequest() { BrokerRequest brokerRequest = new BrokerRequest(); List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>(); aggregationsInfo.add(getCountAggregationInfo()); aggregationsInfo.add(getSumAggregationInfo()); aggregationsInfo.add(getMaxAggregationInfo()); aggregationsInfo.add(getMinAggregationInfo()); aggregationsInfo.add(getAvgAggregationInfo()); aggregationsInfo.add(getDistinctCountDim0AggregationInfo()); aggregationsInfo.add(getDistinctCountDim1AggregationInfo()); brokerRequest.setAggregationsInfo(aggregationsInfo); return brokerRequest; }
private static BrokerRequest getAggregationGroupByNoFilterBrokerRequest() { final BrokerRequest brokerRequest = new BrokerRequest(); final List<AggregationInfo> aggregationsInfo = new ArrayList<AggregationInfo>(); aggregationsInfo.add(getCountAggregationInfo()); aggregationsInfo.add(getSumAggregationInfo()); aggregationsInfo.add(getMaxAggregationInfo()); aggregationsInfo.add(getMinAggregationInfo()); aggregationsInfo.add(getAvgAggregationInfo()); aggregationsInfo.add(getDistinctCountAggregationInfo("column12")); brokerRequest.setAggregationsInfo(aggregationsInfo); brokerRequest.setGroupBy(getGroupBy()); return brokerRequest; }
@Test public void testInnerSegmentPlanMakerForAggregationGroupByOperatorWithFilter() throws Exception { final BrokerRequest brokerRequest = getAggregationGroupByWithFilterBrokerRequest(); final PlanMaker instancePlanMaker = new InstancePlanMakerImplV1(); final PlanNode rootPlanNode = instancePlanMaker.makeInnerSegmentPlan(_indexSegment, brokerRequest); rootPlanNode.showTree(""); final MAggregationGroupByOperator operator = (MAggregationGroupByOperator) rootPlanNode.run(); final IntermediateResultsBlock resultBlock = (IntermediateResultsBlock) operator.nextBlock(); LOGGER.debug("RunningTime : {}", resultBlock.getTimeUsedMs()); LOGGER.debug("NumDocsScanned : {}", resultBlock.getNumDocsScanned()); LOGGER.debug("TotalDocs : {}", resultBlock.getTotalRawDocs()); LOGGER.debug("Result: {}", resultBlock.getAggregationGroupByResultDataTable()); Assert.assertEquals(resultBlock.getNumDocsScanned(), 582); Assert.assertEquals(resultBlock.getTotalRawDocs(), 10001); final AggregationGroupByOperatorService aggregationGroupByOperatorService = new AggregationGroupByOperatorService(_aggregationInfos, brokerRequest.getGroupBy()); final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); instanceResponseMap.put( new ServerInstance("localhost:0000"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:1111"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:2222"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:3333"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:4444"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:5555"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:6666"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:7777"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:8888"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:9999"), resultBlock.getAggregationGroupByResultDataTable()); final List<Map<String, Serializable>> reducedResults = aggregationGroupByOperatorService.reduceGroupByOperators(instanceResponseMap); final List<JSONObject> jsonResult = aggregationGroupByOperatorService.renderGroupByOperators(reducedResults); LOGGER.debug("Result: {}", jsonResult); }
private static List<Serializable> combineAggregationResults( BrokerRequest brokerRequest, List<Serializable> aggregationResult1, List<Serializable> aggregationResult2) { if (aggregationResult1 == null) { return aggregationResult2; } if (aggregationResult2 == null) { return aggregationResult1; } List<List<Serializable>> aggregationResultsList = new ArrayList<List<Serializable>>(); for (int i = 0; i < brokerRequest.getAggregationsInfoSize(); ++i) { aggregationResultsList.add(new ArrayList<Serializable>()); if (aggregationResult1.get(i) != null) { aggregationResultsList.get(i).add(aggregationResult1.get(i)); } if (aggregationResult2.get(i) != null) { aggregationResultsList.get(i).add(aggregationResult2.get(i)); } } List<Serializable> retAggregationResults = new ArrayList<Serializable>(); List<AggregationFunction> aggregationFunctions = AggregationFunctionFactory.getAggregationFunction(brokerRequest); for (int i = 0; i < aggregationFunctions.size(); ++i) { retAggregationResults.add( (Serializable) aggregationFunctions .get(i) .combine(aggregationResultsList.get(i), CombineLevel.INSTANCE) .get(0)); } return retAggregationResults; }
private void logJsonResult(BrokerRequest brokerRequest, IntermediateResultsBlock resultBlock) throws Exception { final AggregationGroupByOperatorService aggregationGroupByOperatorService = new AggregationGroupByOperatorService(_aggregationInfos, brokerRequest.getGroupBy()); final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); instanceResponseMap.put( new ServerInstance("localhost:0000"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:1111"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:2222"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:3333"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:4444"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:5555"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:6666"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:7777"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:8888"), resultBlock.getAggregationGroupByResultDataTable()); instanceResponseMap.put( new ServerInstance("localhost:9999"), resultBlock.getAggregationGroupByResultDataTable()); final List<Map<String, Serializable>> reducedResults = aggregationGroupByOperatorService.reduceGroupByOperators(instanceResponseMap); final List<JSONObject> jsonResult = aggregationGroupByOperatorService.renderGroupByOperators(reducedResults); LOGGER.info("Result: {}", jsonResult); }
@Override public Operator run() { List<AggregationFunctionGroupByOperator> aggregationFunctionOperatorList = new ArrayList<AggregationFunctionGroupByOperator>(); for (AggregationFunctionGroupByPlanNode aggregationFunctionGroupByPlanNode : _aggregationFunctionGroupByPlanNodes) { aggregationFunctionOperatorList.add( (AggregationFunctionGroupByOperator) aggregationFunctionGroupByPlanNode.run()); } return new MAggregationGroupByOperator( _indexSegment, _brokerRequest.getAggregationsInfo(), _brokerRequest.getGroupBy(), _projectionPlanNode.run(), aggregationFunctionOperatorList); }
@Test public void testInterSegmentAggregationPlanMakerAndRun() { PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); BrokerRequest brokerRequest = _brokerRequest.deepCopy(); ExecutorService executorService = Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker")); Plan globalPlan = instancePlanMaker.makeInterSegmentPlan( makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000); globalPlan.print(); globalPlan.execute(); DataTable instanceResponse = globalPlan.getInstanceResponse(); LOGGER.debug(Long.toString(instanceResponse.getLong(0, 0))); LOGGER.debug(Double.toString(instanceResponse.getDouble(0, 1))); LOGGER.debug(Double.toString(instanceResponse.getDouble(0, 2))); LOGGER.debug(Double.toString(instanceResponse.getDouble(0, 3))); LOGGER.debug(instanceResponse.getObject(0, 4).toString()); LOGGER.debug(instanceResponse.getObject(0, 5).toString()); LOGGER.debug(instanceResponse.getObject(0, 6).toString()); LOGGER.debug("Query time: {}", instanceResponse.getMetadata().get("timeUsedMs")); assertEquals(200001L * _indexSegmentList.size(), instanceResponse.getLong(0, 0)); assertEquals(20000100000.0 * _indexSegmentList.size(), instanceResponse.getDouble(0, 1)); assertEquals(200000.0, instanceResponse.getDouble(0, 2)); assertEquals(0.0, instanceResponse.getDouble(0, 3)); assertEquals(100000.0, Double.parseDouble(instanceResponse.getObject(0, 4).toString())); assertEquals(10, ((IntOpenHashSet) instanceResponse.getObject(0, 5)).size()); assertEquals(100, ((IntOpenHashSet) instanceResponse.getObject(0, 6)).size()); DefaultReduceService reduceService = new DefaultReduceService(); Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); instanceResponseMap.put(new ServerInstance("localhost:1111"), instanceResponse); BrokerResponseJSON brokerResponse = reduceService.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.debug(brokerResponse.getAggregationResults().toString()); }
@Override public void updateBrokerRequest(BrokerRequest brokerRequest) { Selection selection = new Selection(); List<String> modifiableList = new ArrayList<>(1); modifiableList.add("*"); selection.setSelectionColumns(modifiableList); brokerRequest.setSelections(selection); }
@Test public void testInterSegmentAggregationPlanMaker() { PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); BrokerRequest brokerRequest = _brokerRequest.deepCopy(); brokerRequest.setSelections(null); brokerRequest.setSelectionsIsSet(false); ExecutorService executorService = Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker")); Plan globalPlan = instancePlanMaker.makeInterSegmentPlan( makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000); globalPlan.print(); brokerRequest = setFilterQuery(brokerRequest); globalPlan = instancePlanMaker.makeInterSegmentPlan( makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000); globalPlan.print(); }
@Test public void testTrace() throws RecognitionException, Exception { String query = "select count(*) from testTable where column1='186154188'"; LOGGER.info("running : " + query); final Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query)); brokerRequest.setEnableTrace(true); // InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest); instanceRequest.setEnableTrace(true); // TODO: add trace settings consistency instanceRequest.setSearchSegments(new ArrayList<String>()); instanceRequest.getSearchSegments().add(segmentName); final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest); instanceResponseMap.clear(); instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse); final BrokerResponse brokerResponse = REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0)); LOGGER.info("TraceInfo is " + brokerResponse.getTraceInfo()); // }
@Override public void showTree(String prefix) { LOGGER.debug(prefix + "Inner-Segment Plan Node :"); LOGGER.debug(prefix + "Operator: MAggregationGroupByOperator"); LOGGER.debug(prefix + "Argument 0: Projection - "); _projectionPlanNode.showTree(prefix + " "); for (int i = 0; i < _brokerRequest.getAggregationsInfo().size(); ++i) { LOGGER.debug(prefix + "Argument " + (i + 1) + ": AggregationGroupBy - "); _aggregationFunctionGroupByPlanNodes.get(i).showTree(prefix + " "); } }
@Test public void testInnerSegmentPlanMakerForSelectionNoFilterNoOrdering() { BrokerRequest brokerRequest = getSelectionNoFilterBrokerRequest(); brokerRequest.getSelections().setSelectionSortSequence(null); PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); PlanNode rootPlanNode = instancePlanMaker.makeInnerSegmentPlan(_indexSegment, brokerRequest); rootPlanNode.showTree(""); // USelectionOperator operator = (USelectionOperator) rootPlanNode.run(); MSelectionOnlyOperator operator = (MSelectionOnlyOperator) rootPlanNode.run(); IntermediateResultsBlock resultBlock = (IntermediateResultsBlock) operator.nextBlock(); ArrayList<Serializable[]> retList = (ArrayList<Serializable[]>) resultBlock.getSelectionResult(); int i = 0; for (Serializable[] row : retList) { LOGGER.debug(Arrays.toString(row)); assertEquals(row[0], (i % 10)); assertEquals(row[1], i); assertEquals(row[2], i); i++; } }
@Test public void testInnerSegmentPlanMakerForAggregationGroupByWithFilter() { BrokerRequest brokerRequest = getAggregationGroupByWithFilterBrokerRequest(); brokerRequest.getGroupBy().getColumns().clear(); brokerRequest.getGroupBy().getColumns().add("dim0"); brokerRequest.getGroupBy().getColumns().add("dim1"); PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); PlanNode rootPlanNode = instancePlanMaker.makeInnerSegmentPlan(_indexSegment, brokerRequest); rootPlanNode.showTree(""); IntermediateResultsBlock resultBlock = (IntermediateResultsBlock) rootPlanNode.run().nextBlock(); LOGGER.debug("RunningTime: {}", resultBlock.getTimeUsedMs()); LOGGER.debug("NumDocsScanned: {}", resultBlock.getNumDocsScanned()); LOGGER.debug("TotalDocs: {}", resultBlock.getTotalRawDocs()); List<Map<String, Serializable>> combinedGroupByResult = resultBlock.getAggregationGroupByOperatorResult(); for (int i = 0; i < combinedGroupByResult.size(); ++i) { LOGGER.debug("function: {}", brokerRequest.getAggregationsInfo().get(i)); for (String keyString : combinedGroupByResult.get(i).keySet()) { LOGGER.debug( "grouped key: {}, value: {}", keyString, combinedGroupByResult.get(i).get(keyString)); } } }
private static List<Map<String, Serializable>> combineAggregationGroupByResults1( BrokerRequest brokerRequest, List<Map<String, Serializable>> list1, List<Map<String, Serializable>> list2) { if (list1 == null) { return list2; } if (list2 == null) { return list1; } for (int i = 0; i < list1.size(); ++i) { list1.set( i, mergeTwoGroupedResults( brokerRequest.getAggregationsInfo().get(i), list1.get(i), list2.get(i))); } return list1; }
@Test public void testInterSegmentSelectionNoOrderingPlanMaker() throws JSONException { PlanMaker instancePlanMaker = new InstancePlanMakerImplV2(); BrokerRequest brokerRequest = _brokerRequest.deepCopy(); brokerRequest.setAggregationsInfo(null); brokerRequest.setAggregationsInfoIsSet(false); brokerRequest.setSelections(getSelectionQuery()); brokerRequest.getSelections().setSelectionSortSequence(null); brokerRequest.getSelections().setOffset(0); brokerRequest.getSelections().setSize(20); ExecutorService executorService = Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker")); Plan globalPlan = instancePlanMaker.makeInterSegmentPlan( makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000); globalPlan.print(); brokerRequest = setFilterQuery(brokerRequest); globalPlan = instancePlanMaker.makeInterSegmentPlan( makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000); globalPlan.print(); globalPlan.execute(); DataTable instanceResponse = globalPlan.getInstanceResponse(); DefaultReduceService defaultReduceService = new DefaultReduceService(); Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>(); instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse); instanceResponseMap.put(new ServerInstance("localhost:1111"), instanceResponse); BrokerResponseJSON brokerResponse = defaultReduceService.reduceOnDataTable(brokerRequest, instanceResponseMap); LOGGER.debug(brokerResponse.getSelectionResults().toString()); LOGGER.debug("TimeUsedMs: {}", brokerResponse.getTimeUsedMs()); LOGGER.debug(brokerResponse.toString()); JSONArray selectionResultsArray = brokerResponse.getSelectionResults().getJSONArray("results"); for (int j = 0; j < selectionResultsArray.length(); ++j) { assertEquals(selectionResultsArray.getJSONArray(j).getInt(0), 1); } }
private static BrokerRequest getSelectionWithFilterBrokerRequest() { BrokerRequest brokerRequest = new BrokerRequest(); brokerRequest.setSelections(getSelectionQuery()); brokerRequest = setFilterQuery(brokerRequest); return brokerRequest; }
public static void mergeTwoBlocks( BrokerRequest brokerRequest, IntermediateResultsBlock mergedBlock, IntermediateResultsBlock blockToMerge) { // Sanity check if (blockToMerge == null) { return; } if (mergedBlock == null) { mergedBlock = blockToMerge; return; } // Combine NumDocsScanned mergedBlock.setNumDocsScanned( mergedBlock.getNumDocsScanned() + blockToMerge.getNumDocsScanned()); // Combine TotalDocs mergedBlock.setTotalDocs(mergedBlock.getTotalDocs() + blockToMerge.getTotalDocs()); // Debug mode enable : Combine SegmentStatistics and TraceInfo if (brokerRequest.isEnableTrace()) { mergedBlock.getSegmentStatistics().addAll(blockToMerge.getSegmentStatistics()); mergedBlock.getTraceInfo().putAll(blockToMerge.getTraceInfo()); } // Combine Exceptions mergedBlock.setExceptionsList( combineExceptions(mergedBlock.getExceptions(), blockToMerge.getExceptions())); if (brokerRequest.isSetAggregationsInfo()) { if (brokerRequest.isSetGroupBy()) { // Combine AggregationGroupBy mergedBlock.setAggregationGroupByResult1( combineAggregationGroupByResults1( brokerRequest, mergedBlock.getAggregationGroupByOperatorResult(), blockToMerge.getAggregationGroupByOperatorResult())); } else { // Combine Aggregations List<AggregationFunction> aggregationFunctions = AggregationFunctionFactory.getAggregationFunction(brokerRequest); mergedBlock.setAggregationFunctions(aggregationFunctions); mergedBlock.setAggregationResults( combineAggregationResults( brokerRequest, mergedBlock.getAggregationResult(), blockToMerge.getAggregationResult())); } } else { // Combine Selections if (brokerRequest.getSelections().isSetSelectionSortSequence()) { SelectionOperatorService selectionService = new SelectionOperatorService( brokerRequest.getSelections(), mergedBlock.getSelectionDataSchema()); mergedBlock.setSelectionResult( selectionService.merge( mergedBlock.getSelectionResult(), blockToMerge.getSelectionResult())); } else { mergedBlock.setSelectionResult( SelectionOperatorUtils.merge( mergedBlock.getSelectionResult(), blockToMerge.getSelectionResult(), brokerRequest.getSelections().getSize())); } } }