Exemple #1
0
 @Test
 public void testInterSegmentAggregationPlanMakerAndRun() {
   PlanMaker instancePlanMaker = new InstancePlanMakerImplV2();
   BrokerRequest brokerRequest = _brokerRequest.deepCopy();
   ExecutorService executorService =
       Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker"));
   Plan globalPlan =
       instancePlanMaker.makeInterSegmentPlan(
           makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000);
   globalPlan.print();
   globalPlan.execute();
   DataTable instanceResponse = globalPlan.getInstanceResponse();
   LOGGER.debug(Long.toString(instanceResponse.getLong(0, 0)));
   LOGGER.debug(Double.toString(instanceResponse.getDouble(0, 1)));
   LOGGER.debug(Double.toString(instanceResponse.getDouble(0, 2)));
   LOGGER.debug(Double.toString(instanceResponse.getDouble(0, 3)));
   LOGGER.debug(instanceResponse.getObject(0, 4).toString());
   LOGGER.debug(instanceResponse.getObject(0, 5).toString());
   LOGGER.debug(instanceResponse.getObject(0, 6).toString());
   LOGGER.debug("Query time: {}", instanceResponse.getMetadata().get("timeUsedMs"));
   assertEquals(200001L * _indexSegmentList.size(), instanceResponse.getLong(0, 0));
   assertEquals(20000100000.0 * _indexSegmentList.size(), instanceResponse.getDouble(0, 1));
   assertEquals(200000.0, instanceResponse.getDouble(0, 2));
   assertEquals(0.0, instanceResponse.getDouble(0, 3));
   assertEquals(100000.0, Double.parseDouble(instanceResponse.getObject(0, 4).toString()));
   assertEquals(10, ((IntOpenHashSet) instanceResponse.getObject(0, 5)).size());
   assertEquals(100, ((IntOpenHashSet) instanceResponse.getObject(0, 6)).size());
   DefaultReduceService reduceService = new DefaultReduceService();
   Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
   instanceResponseMap.put(new ServerInstance("localhost:1111"), instanceResponse);
   BrokerResponseJSON brokerResponse =
       reduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
   LOGGER.debug(brokerResponse.getAggregationResults().toString());
 }
Exemple #2
0
 @Test
 public void testInterSegmentAggregationPlanMaker() {
   PlanMaker instancePlanMaker = new InstancePlanMakerImplV2();
   BrokerRequest brokerRequest = _brokerRequest.deepCopy();
   brokerRequest.setSelections(null);
   brokerRequest.setSelectionsIsSet(false);
   ExecutorService executorService =
       Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker"));
   Plan globalPlan =
       instancePlanMaker.makeInterSegmentPlan(
           makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000);
   globalPlan.print();
   brokerRequest = setFilterQuery(brokerRequest);
   globalPlan =
       instancePlanMaker.makeInterSegmentPlan(
           makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000);
   globalPlan.print();
 }
Exemple #3
0
  @Test
  public void testInterSegmentSelectionPlanMaker() throws JSONException {
    PlanMaker instancePlanMaker = new InstancePlanMakerImplV2();
    BrokerRequest brokerRequest = _brokerRequest.deepCopy();
    brokerRequest.setAggregationsInfo(null);
    brokerRequest.setAggregationsInfoIsSet(false);
    brokerRequest.setSelections(getSelectionQuery());
    brokerRequest.getSelections().setOffset(0);
    brokerRequest.getSelections().setSize(20);
    ExecutorService executorService =
        Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker"));
    Plan globalPlan =
        instancePlanMaker.makeInterSegmentPlan(
            makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000);
    globalPlan.print();
    brokerRequest = setFilterQuery(brokerRequest);
    globalPlan =
        instancePlanMaker.makeInterSegmentPlan(
            makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000);
    globalPlan.print();
    globalPlan.execute();
    DataTable instanceResponse = globalPlan.getInstanceResponse();

    DefaultReduceService defaultReduceService = new DefaultReduceService();
    Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
    instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
    instanceResponseMap.put(new ServerInstance("localhost:1111"), instanceResponse);

    BrokerResponseJSON brokerResponse =
        defaultReduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
    LOGGER.debug(brokerResponse.getSelectionResults().toString());
    LOGGER.debug("Time used: {}", brokerResponse.getTimeUsedMs());
    LOGGER.debug(brokerResponse.toString());
    JSONArray selectionResultsArray = brokerResponse.getSelectionResults().getJSONArray("results");
    for (int j = 0; j < selectionResultsArray.length(); ++j) {
      LOGGER.debug(selectionResultsArray.getJSONArray(j).toString());
      assertEquals(selectionResultsArray.getJSONArray(j).getInt(0), 1);
      assertEquals(selectionResultsArray.getJSONArray(j).getInt(1), 91);
      assertEquals(selectionResultsArray.getJSONArray(j).getInt(2) % 100, 91);
    }
  }
  private BrokerResponse getBrokerResponse(
      PlanMaker instancePlanMaker, BrokerRequest brokerRequest) {
    final ExecutorService executorService =
        Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker"));
    final Plan globalPlan =
        instancePlanMaker.makeInterSegmentPlan(
            _indexSegmentList, brokerRequest, executorService, 150000);
    globalPlan.print();
    globalPlan.execute();
    final DataTable instanceResponse = globalPlan.getInstanceResponse();
    LOGGER.debug("Instance Response: {}", instanceResponse);

    final DefaultReduceService defaultReduceService = new DefaultReduceService();
    final Map<ServerInstance, DataTable> instanceResponseMap =
        new HashMap<ServerInstance, DataTable>();
    instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
    final BrokerResponse brokerResponse =
        defaultReduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
    LOGGER.debug("Broker Response: {}", new JSONArray(brokerResponse.getAggregationResults()));
    LOGGER.debug("Time used : {}", brokerResponse.getTimeUsedMs());
    return brokerResponse;
  }
Exemple #5
0
  @Test
  public void testInterSegmentAggregationGroupByPlanMakerAndRun() {
    PlanMaker instancePlanMaker = new InstancePlanMakerImplV2();
    BrokerRequest brokerRequest = getAggregationGroupByNoFilterBrokerRequest();
    ExecutorService executorService =
        Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker"));
    Plan globalPlan =
        instancePlanMaker.makeInterSegmentPlan(
            makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000);
    globalPlan.print();
    globalPlan.execute();
    DataTable instanceResponse = globalPlan.getInstanceResponse();

    LOGGER.debug(instanceResponse.toString());
    List<DataTable> instanceResponseList = new ArrayList<DataTable>();
    instanceResponseList.add(instanceResponse);

    List<Map<String, Serializable>> combinedGroupByResult =
        AggregationGroupByOperatorService.transformDataTableToGroupByResult(instanceResponse);

    Map<String, Serializable> singleGroupByResult =
        combinedGroupByResult.get(COUNT_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      if (keyString.equals("0")) {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(400020, ((Number) resultList).longValue());
      } else {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(400000, ((Number) resultList).longValue());
      }
    }

    singleGroupByResult = combinedGroupByResult.get(SUM_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      if (keyString.equals("0")) {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        double expectedSumValue =
            (((Double.parseDouble(keyString) + 200000 + Double.parseDouble(keyString)) * 20001) / 2)
                * 20;
        assertEquals(expectedSumValue, ((Double) resultList).doubleValue());
      } else {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        double expectedSumValue =
            (((Double.parseDouble(keyString) + 200000) - 10) + Double.parseDouble(keyString))
                * 10000
                * 20;

        assertEquals(expectedSumValue, ((Double) resultList).doubleValue());
      }
    }

    singleGroupByResult = combinedGroupByResult.get(MAX_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      if (keyString.equals("0")) {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(200000 + Double.parseDouble(keyString), ((Double) resultList).doubleValue());
      } else {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(
            (200000 - 10) + Double.parseDouble(keyString), ((Double) resultList).doubleValue());
      }
    }

    singleGroupByResult = combinedGroupByResult.get(MIN_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      if (keyString.equals("0")) {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(Double.parseDouble(keyString), ((Double) resultList).doubleValue());
      } else {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(Double.parseDouble(keyString), ((Double) resultList).doubleValue());
      }
    }

    singleGroupByResult = combinedGroupByResult.get(AVG_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      if (keyString.equals("0")) {
        Serializable resultList = singleGroupByResult.get(keyString);

        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        double expectedAvgValue =
            ((((Double.parseDouble(keyString) + 200000 + Double.parseDouble(keyString)) * 20001)
                        / 2)
                    * 20)
                / 400020;
        assertEquals(expectedAvgValue, Double.parseDouble((resultList.toString())));
      } else {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        double expectedAvgValue =
            ((((Double.parseDouble(keyString) + 200000) - 10) + Double.parseDouble(keyString))
                    * 10000
                    * 20)
                / 400000;
        assertEquals(expectedAvgValue, Double.parseDouble((resultList.toString())));
      }
    }

    singleGroupByResult = combinedGroupByResult.get(DISTINCT_DIM0_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      Serializable resultList = singleGroupByResult.get(keyString);
      LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
      int expectedAvgValue = 1;
      assertEquals(expectedAvgValue, ((IntOpenHashSet) resultList).size());
    }

    singleGroupByResult = combinedGroupByResult.get(DISTINCT_DIM1_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      Serializable resultList = singleGroupByResult.get(keyString);
      LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
      int expectedAvgValue = 10;
      assertEquals(expectedAvgValue, ((IntOpenHashSet) resultList).size());
    }

    DefaultReduceService defaultReduceService = new DefaultReduceService();
    Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
    instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
    BrokerResponseJSON brokerResponse =
        defaultReduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
    LOGGER.debug(new JSONArray(brokerResponse.getAggregationResults()).toString());
    LOGGER.debug("Time used: {}", brokerResponse.getTimeUsedMs());
  }