示例#1
0
  @Test
  public void testMaxQuery() {

    BrokerRequest brokerRequest = getMaxQuery();

    QuerySource querySource = new QuerySource();
    querySource.setTableName("midas");
    brokerRequest.setQuerySource(querySource);
    InstanceRequest instanceRequest = new InstanceRequest(0, brokerRequest);
    instanceRequest.setSearchSegments(new ArrayList<String>());
    for (IndexSegment segment : _indexSegmentList) {
      instanceRequest.getSearchSegments().add(segment.getSegmentName());
    }
    try {
      DataTable instanceResponse = _queryExecutor.processQuery(instanceRequest);
      LOGGER.info("InstanceResponse is " + instanceResponse.getDouble(0, 0));
      Assert.assertEquals(instanceResponse.getDouble(0, 0), 200000.0);

      LOGGER.info(
          "Time used for instanceResponse is " + instanceResponse.getMetadata().get("timeUsedMs"));
    } catch (Exception e) {
      e.printStackTrace();
      // Should never happen
      Assert.assertEquals(true, false);
    }
  }
 public static List<Map<String, Serializable>> transformDataTableToGroupByResult(
     DataTable dataTable) {
   List<Map<String, Serializable>> aggregationGroupByResults =
       new ArrayList<Map<String, Serializable>>();
   for (int i = 0; i < dataTable.getNumberOfRows(); i++) {
     String key = dataTable.getString(i, 0);
     Map<String, Serializable> hashMap = (Map<String, Serializable>) dataTable.getObject(i, 1);
     aggregationGroupByResults.add(hashMap);
   }
   return aggregationGroupByResults;
 }
 public static Collection<Serializable[]> reduce(
     Map<ServerInstance, DataTable> selectionResults, int maxRowSize) {
   Collection<Serializable[]> rowEventsSet = new ArrayList<Serializable[]>(maxRowSize);
   for (final DataTable dt : selectionResults.values()) {
     for (int rowId = 0; rowId < dt.getNumberOfRows(); ++rowId) {
       final Serializable[] row = extractRowFromDataTable(dt, rowId);
       rowEventsSet.add(row);
       if (rowEventsSet.size() == maxRowSize) {
         return rowEventsSet;
       }
     }
   }
   return rowEventsSet;
 }
 public static Serializable[] extractRowFromDataTable(DataTable dt, int rowId) {
   final Serializable[] row = new Serializable[dt.getDataSchema().size()];
   for (int i = 0; i < dt.getDataSchema().size(); ++i) {
     if (dt.getDataSchema().getColumnType(i).isSingleValue()) {
       switch (dt.getDataSchema().getColumnType(i)) {
         case INT:
           row[i] = dt.getInt(rowId, i);
           break;
         case LONG:
           row[i] = dt.getLong(rowId, i);
           break;
         case DOUBLE:
           row[i] = dt.getDouble(rowId, i);
           break;
         case FLOAT:
           row[i] = dt.getFloat(rowId, i);
           break;
         case STRING:
           row[i] = dt.getString(rowId, i);
           break;
         case SHORT:
           row[i] = dt.getShort(rowId, i);
           break;
         case CHAR:
           row[i] = dt.getChar(rowId, i);
           break;
         case BYTE:
           row[i] = dt.getByte(rowId, i);
           break;
         default:
           row[i] = dt.getObject(rowId, i);
           break;
       }
     } else {
       switch (dt.getDataSchema().getColumnType(i)) {
         case INT_ARRAY:
           row[i] = dt.getIntArray(rowId, i);
           break;
         case LONG_ARRAY:
           row[i] = dt.getLongArray(rowId, i);
           break;
         case DOUBLE_ARRAY:
           row[i] = dt.getDoubleArray(rowId, i);
           break;
         case FLOAT_ARRAY:
           row[i] = dt.getFloatArray(rowId, i);
           break;
         case STRING_ARRAY:
           row[i] = dt.getStringArray(rowId, i);
           break;
         case CHAR_ARRAY:
           row[i] = dt.getCharArray(rowId, i);
           break;
         case BYTE_ARRAY:
           row[i] = dt.getByteArray(rowId, i);
           break;
         default:
           row[i] = dt.getObject(rowId, i);
           break;
       }
     }
   }
   return row;
 }
示例#5
0
  @Test
  public void testInterSegmentAggregationGroupByPlanMakerAndRun() {
    PlanMaker instancePlanMaker = new InstancePlanMakerImplV2();
    BrokerRequest brokerRequest = getAggregationGroupByNoFilterBrokerRequest();
    ExecutorService executorService =
        Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker"));
    Plan globalPlan =
        instancePlanMaker.makeInterSegmentPlan(
            makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000);
    globalPlan.print();
    globalPlan.execute();
    DataTable instanceResponse = globalPlan.getInstanceResponse();

    LOGGER.debug(instanceResponse.toString());
    List<DataTable> instanceResponseList = new ArrayList<DataTable>();
    instanceResponseList.add(instanceResponse);

    List<Map<String, Serializable>> combinedGroupByResult =
        AggregationGroupByOperatorService.transformDataTableToGroupByResult(instanceResponse);

    Map<String, Serializable> singleGroupByResult =
        combinedGroupByResult.get(COUNT_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      if (keyString.equals("0")) {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(400020, ((Number) resultList).longValue());
      } else {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(400000, ((Number) resultList).longValue());
      }
    }

    singleGroupByResult = combinedGroupByResult.get(SUM_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      if (keyString.equals("0")) {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        double expectedSumValue =
            (((Double.parseDouble(keyString) + 200000 + Double.parseDouble(keyString)) * 20001) / 2)
                * 20;
        assertEquals(expectedSumValue, ((Double) resultList).doubleValue());
      } else {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        double expectedSumValue =
            (((Double.parseDouble(keyString) + 200000) - 10) + Double.parseDouble(keyString))
                * 10000
                * 20;

        assertEquals(expectedSumValue, ((Double) resultList).doubleValue());
      }
    }

    singleGroupByResult = combinedGroupByResult.get(MAX_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      if (keyString.equals("0")) {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(200000 + Double.parseDouble(keyString), ((Double) resultList).doubleValue());
      } else {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(
            (200000 - 10) + Double.parseDouble(keyString), ((Double) resultList).doubleValue());
      }
    }

    singleGroupByResult = combinedGroupByResult.get(MIN_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      if (keyString.equals("0")) {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(Double.parseDouble(keyString), ((Double) resultList).doubleValue());
      } else {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        assertEquals(Double.parseDouble(keyString), ((Double) resultList).doubleValue());
      }
    }

    singleGroupByResult = combinedGroupByResult.get(AVG_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      if (keyString.equals("0")) {
        Serializable resultList = singleGroupByResult.get(keyString);

        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        double expectedAvgValue =
            ((((Double.parseDouble(keyString) + 200000 + Double.parseDouble(keyString)) * 20001)
                        / 2)
                    * 20)
                / 400020;
        assertEquals(expectedAvgValue, Double.parseDouble((resultList.toString())));
      } else {
        Serializable resultList = singleGroupByResult.get(keyString);
        LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
        double expectedAvgValue =
            ((((Double.parseDouble(keyString) + 200000) - 10) + Double.parseDouble(keyString))
                    * 10000
                    * 20)
                / 400000;
        assertEquals(expectedAvgValue, Double.parseDouble((resultList.toString())));
      }
    }

    singleGroupByResult = combinedGroupByResult.get(DISTINCT_DIM0_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      Serializable resultList = singleGroupByResult.get(keyString);
      LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
      int expectedAvgValue = 1;
      assertEquals(expectedAvgValue, ((IntOpenHashSet) resultList).size());
    }

    singleGroupByResult = combinedGroupByResult.get(DISTINCT_DIM1_AGGREGATION_INDEX);
    for (String keyString : singleGroupByResult.keySet()) {
      Serializable resultList = singleGroupByResult.get(keyString);
      LOGGER.debug("grouped key: {}, value: {}", keyString, resultList);
      int expectedAvgValue = 10;
      assertEquals(expectedAvgValue, ((IntOpenHashSet) resultList).size());
    }

    DefaultReduceService defaultReduceService = new DefaultReduceService();
    Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
    instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
    BrokerResponseJSON brokerResponse =
        defaultReduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
    LOGGER.debug(new JSONArray(brokerResponse.getAggregationResults()).toString());
    LOGGER.debug("Time used: {}", brokerResponse.getTimeUsedMs());
  }
示例#6
0
 @Test
 public void testInterSegmentAggregationPlanMakerAndRun() {
   PlanMaker instancePlanMaker = new InstancePlanMakerImplV2();
   BrokerRequest brokerRequest = _brokerRequest.deepCopy();
   ExecutorService executorService =
       Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker"));
   Plan globalPlan =
       instancePlanMaker.makeInterSegmentPlan(
           makeSegMgrList(_indexSegmentList), brokerRequest, executorService, 150000);
   globalPlan.print();
   globalPlan.execute();
   DataTable instanceResponse = globalPlan.getInstanceResponse();
   LOGGER.debug(Long.toString(instanceResponse.getLong(0, 0)));
   LOGGER.debug(Double.toString(instanceResponse.getDouble(0, 1)));
   LOGGER.debug(Double.toString(instanceResponse.getDouble(0, 2)));
   LOGGER.debug(Double.toString(instanceResponse.getDouble(0, 3)));
   LOGGER.debug(instanceResponse.getObject(0, 4).toString());
   LOGGER.debug(instanceResponse.getObject(0, 5).toString());
   LOGGER.debug(instanceResponse.getObject(0, 6).toString());
   LOGGER.debug("Query time: {}", instanceResponse.getMetadata().get("timeUsedMs"));
   assertEquals(200001L * _indexSegmentList.size(), instanceResponse.getLong(0, 0));
   assertEquals(20000100000.0 * _indexSegmentList.size(), instanceResponse.getDouble(0, 1));
   assertEquals(200000.0, instanceResponse.getDouble(0, 2));
   assertEquals(0.0, instanceResponse.getDouble(0, 3));
   assertEquals(100000.0, Double.parseDouble(instanceResponse.getObject(0, 4).toString()));
   assertEquals(10, ((IntOpenHashSet) instanceResponse.getObject(0, 5)).size());
   assertEquals(100, ((IntOpenHashSet) instanceResponse.getObject(0, 6)).size());
   DefaultReduceService reduceService = new DefaultReduceService();
   Map<ServerInstance, DataTable> instanceResponseMap = new HashMap<ServerInstance, DataTable>();
   instanceResponseMap.put(new ServerInstance("localhost:1111"), instanceResponse);
   BrokerResponseJSON brokerResponse =
       reduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
   LOGGER.debug(brokerResponse.getAggregationResults().toString());
 }