private void assertBrokerResponse(BrokerResponse brokerResponse, int groupSize)
throws JSONException {
Assert.assertEquals(_numAggregations, brokerResponse.getAggregationResults().size());
for (int i = 0; i < _numAggregations; ++i) {
Assert.assertEquals(
"[\"column11\",\"column10\"]",
brokerResponse.getAggregationResults().get(i).getJSONArray("groupByColumns").toString());
Assert.assertEquals(
groupSize,
brokerResponse.getAggregationResults().get(i).getJSONArray("groupByResult").length());
}
// Assertion on Count
Assert.assertEquals(
"count_star",
brokerResponse.getAggregationResults().get(0).getString("function").toString());
Assert.assertEquals(
"sum_met_impressionCount",
brokerResponse.getAggregationResults().get(1).getString("function").toString());
Assert.assertEquals(
"max_met_impressionCount",
brokerResponse.getAggregationResults().get(2).getString("function").toString());
Assert.assertEquals(
"min_met_impressionCount",
brokerResponse.getAggregationResults().get(3).getString("function").toString());
Assert.assertEquals(
"avg_met_impressionCount",
brokerResponse.getAggregationResults().get(4).getString("function").toString());
Assert.assertEquals(
"distinctCount_column12",
brokerResponse.getAggregationResults().get(5).getString("function").toString());
}
private void assertEmptyBrokerResponse(BrokerResponse brokerResponse) throws JSONException {
Assert.assertEquals(0, brokerResponse.getNumDocsScanned());
Assert.assertEquals(_numAggregations, brokerResponse.getAggregationResults().size());
for (int i = 0; i < _numAggregations; ++i) {
Assert.assertEquals(
"[\"column11\",\"column10\"]",
brokerResponse.getAggregationResults().get(i).getJSONArray("groupByColumns").toString());
Assert.assertEquals(
0, brokerResponse.getAggregationResults().get(i).getJSONArray("groupByResult").length());
}
// Assertion on Count
assertionOnCount(brokerResponse);
}
private void runApproximationQueries(
List<? extends AvroQueryGenerator.TestAggreationQuery> queries, double precision)
throws Exception {
boolean isAccurate = true;
Object accurateValue = null;
int counter = 0;
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
for (final AvroQueryGenerator.TestAggreationQuery query : queries) {
LOGGER.info("**************************");
LOGGER.info("running " + counter + " : " + query.getPql());
final BrokerRequest brokerRequest =
RequestConverter.fromJSON(REQUEST_COMPILER.compile(query.getPql()));
InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest);
instanceRequest.setSearchSegments(new ArrayList<String>());
instanceRequest.getSearchSegments().add(segmentName);
final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
instanceResponseMap.clear();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
// compute value
Object val;
if (query instanceof AvroQueryGenerator.TestSimpleAggreationQuery) {
val = Double.parseDouble(brokerResponse.getAggregationResults().get(0).getString("value"));
} else {
val = brokerResponse.getAggregationResults().get(0).getJSONArray("groupByResult");
}
if (isAccurate) {
// store accurate value
accurateValue = val;
isAccurate = false;
} else {
// compare value with accurate value
// it's estimation so we need to test its result within error bound
if (query instanceof AvroQueryGenerator.TestSimpleAggreationQuery) {
TestUtils.assertApproximation((Double) val, (Double) accurateValue, precision);
} else {
TestUtils.assertJSONArrayApproximation(
(JSONArray) val, (JSONArray) accurateValue, precision);
}
isAccurate = true;
}
}
}
@Test
public void testSingleQuery() throws RecognitionException, Exception {
String query;
query = "select count(*) from testTable where column5='kCMyNVGCASKYDdQbftOPaqVMWc'";
// query= "select sum('count') from testTable where column1='660156454'";
LOGGER.info("running : " + query);
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query));
InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest);
instanceRequest.setSearchSegments(new ArrayList<String>());
instanceRequest.getSearchSegments().add(segmentName);
final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
instanceResponseMap.clear();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
}
@Test
public void testTrace() throws RecognitionException, Exception {
String query = "select count(*) from testTable where column1='186154188'";
LOGGER.info("running : " + query);
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query));
brokerRequest.setEnableTrace(true); //
InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest);
instanceRequest.setEnableTrace(true); // TODO: add trace settings consistency
instanceRequest.setSearchSegments(new ArrayList<String>());
instanceRequest.getSearchSegments().add(segmentName);
final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
instanceResponseMap.clear();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
LOGGER.info("TraceInfo is " + brokerResponse.getTraceInfo()); //
}
@Test
public void testRangeQuery() throws RecognitionException, Exception {
String query =
"select count(*) from testTable where column1 in ('999983251', '510705831', '1000720716', '1001058817', '1001099410')";
LOGGER.info("running : " + query);
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query));
InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest);
instanceRequest.setSearchSegments(new ArrayList<String>());
instanceRequest.getSearchSegments().add(segmentName);
final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
instanceResponseMap.clear();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
Assert.assertEquals(brokerResponse.getAggregationResults().get(0).getInt("value"), 14);
Assert.assertEquals(brokerResponse.getNumDocsScanned(), 14);
}
@Test
public void testAggregationGroupBy() throws Exception {
final List<TestGroupByAggreationQuery> groupByCalls =
AVRO_QUERY_GENERATOR.giveMeNGroupByAggregationQueries(10000);
int counter = 0;
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
for (final TestGroupByAggreationQuery groupBy : groupByCalls) {
LOGGER.info("running " + counter + " : " + groupBy.pql);
final BrokerRequest brokerRequest =
RequestConverter.fromJSON(REQUEST_COMPILER.compile(groupBy.pql));
InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest);
instanceRequest.setSearchSegments(new ArrayList<String>());
instanceRequest.getSearchSegments().add(segmentName);
final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
instanceResponseMap.clear();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
LOGGER.info("Result from avro is : " + groupBy.groupResults);
try {
assertGroupByResults(
brokerResponse.getAggregationResults().get(0).getJSONArray("groupByResult"),
groupBy.groupResults);
} catch (AssertionError e) {
System.out.println(groupBy.pql);
System.out.println(
"from broker : "
+ brokerResponse
.getAggregationResults()
.get(0)
.getJSONArray("groupByResult")
.toString());
System.out.println("from precomp : " + groupBy.groupResults);
throw new AssertionError(e);
}
}
}
private void assertBrokerResponse(int numSegments, BrokerResponse brokerResponse)
throws JSONException {
Assert.assertEquals(10001 * numSegments, brokerResponse.getNumDocsScanned());
final int groupSize = 15;
assertBrokerResponse(brokerResponse, groupSize);
// Assertion on Aggregation Results
final List<double[]> aggregationResult = getAggregationResult(numSegments);
final List<String[]> groupByResult = getGroupResult();
for (int j = 0; j < _numAggregations; ++j) {
final double[] aggResult = aggregationResult.get(j);
final String[] groupResult = groupByResult.get(j);
for (int i = 0; i < 15; ++i) {
Assert.assertEquals(
0,
DoubleComparisonUtil.defaultDoubleCompare(
aggResult[i],
brokerResponse
.getAggregationResults()
.get(j)
.getJSONArray("groupByResult")
.getJSONObject(i)
.getDouble("value")));
if ((i < 14 && aggResult[i] == aggResult[i + 1])
|| (i > 0 && aggResult[i] == aggResult[i - 1])) {
// do nothing, as we have multiple groups within same value.
} else {
Assert.assertEquals(
groupResult[i],
brokerResponse
.getAggregationResults()
.get(j)
.getJSONArray("groupByResult")
.getJSONObject(i)
.getString("group"));
}
}
}
}
private BrokerResponse getBrokerResponse(
PlanMaker instancePlanMaker, BrokerRequest brokerRequest) {
final ExecutorService executorService =
Executors.newCachedThreadPool(new NamedThreadFactory("test-plan-maker"));
final Plan globalPlan =
instancePlanMaker.makeInterSegmentPlan(
_indexSegmentList, brokerRequest, executorService, 150000);
globalPlan.print();
globalPlan.execute();
final DataTable instanceResponse = globalPlan.getInstanceResponse();
LOGGER.debug("Instance Response: {}", instanceResponse);
final DefaultReduceService defaultReduceService = new DefaultReduceService();
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
defaultReduceService.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.debug("Broker Response: {}", new JSONArray(brokerResponse.getAggregationResults()));
LOGGER.debug("Time used : {}", brokerResponse.getTimeUsedMs());
return brokerResponse;
}
@Test
public void testMatchAllQuery() throws RecognitionException, Exception {
String query = "select count(*),sum(count) from testTable ";
LOGGER.info("running : " + query);
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query));
InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest);
instanceRequest.setSearchSegments(new ArrayList<String>());
instanceRequest.getSearchSegments().add(segmentName);
final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
instanceResponseMap.clear();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.info("BrokerResponse is " + brokerResponse);
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(1));
Assert.assertEquals(brokerResponse.getAggregationResults().get(0).getInt("value"), 100000);
Assert.assertEquals(
brokerResponse.getAggregationResults().get(1).getDouble("value"), 8.90662862E13);
Assert.assertEquals(brokerResponse.getNumDocsScanned(), 100000);
}
@Test
public void testAggregation() throws Exception {
int counter = 0;
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
final List<TestSimpleAggreationQuery> aggCalls =
AVRO_QUERY_GENERATOR.giveMeNSimpleAggregationQueries(10000);
for (final TestSimpleAggreationQuery aggCall : aggCalls) {
LOGGER.info("running " + counter + " : " + aggCall.pql);
final BrokerRequest brokerRequest =
RequestConverter.fromJSON(REQUEST_COMPILER.compile(aggCall.pql));
InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest);
instanceRequest.setSearchSegments(new ArrayList<String>());
instanceRequest.getSearchSegments().add(segmentName);
final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
instanceResponseMap.clear();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
LOGGER.info("Result from avro is : " + aggCall.result);
try {
Assert.assertEquals(
Double.parseDouble(brokerResponse.getAggregationResults().get(0).getString("value")),
aggCall.result);
} catch (AssertionError e) {
System.out.println(aggCall.pql);
System.out.println(
"from broker : "
+ Double.parseDouble(
brokerResponse.getAggregationResults().get(0).getString("value")));
System.out.println("from precomp : " + aggCall.result);
throw new AssertionError(e);
}
}
}
private void assertBrokerResponse(int numSegments, BrokerResponse brokerResponse)
throws JSONException {
Assert.assertEquals(10001 * numSegments, brokerResponse.getNumDocsScanned());
Assert.assertEquals(_numAggregations, brokerResponse.getAggregationResults().size());
for (int i = 0; i < _numAggregations; ++i) {
Assert.assertEquals(
"[\"column11\",\"column10\"]",
brokerResponse.getAggregationResults().get(i).getJSONArray("groupByColumns").toString());
Assert.assertEquals(
15, brokerResponse.getAggregationResults().get(i).getJSONArray("groupByResult").length());
}
// Assertion on Count
assertionOnCount(brokerResponse);
// Assertion on Aggregation Results
LOGGER.info("brokerResponse = {}", brokerResponse);
final List<double[]> aggregationResult = getAggregationResult(numSegments);
final List<String[]> groupByResult = getGroupResult();
for (int j = 0; j < _numAggregations; ++j) {
final double[] aggResult = aggregationResult.get(j);
final String[] groupResult = groupByResult.get(j);
for (int i = 0; i < 15; ++i) {
Assert.assertEquals(
0,
DoubleComparisonUtil.defaultDoubleCompare(
aggResult[i],
brokerResponse
.getAggregationResults()
.get(j)
.getJSONArray("groupByResult")
.getJSONObject(i)
.getDouble("value")));
if ((i < 14 && aggResult[i] == aggResult[i + 1])
|| (i > 0 && aggResult[i] == aggResult[i - 1])) {
// do nothing, as we have multiple groups within same value.
} else {
Assert.assertEquals(
groupResult[i],
brokerResponse
.getAggregationResults()
.get(j)
.getJSONArray("groupByResult")
.getJSONObject(i)
.getString("group"));
}
}
}
}
private void assertionOnCount(BrokerResponse brokerResponse) throws JSONException {
Assert.assertEquals(
"count_star",
brokerResponse.getAggregationResults().get(0).getString("function").toString());
Assert.assertEquals(
"sum_met_impressionCount",
brokerResponse.getAggregationResults().get(1).getString("function").toString());
Assert.assertEquals(
"max_met_impressionCount",
brokerResponse.getAggregationResults().get(2).getString("function").toString());
Assert.assertEquals(
"min_met_impressionCount",
brokerResponse.getAggregationResults().get(3).getString("function").toString());
Assert.assertEquals(
"avg_met_impressionCount",
brokerResponse.getAggregationResults().get(4).getString("function").toString());
Assert.assertEquals(
"distinctCount_column12",
brokerResponse.getAggregationResults().get(5).getString("function").toString());
}
private void assertEmptyBrokerResponse(BrokerResponse brokerResponse) throws JSONException {
Assert.assertEquals(0, brokerResponse.getNumDocsScanned());
final int groupSize = 0;
assertBrokerResponse(brokerResponse, groupSize);
}