@BeforeClass
public void setup() throws Exception {
CONFIG_BUILDER = new TestingServerPropertiesBuilder("testTable");
setupSegmentFor("testTable");
setUpTestQueries("testTable");
final PropertiesConfiguration serverConf = CONFIG_BUILDER.build();
serverConf.setDelimiterParsingDisabled(false);
final FileBasedInstanceDataManager instanceDataManager =
FileBasedInstanceDataManager.getInstanceDataManager();
instanceDataManager.init(
new FileBasedInstanceDataManagerConfig(serverConf.subset("pinot.server.instance")));
instanceDataManager.start();
System.out.println(
"************************** : " + new File(INDEX_DIR, "segment").getAbsolutePath());
File segmentFile = new File(INDEX_DIR, "segment").listFiles()[0];
segmentName = segmentFile.getName();
final IndexSegment indexSegment = ColumnarSegmentLoader.load(segmentFile, ReadMode.heap);
instanceDataManager.getTableDataManager("testTable");
instanceDataManager.getTableDataManager("testTable").addSegment(indexSegment);
QUERY_EXECUTOR = new ServerQueryExecutorV1Impl(false);
QUERY_EXECUTOR.init(
serverConf.subset("pinot.server.query.executor"),
instanceDataManager,
new ServerMetrics(new MetricsRegistry()));
}
private void runApproximationQueries(
List<? extends AvroQueryGenerator.TestAggreationQuery> queries, double precision)
throws Exception {
boolean isAccurate = true;
Object accurateValue = null;
int counter = 0;
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
for (final AvroQueryGenerator.TestAggreationQuery query : queries) {
LOGGER.info("**************************");
LOGGER.info("running " + counter + " : " + query.getPql());
final BrokerRequest brokerRequest =
RequestConverter.fromJSON(REQUEST_COMPILER.compile(query.getPql()));
InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest);
instanceRequest.setSearchSegments(new ArrayList<String>());
instanceRequest.getSearchSegments().add(segmentName);
final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
instanceResponseMap.clear();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
// compute value
Object val;
if (query instanceof AvroQueryGenerator.TestSimpleAggreationQuery) {
val = Double.parseDouble(brokerResponse.getAggregationResults().get(0).getString("value"));
} else {
val = brokerResponse.getAggregationResults().get(0).getJSONArray("groupByResult");
}
if (isAccurate) {
// store accurate value
accurateValue = val;
isAccurate = false;
} else {
// compare value with accurate value
// it's estimation so we need to test its result within error bound
if (query instanceof AvroQueryGenerator.TestSimpleAggreationQuery) {
TestUtils.assertApproximation((Double) val, (Double) accurateValue, precision);
} else {
TestUtils.assertJSONArrayApproximation(
(JSONArray) val, (JSONArray) accurateValue, precision);
}
isAccurate = true;
}
}
}
@Test
public void testSingleQuery() throws RecognitionException, Exception {
String query;
query = "select count(*) from testTable where column5='kCMyNVGCASKYDdQbftOPaqVMWc'";
// query= "select sum('count') from testTable where column1='660156454'";
LOGGER.info("running : " + query);
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query));
InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest);
instanceRequest.setSearchSegments(new ArrayList<String>());
instanceRequest.getSearchSegments().add(segmentName);
final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
instanceResponseMap.clear();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
}
@Test
public void testTrace() throws RecognitionException, Exception {
String query = "select count(*) from testTable where column1='186154188'";
LOGGER.info("running : " + query);
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query));
brokerRequest.setEnableTrace(true); //
InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest);
instanceRequest.setEnableTrace(true); // TODO: add trace settings consistency
instanceRequest.setSearchSegments(new ArrayList<String>());
instanceRequest.getSearchSegments().add(segmentName);
final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
instanceResponseMap.clear();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
LOGGER.info("TraceInfo is " + brokerResponse.getTraceInfo()); //
}
@Test
public void testRangeQuery() throws RecognitionException, Exception {
String query =
"select count(*) from testTable where column1 in ('999983251', '510705831', '1000720716', '1001058817', '1001099410')";
LOGGER.info("running : " + query);
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query));
InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest);
instanceRequest.setSearchSegments(new ArrayList<String>());
instanceRequest.getSearchSegments().add(segmentName);
final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
instanceResponseMap.clear();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
Assert.assertEquals(brokerResponse.getAggregationResults().get(0).getInt("value"), 14);
Assert.assertEquals(brokerResponse.getNumDocsScanned(), 14);
}
@Test
public void testAggregationGroupBy() throws Exception {
final List<TestGroupByAggreationQuery> groupByCalls =
AVRO_QUERY_GENERATOR.giveMeNGroupByAggregationQueries(10000);
int counter = 0;
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
for (final TestGroupByAggreationQuery groupBy : groupByCalls) {
LOGGER.info("running " + counter + " : " + groupBy.pql);
final BrokerRequest brokerRequest =
RequestConverter.fromJSON(REQUEST_COMPILER.compile(groupBy.pql));
InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest);
instanceRequest.setSearchSegments(new ArrayList<String>());
instanceRequest.getSearchSegments().add(segmentName);
final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
instanceResponseMap.clear();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
LOGGER.info("Result from avro is : " + groupBy.groupResults);
try {
assertGroupByResults(
brokerResponse.getAggregationResults().get(0).getJSONArray("groupByResult"),
groupBy.groupResults);
} catch (AssertionError e) {
System.out.println(groupBy.pql);
System.out.println(
"from broker : "
+ brokerResponse
.getAggregationResults()
.get(0)
.getJSONArray("groupByResult")
.toString());
System.out.println("from precomp : " + groupBy.groupResults);
throw new AssertionError(e);
}
}
}
@Test
public void testMatchAllQuery() throws RecognitionException, Exception {
String query = "select count(*),sum(count) from testTable ";
LOGGER.info("running : " + query);
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
final BrokerRequest brokerRequest = RequestConverter.fromJSON(REQUEST_COMPILER.compile(query));
InstanceRequest instanceRequest = new InstanceRequest(1, brokerRequest);
instanceRequest.setSearchSegments(new ArrayList<String>());
instanceRequest.getSearchSegments().add(segmentName);
final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
instanceResponseMap.clear();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.info("BrokerResponse is " + brokerResponse);
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(1));
Assert.assertEquals(brokerResponse.getAggregationResults().get(0).getInt("value"), 100000);
Assert.assertEquals(
brokerResponse.getAggregationResults().get(1).getDouble("value"), 8.90662862E13);
Assert.assertEquals(brokerResponse.getNumDocsScanned(), 100000);
}
@Test
public void testAggregation() throws Exception {
int counter = 0;
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
final List<TestSimpleAggreationQuery> aggCalls =
AVRO_QUERY_GENERATOR.giveMeNSimpleAggregationQueries(10000);
for (final TestSimpleAggreationQuery aggCall : aggCalls) {
LOGGER.info("running " + counter + " : " + aggCall.pql);
final BrokerRequest brokerRequest =
RequestConverter.fromJSON(REQUEST_COMPILER.compile(aggCall.pql));
InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest);
instanceRequest.setSearchSegments(new ArrayList<String>());
instanceRequest.getSearchSegments().add(segmentName);
final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
instanceResponseMap.clear();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
LOGGER.info("Result from avro is : " + aggCall.result);
try {
Assert.assertEquals(
Double.parseDouble(brokerResponse.getAggregationResults().get(0).getString("value")),
aggCall.result);
} catch (AssertionError e) {
System.out.println(aggCall.pql);
System.out.println(
"from broker : "
+ Double.parseDouble(
brokerResponse.getAggregationResults().get(0).getString("value")));
System.out.println("from precomp : " + aggCall.result);
throw new AssertionError(e);
}
}
}
