/**
* @param functionNames: accurate function comes first
* @param testColumns
* @param groupByColumn
* @param precision
* @throws Exception
*/
private void testApproximationQuery(
String[] functionNames, String[] testColumns, String groupByColumn, double precision)
throws Exception {
String query;
boolean hasWhere = true;
LOGGER.debug(
"========================== Test Total "
+ testColumns.length * 2
+ " Queries ==========================");
for (String column : testColumns) {
for (int i = 0; i < 2; i++) {
query = "select " + functionNames[0] + "(" + column + ") from 'mytable'";
if (hasWhere) {
query += " where DaysSinceEpoch >= 16312";
}
if (groupByColumn != null) {
query += " group by " + groupByColumn;
JSONArray accurate = getGroupByArrayFromJSONAggregationResults(postQuery(query));
query = query.replace(functionNames[0], functionNames[1]);
JSONArray estimate = getGroupByArrayFromJSONAggregationResults(postQuery(query));
TestUtils.assertJSONArrayApproximation(estimate, accurate, precision);
} else {
double accurate =
Double.parseDouble(getSingleStringValueFromJSONAggregationResults(postQuery(query)));
query = query.replace(functionNames[0], functionNames[1]);
double estimate =
Double.parseDouble(getSingleStringValueFromJSONAggregationResults(postQuery(query)));
TestUtils.assertApproximation(estimate, accurate, precision);
//
}
LOGGER.debug("========================== End ==========================");
hasWhere = !hasWhere;
}
}
}
private void runApproximationQueries(
List<? extends AvroQueryGenerator.TestAggreationQuery> queries, double precision)
throws Exception {
boolean isAccurate = true;
Object accurateValue = null;
int counter = 0;
final Map<ServerInstance, DataTable> instanceResponseMap =
new HashMap<ServerInstance, DataTable>();
for (final AvroQueryGenerator.TestAggreationQuery query : queries) {
LOGGER.info("**************************");
LOGGER.info("running " + counter + " : " + query.getPql());
final BrokerRequest brokerRequest =
RequestConverter.fromJSON(REQUEST_COMPILER.compile(query.getPql()));
InstanceRequest instanceRequest = new InstanceRequest(counter++, brokerRequest);
instanceRequest.setSearchSegments(new ArrayList<String>());
instanceRequest.getSearchSegments().add(segmentName);
final DataTable instanceResponse = QUERY_EXECUTOR.processQuery(instanceRequest);
instanceResponseMap.clear();
instanceResponseMap.put(new ServerInstance("localhost:0000"), instanceResponse);
final BrokerResponse brokerResponse =
REDUCE_SERVICE.reduceOnDataTable(brokerRequest, instanceResponseMap);
LOGGER.info("BrokerResponse is " + brokerResponse.getAggregationResults().get(0));
// compute value
Object val;
if (query instanceof AvroQueryGenerator.TestSimpleAggreationQuery) {
val = Double.parseDouble(brokerResponse.getAggregationResults().get(0).getString("value"));
} else {
val = brokerResponse.getAggregationResults().get(0).getJSONArray("groupByResult");
}
if (isAccurate) {
// store accurate value
accurateValue = val;
isAccurate = false;
} else {
// compare value with accurate value
// it's estimation so we need to test its result within error bound
if (query instanceof AvroQueryGenerator.TestSimpleAggreationQuery) {
TestUtils.assertApproximation((Double) val, (Double) accurateValue, precision);
} else {
TestUtils.assertJSONArrayApproximation(
(JSONArray) val, (JSONArray) accurateValue, precision);
}
isAccurate = true;
}
}
}
private void setupSegmentList(int numberOfSegments) throws Exception {
final String filePath =
TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(SMALL_AVRO_DATA));
_indexSegmentList.clear();
if (INDEXES_DIR.exists()) {
FileUtils.deleteQuietly(INDEXES_DIR);
}
INDEXES_DIR.mkdir();
for (int i = 0; i < numberOfSegments; ++i) {
final File segmentDir = new File(INDEXES_DIR, "segment_" + i);
final SegmentGeneratorConfig config =
SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(
new File(filePath), segmentDir, "dim1", TimeUnit.DAYS, "test");
final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
driver.init(config);
driver.build();
LOGGER.debug("built at: {}", segmentDir.getAbsolutePath());
_indexSegmentList.add(
ColumnarSegmentLoader.load(new File(segmentDir, driver.getSegmentName()), ReadMode.mmap));
}
}
private void setUpTestQueries(String table) throws FileNotFoundException, IOException {
final String filePath =
TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA));
System.out.println(filePath);
final List<String> dims = new ArrayList<String>();
dims.add("column1");
dims.add("column2");
dims.add("column3");
dims.add("column4");
dims.add("column5");
dims.add("column6");
dims.add("column7");
dims.add("column8");
dims.add("column9");
dims.add("column10");
dims.add("weeksSinceEpochSunday");
dims.add("daysSinceEpoch");
dims.add("count");
final List<String> mets = new ArrayList<String>();
mets.add("count");
final String time = "minutesSinceEpoch";
AVRO_QUERY_GENERATOR = new AvroQueryGenerator(new File(filePath), dims, mets, time, table);
AVRO_QUERY_GENERATOR.init();
AVRO_QUERY_GENERATOR.generateSimpleAggregationOnSingleColumnFilters();
}
private void setupSegmentList(int numberOfSegments) throws Exception {
final String filePath =
TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(SMALL_AVRO_DATA));
_indexSegmentList.clear();
if (INDEXES_DIR.exists()) {
FileUtils.deleteQuietly(INDEXES_DIR);
}
INDEXES_DIR.mkdir();
for (int i = 0; i < numberOfSegments; ++i) {
final File segmentDir = new File(INDEXES_DIR, "segment_" + i);
final SegmentGeneratorConfig config =
SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(
new File(filePath), segmentDir, "dim" + i, TimeUnit.DAYS, "midas");
config.setSegmentNamePostfix(String.valueOf(i));
final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
driver.init(config);
driver.build();
File parent = new File(INDEXES_DIR, "segment_" + String.valueOf(i));
String segmentName = parent.list()[0];
_indexSegmentList.add(
ColumnarSegmentLoader.load(new File(parent, segmentName), ReadMode.mmap));
System.out.println("built at : " + segmentDir.getAbsolutePath());
}
}
@BeforeClass
public static void before() throws Exception {
final String filePath =
TestUtils.getFileFromResourceUrl(
DictionariesTest.class.getClassLoader().getResource(AVRO_DATA));
if (INDEX_DIR.exists()) {
FileUtils.deleteQuietly(INDEX_DIR);
}
final SegmentGeneratorConfig config =
SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(
new File(filePath), INDEX_DIR, "time_day", TimeUnit.DAYS, "test");
final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
driver.init(config);
driver.build();
final Schema schema = AvroUtils.extractSchemaFromAvro(new File(filePath));
final DataFileStream<GenericRecord> avroReader = AvroUtils.getAvroReader(new File(filePath));
final org.apache.avro.Schema avroSchema = avroReader.getSchema();
final String[] columns = new String[avroSchema.getFields().size()];
int i = 0;
for (final Field f : avroSchema.getFields()) {
columns[i] = f.name();
i++;
}
uniqueEntries = new HashMap<String, Set<Object>>();
for (final String column : columns) {
uniqueEntries.put(column, new HashSet<Object>());
}
while (avroReader.hasNext()) {
final GenericRecord rec = avroReader.next();
for (final String column : columns) {
Object val = rec.get(column);
if (val instanceof Utf8) {
val = ((Utf8) val).toString();
}
uniqueEntries
.get(column)
.add(getAppropriateType(schema.getFieldSpecFor(column).getDataType(), val));
}
}
}
private void setupSegment() throws Exception {
final String filePath =
TestUtils.getFileFromResourceUrl(
PlanMakerTest.class.getClassLoader().getResource(LARGE_AVRO_DATA));
if (INDEX_DIR.exists()) {
FileUtils.deleteQuietly(INDEX_DIR);
}
final SegmentGeneratorConfig config =
SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(
new File(filePath), INDEX_DIR, "dim1", TimeUnit.DAYS, "test");
final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
driver.init(config);
driver.build();
LOGGER.debug("built at: {}", INDEX_DIR.getAbsolutePath());
final File indexSegmentDir = new File(INDEX_DIR, driver.getSegmentName());
_indexSegment = ColumnarSegmentLoader.load(indexSegmentDir, ReadMode.mmap);
}
private void setupSegment() throws Exception {
final String filePath =
TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA));
if (INDEX_DIR.exists()) {
FileUtils.deleteQuietly(INDEX_DIR);
}
final SegmentGeneratorConfig config =
SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(
new File(filePath), INDEX_DIR, "time_day", TimeUnit.DAYS, "test");
final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
driver.init(config);
driver.build();
LOGGER.info("built at : {}", INDEX_DIR.getAbsolutePath());
final File indexSegmentDir = new File(INDEX_DIR, driver.getSegmentName());
_indexSegment = ColumnarSegmentLoader.load(indexSegmentDir, ReadMode.heap);
_medataMap =
((SegmentMetadataImpl) ((IndexSegmentImpl) _indexSegment).getSegmentMetadata())
.getColumnMetadataMap();
}
private void setupSegmentFor(String table) throws Exception {
final String filePath =
TestUtils.getFileFromResourceUrl(getClass().getClassLoader().getResource(AVRO_DATA));
if (INDEX_DIR.exists()) {
FileUtils.deleteQuietly(INDEX_DIR);
}
INDEX_DIR.mkdir();
final SegmentGeneratorConfig config =
SegmentTestUtils.getSegmentGenSpecWithSchemAndProjectedColumns(
new File(filePath),
new File(INDEX_DIR, "segment"),
"daysSinceEpoch",
TimeUnit.DAYS,
table);
final SegmentIndexCreationDriver driver = new SegmentIndexCreationDriverImpl();
driver.init(config);
driver.build();
System.out.println("built at : " + INDEX_DIR.getAbsolutePath());
}