/**
* Produces single-node and distributed plans for testCase and compares plan and scan range
* results. Appends the actual single-node and distributed plan as well as the printed scan ranges
* to actualOutput, along with the requisite section header. locations to
* actualScanRangeLocations; compares both to the appropriate sections of 'testCase'.
*/
private void RunTestCase(
TestCase testCase,
StringBuilder errorLog,
StringBuilder actualOutput,
String dbName,
TQueryOptions options)
throws CatalogException {
if (options == null) {
options = defaultQueryOptions();
} else {
options = mergeQueryOptions(defaultQueryOptions(), options);
}
String query = testCase.getQuery();
LOG.info("running query " + query);
if (query.isEmpty()) {
throw new IllegalStateException(
"Cannot plan empty query in line: " + testCase.getStartingLineNum());
}
TQueryCtx queryCtx = TestUtils.createQueryContext(dbName, System.getProperty("user.name"));
queryCtx.request.query_options = options;
// single-node plan and scan range locations
testSingleNodePlan(testCase, queryCtx, errorLog, actualOutput);
testDistributedPlan(testCase, queryCtx, errorLog, actualOutput);
testColumnLineageOutput(testCase, queryCtx, errorLog, actualOutput);
}
/**
* Produces distributed plan for testCase and compares actual plan with expected plan. If testCase
* contains no expected distributed plan then this function is a no-op.
*/
private void testDistributedPlan(
TestCase testCase,
TQueryContext queryCtxt,
StringBuilder errorLog,
StringBuilder actualOutput)
throws CatalogException {
ArrayList<String> expectedPlan = testCase.getSectionContents(Section.DISTRIBUTEDPLAN);
// Test case has no expected distributed plan. Do not test it.
if (expectedPlan == null || expectedPlan.isEmpty()) return;
String query = testCase.getQuery();
String expectedErrorMsg = getExpectedErrorMessage(expectedPlan);
queryCtxt.request.getQuery_options().setNum_nodes(ImpalaInternalServiceConstants.NUM_NODES_ALL);
queryCtxt.request.setStmt(query);
boolean isImplemented = expectedErrorMsg == null;
StringBuilder explainBuilder = new StringBuilder();
actualOutput.append(Section.DISTRIBUTEDPLAN.getHeader() + "\n");
TExecRequest execRequest = null;
try {
// distributed plan
execRequest = frontend_.createExecRequest(queryCtxt, explainBuilder);
String explainStr = removeExplainHeader(explainBuilder.toString());
actualOutput.append(explainStr);
if (!isImplemented) {
errorLog.append(
"query produced DISTRIBUTEDPLAN\nquery=" + query + "\nplan=\n" + explainStr);
} else {
LOG.info("distributed plan: " + explainStr);
String result =
TestUtils.compareOutput(Lists.newArrayList(explainStr.split("\n")), expectedPlan, true);
if (!result.isEmpty()) {
errorLog.append(
"section "
+ Section.DISTRIBUTEDPLAN.toString()
+ " of query:\n"
+ query
+ "\n"
+ result);
}
}
} catch (ImpalaException e) {
if (e instanceof AnalysisException) {
errorLog.append("query:\n" + query + "\nanalysis error: " + e.getMessage() + "\n");
return;
} else if (e instanceof InternalException) {
errorLog.append("query:\n" + query + "\ninternal error: " + e.getMessage() + "\n");
return;
}
if (e instanceof NotImplementedException) {
handleNotImplException(query, expectedErrorMsg, errorLog, actualOutput, e);
} else if (e instanceof CatalogException) {
throw (CatalogException) e;
} else {
errorLog.append("query:\n" + query + "\nunhandled exception: " + e.getMessage() + "\n");
}
}
}
private void testColumnLineageOutput(
TestCase testCase, TQueryCtx queryCtx, StringBuilder errorLog, StringBuilder actualOutput)
throws CatalogException {
ArrayList<String> expectedLineage = testCase.getSectionContents(Section.LINEAGE);
if (expectedLineage == null || expectedLineage.isEmpty()) return;
String query = testCase.getQuery();
queryCtx.request.getQuery_options().setNum_nodes(1);
queryCtx.request.setStmt(query);
StringBuilder explainBuilder = new StringBuilder();
TExecRequest execRequest = null;
String lineageGraph = null;
try {
execRequest = frontend_.createExecRequest(queryCtx, explainBuilder);
if (execRequest.isSetQuery_exec_request()) {
lineageGraph = execRequest.query_exec_request.lineage_graph;
} else if (execRequest.isSetCatalog_op_request()) {
lineageGraph = execRequest.catalog_op_request.lineage_graph;
}
} catch (ImpalaException e) {
if (e instanceof AnalysisException) {
errorLog.append("query:\n" + query + "\nanalysis error: " + e.getMessage() + "\n");
return;
} else if (e instanceof InternalException) {
errorLog.append("query:\n" + query + "\ninternal error: " + e.getMessage() + "\n");
return;
}
if (e instanceof NotImplementedException) {
handleNotImplException(query, "", errorLog, actualOutput, e);
} else if (e instanceof CatalogException) {
throw (CatalogException) e;
} else {
errorLog.append("query:\n" + query + "\nunhandled exception: " + e.getMessage() + "\n");
}
}
LOG.info("lineage graph: " + lineageGraph);
ArrayList<String> expected = testCase.getSectionContents(Section.LINEAGE);
if (expected.size() > 0 && lineageGraph != null) {
String serializedGraph = Joiner.on("\n").join(expected);
ColumnLineageGraph expectedGraph = ColumnLineageGraph.createFromJSON(serializedGraph);
ColumnLineageGraph outputGraph = ColumnLineageGraph.createFromJSON(lineageGraph);
if (expectedGraph == null || outputGraph == null || !outputGraph.equals(expectedGraph)) {
StringBuilder lineageError = new StringBuilder();
lineageError.append("section " + Section.LINEAGE + " of query:\n" + query + "\n");
lineageError.append("Output:\n");
lineageError.append(lineageGraph + "\n");
lineageError.append("Expected:\n");
lineageError.append(serializedGraph + "\n");
errorLog.append(lineageError.toString());
}
actualOutput.append(Section.LINEAGE.getHeader());
actualOutput.append(TestUtils.prettyPrintJson(lineageGraph) + "\n");
}
}
/**
* Produces single-node and distributed plans for testCase and compares plan and scan range
* results. Appends the actual single-node and distributed plan as well as the printed scan ranges
* to actualOutput, along with the requisite section header. locations to
* actualScanRangeLocations; compares both to the appropriate sections of 'testCase'.
*/
private void RunTestCase(
TestCase testCase, StringBuilder errorLog, StringBuilder actualOutput, String dbName)
throws CatalogException {
String query = testCase.getQuery();
LOG.info("running query " + query);
TQueryContext queryCtxt = TestUtils.createQueryContext(dbName, System.getProperty("user.name"));
queryCtxt.request.query_options.setExplain_level(TExplainLevel.STANDARD);
queryCtxt.request.query_options.allow_unsupported_formats = true;
// single-node plan and scan range locations
testSingleNodePlan(testCase, queryCtxt, errorLog, actualOutput);
// distributed plan
testDistributedPlan(testCase, queryCtxt, errorLog, actualOutput);
}
private void runPlannerTestFile(String testFile, String dbName) {
String fileName = testDir_ + "/" + testFile + ".test";
TestFileParser queryFileParser = new TestFileParser(fileName);
StringBuilder actualOutput = new StringBuilder();
queryFileParser.parseFile();
StringBuilder errorLog = new StringBuilder();
for (TestCase testCase : queryFileParser.getTestCases()) {
actualOutput.append(testCase.getSectionAsString(Section.QUERY, true, "\n"));
actualOutput.append("\n");
try {
RunTestCase(testCase, errorLog, actualOutput, dbName);
} catch (CatalogException e) {
errorLog.append(
String.format("Failed to plan query\n%s\n%s", testCase.getQuery(), e.getMessage()));
}
actualOutput.append("====\n");
}
// Create the actual output file
if (GENERATE_OUTPUT_FILE) {
try {
File outDirFile = new File(outDir_);
outDirFile.mkdirs();
FileWriter fw = new FileWriter(outDir_ + testFile + ".test");
fw.write(actualOutput.toString());
fw.close();
} catch (IOException e) {
errorLog.append("Unable to create output file: " + e.getMessage());
}
}
if (errorLog.length() != 0) {
fail(errorLog.toString());
}
}
/**
* Produces single-node plan for testCase and compares actual plan with expected plan, as well as
* the scan range locations. If testCase contains no expected single-node plan then this function
* is a no-op.
*/
private void testSingleNodePlan(
TestCase testCase,
TQueryContext queryCtxt,
StringBuilder errorLog,
StringBuilder actualOutput)
throws CatalogException {
ArrayList<String> expectedPlan = testCase.getSectionContents(Section.PLAN);
// Test case has no expected single-node plan. Do not test it.
if (expectedPlan == null || expectedPlan.isEmpty()) return;
String query = testCase.getQuery();
String expectedErrorMsg = getExpectedErrorMessage(expectedPlan);
queryCtxt.request.getQuery_options().setNum_nodes(1);
queryCtxt.request.setStmt(query);
boolean isImplemented = expectedErrorMsg == null;
StringBuilder explainBuilder = new StringBuilder();
TExecRequest execRequest = null;
String locationsStr = null;
actualOutput.append(Section.PLAN.getHeader() + "\n");
try {
execRequest = frontend_.createExecRequest(queryCtxt, explainBuilder);
String explainStr = removeExplainHeader(explainBuilder.toString());
actualOutput.append(explainStr);
if (!isImplemented) {
errorLog.append("query produced PLAN\nquery=" + query + "\nplan=\n" + explainStr);
} else {
LOG.info("single-node plan: " + explainStr);
String result =
TestUtils.compareOutput(Lists.newArrayList(explainStr.split("\n")), expectedPlan, true);
if (!result.isEmpty()) {
errorLog.append(
"section " + Section.PLAN.toString() + " of query:\n" + query + "\n" + result);
}
// Query exec request may not be set for DDL, e.g., CTAS.
if (execRequest.isSetQuery_exec_request()) {
locationsStr = PrintScanRangeLocations(execRequest.query_exec_request).toString();
}
}
} catch (ImpalaException e) {
if (e instanceof AnalysisException) {
errorLog.append("query:\n" + query + "\nanalysis error: " + e.getMessage() + "\n");
return;
} else if (e instanceof InternalException) {
errorLog.append("query:\n" + query + "\ninternal error: " + e.getMessage() + "\n");
return;
}
if (e instanceof NotImplementedException) {
handleNotImplException(query, expectedErrorMsg, errorLog, actualOutput, e);
} else if (e instanceof CatalogException) {
// TODO: do we need to rethrow?
throw (CatalogException) e;
} else {
errorLog.append("query:\n" + query + "\nunhandled exception: " + e.getMessage() + "\n");
}
}
// compare scan range locations
LOG.info("scan range locations: " + locationsStr);
ArrayList<String> expectedLocations = testCase.getSectionContents(Section.SCANRANGELOCATIONS);
if (expectedLocations.size() > 0 && locationsStr != null) {
// Locations' order does not matter.
String result =
TestUtils.compareOutput(
Lists.newArrayList(locationsStr.split("\n")), expectedLocations, false);
if (!result.isEmpty()) {
errorLog.append(
"section " + Section.SCANRANGELOCATIONS + " of query:\n" + query + "\n" + result);
}
actualOutput.append(Section.SCANRANGELOCATIONS.getHeader() + "\n");
actualOutput.append(locationsStr);
// TODO: check that scan range locations are identical in both cases
}
}
/**
* Produces single-node plan for testCase and compares actual plan with expected plan, as well as
* the scan range locations. If testCase contains no expected single-node plan then this function
* is a no-op.
*/
private void testSingleNodePlan(
TestCase testCase, TQueryCtx queryCtx, StringBuilder errorLog, StringBuilder actualOutput)
throws CatalogException {
ArrayList<String> expectedPlan = testCase.getSectionContents(Section.PLAN);
// Test case has no expected single-node plan. Do not test it.
if (expectedPlan == null || expectedPlan.isEmpty()) return;
String query = testCase.getQuery();
String expectedErrorMsg = getExpectedErrorMessage(expectedPlan);
queryCtx.request.getQuery_options().setNum_nodes(1);
queryCtx.request.setStmt(query);
boolean isImplemented = expectedErrorMsg == null;
StringBuilder explainBuilder = new StringBuilder();
TExecRequest execRequest = null;
String locationsStr = null;
actualOutput.append(Section.PLAN.getHeader() + "\n");
try {
execRequest = frontend_.createExecRequest(queryCtx, explainBuilder);
buildMaps(execRequest.query_exec_request);
String explainStr = removeExplainHeader(explainBuilder.toString());
actualOutput.append(explainStr);
if (!isImplemented) {
errorLog.append("query produced PLAN\nquery=" + query + "\nplan=\n" + explainStr);
} else {
LOG.info("single-node plan: " + explainStr);
String result =
TestUtils.compareOutput(Lists.newArrayList(explainStr.split("\n")), expectedPlan, true);
if (!result.isEmpty()) {
errorLog.append(
"section " + Section.PLAN.toString() + " of query:\n" + query + "\n" + result);
}
// Query exec request may not be set for DDL, e.g., CTAS.
if (execRequest.isSetQuery_exec_request()) {
testHdfsPartitionsReferenced(execRequest.query_exec_request, query, errorLog);
locationsStr = PrintScanRangeLocations(execRequest.query_exec_request).toString();
}
}
} catch (ImpalaException e) {
if (e instanceof AnalysisException) {
errorLog.append("query:\n" + query + "\nanalysis error: " + e.getMessage() + "\n");
return;
} else if (e instanceof InternalException) {
errorLog.append("query:\n" + query + "\ninternal error: " + e.getMessage() + "\n");
return;
}
if (e instanceof NotImplementedException) {
handleNotImplException(query, expectedErrorMsg, errorLog, actualOutput, e);
} else if (e instanceof CatalogException) {
// TODO: do we need to rethrow?
throw (CatalogException) e;
} else {
errorLog.append("query:\n" + query + "\nunhandled exception: " + e.getMessage() + "\n");
}
}
// compare scan range locations
LOG.info("scan range locations: " + locationsStr);
ArrayList<String> expectedLocations = testCase.getSectionContents(Section.SCANRANGELOCATIONS);
if (expectedLocations.size() > 0 && locationsStr != null) {
// Locations' order does not matter.
String result =
TestUtils.compareOutput(
Lists.newArrayList(locationsStr.split("\n")), expectedLocations, false);
if (!result.isEmpty()) {
errorLog.append(
"section " + Section.SCANRANGELOCATIONS + " of query:\n" + query + "\n" + result);
}
actualOutput.append(Section.SCANRANGELOCATIONS.getHeader() + "\n");
// Print the locations out sorted since the order is random and messed up
// the diffs. The values in locationStr contains "Node X" labels as well
// as paths.
ArrayList<String> locations = Lists.newArrayList(locationsStr.split("\n"));
ArrayList<String> perNodeLocations = Lists.newArrayList();
for (int i = 0; i < locations.size(); ++i) {
if (locations.get(i).startsWith("NODE")) {
if (!perNodeLocations.isEmpty()) {
Collections.sort(perNodeLocations);
actualOutput.append(Joiner.on("\n").join(perNodeLocations)).append("\n");
perNodeLocations.clear();
}
actualOutput.append(locations.get(i)).append("\n");
} else {
perNodeLocations.add(locations.get(i));
}
}
if (!perNodeLocations.isEmpty()) {
Collections.sort(perNodeLocations);
actualOutput.append(Joiner.on("\n").join(perNodeLocations)).append("\n");
}
// TODO: check that scan range locations are identical in both cases
}
}