private void testColumnLineageOutput(
TestCase testCase, TQueryCtx queryCtx, StringBuilder errorLog, StringBuilder actualOutput)
throws CatalogException {
ArrayList<String> expectedLineage = testCase.getSectionContents(Section.LINEAGE);
if (expectedLineage == null || expectedLineage.isEmpty()) return;
String query = testCase.getQuery();
queryCtx.request.getQuery_options().setNum_nodes(1);
queryCtx.request.setStmt(query);
StringBuilder explainBuilder = new StringBuilder();
TExecRequest execRequest = null;
String lineageGraph = null;
try {
execRequest = frontend_.createExecRequest(queryCtx, explainBuilder);
if (execRequest.isSetQuery_exec_request()) {
lineageGraph = execRequest.query_exec_request.lineage_graph;
} else if (execRequest.isSetCatalog_op_request()) {
lineageGraph = execRequest.catalog_op_request.lineage_graph;
}
} catch (ImpalaException e) {
if (e instanceof AnalysisException) {
errorLog.append("query:\n" + query + "\nanalysis error: " + e.getMessage() + "\n");
return;
} else if (e instanceof InternalException) {
errorLog.append("query:\n" + query + "\ninternal error: " + e.getMessage() + "\n");
return;
}
if (e instanceof NotImplementedException) {
handleNotImplException(query, "", errorLog, actualOutput, e);
} else if (e instanceof CatalogException) {
throw (CatalogException) e;
} else {
errorLog.append("query:\n" + query + "\nunhandled exception: " + e.getMessage() + "\n");
}
}
LOG.info("lineage graph: " + lineageGraph);
ArrayList<String> expected = testCase.getSectionContents(Section.LINEAGE);
if (expected.size() > 0 && lineageGraph != null) {
String serializedGraph = Joiner.on("\n").join(expected);
ColumnLineageGraph expectedGraph = ColumnLineageGraph.createFromJSON(serializedGraph);
ColumnLineageGraph outputGraph = ColumnLineageGraph.createFromJSON(lineageGraph);
if (expectedGraph == null || outputGraph == null || !outputGraph.equals(expectedGraph)) {
StringBuilder lineageError = new StringBuilder();
lineageError.append("section " + Section.LINEAGE + " of query:\n" + query + "\n");
lineageError.append("Output:\n");
lineageError.append(lineageGraph + "\n");
lineageError.append("Expected:\n");
lineageError.append(serializedGraph + "\n");
errorLog.append(lineageError.toString());
}
actualOutput.append(Section.LINEAGE.getHeader());
actualOutput.append(TestUtils.prettyPrintJson(lineageGraph) + "\n");
}
}
/**
* Jni wrapper for Frontend.createQueryExecRequest2(). Accepts a serialized TClientRequest;
* returns a serialized TQueryExecRequest2.
*/
public byte[] createExecRequest(byte[] thriftClientRequest) throws ImpalaException {
TClientRequest request = new TClientRequest();
deserializeThrift(request, thriftClientRequest);
StringBuilder explainString = new StringBuilder();
TExecRequest result = frontend.createExecRequest(request, explainString);
LOG.info(explainString.toString());
LOG.info("returned TQueryExecRequest2: " + result.toString());
// TODO: avoid creating serializer for each query?
TSerializer serializer = new TSerializer(protocolFactory);
try {
return serializer.serialize(result);
} catch (TException e) {
throw new InternalException(e.getMessage());
}
}
/**
* Produces single-node plan for testCase and compares actual plan with expected plan, as well as
* the scan range locations. If testCase contains no expected single-node plan then this function
* is a no-op.
*/
private void testSingleNodePlan(
TestCase testCase,
TQueryContext queryCtxt,
StringBuilder errorLog,
StringBuilder actualOutput)
throws CatalogException {
ArrayList<String> expectedPlan = testCase.getSectionContents(Section.PLAN);
// Test case has no expected single-node plan. Do not test it.
if (expectedPlan == null || expectedPlan.isEmpty()) return;
String query = testCase.getQuery();
String expectedErrorMsg = getExpectedErrorMessage(expectedPlan);
queryCtxt.request.getQuery_options().setNum_nodes(1);
queryCtxt.request.setStmt(query);
boolean isImplemented = expectedErrorMsg == null;
StringBuilder explainBuilder = new StringBuilder();
TExecRequest execRequest = null;
String locationsStr = null;
actualOutput.append(Section.PLAN.getHeader() + "\n");
try {
execRequest = frontend_.createExecRequest(queryCtxt, explainBuilder);
String explainStr = removeExplainHeader(explainBuilder.toString());
actualOutput.append(explainStr);
if (!isImplemented) {
errorLog.append("query produced PLAN\nquery=" + query + "\nplan=\n" + explainStr);
} else {
LOG.info("single-node plan: " + explainStr);
String result =
TestUtils.compareOutput(Lists.newArrayList(explainStr.split("\n")), expectedPlan, true);
if (!result.isEmpty()) {
errorLog.append(
"section " + Section.PLAN.toString() + " of query:\n" + query + "\n" + result);
}
// Query exec request may not be set for DDL, e.g., CTAS.
if (execRequest.isSetQuery_exec_request()) {
locationsStr = PrintScanRangeLocations(execRequest.query_exec_request).toString();
}
}
} catch (ImpalaException e) {
if (e instanceof AnalysisException) {
errorLog.append("query:\n" + query + "\nanalysis error: " + e.getMessage() + "\n");
return;
} else if (e instanceof InternalException) {
errorLog.append("query:\n" + query + "\ninternal error: " + e.getMessage() + "\n");
return;
}
if (e instanceof NotImplementedException) {
handleNotImplException(query, expectedErrorMsg, errorLog, actualOutput, e);
} else if (e instanceof CatalogException) {
// TODO: do we need to rethrow?
throw (CatalogException) e;
} else {
errorLog.append("query:\n" + query + "\nunhandled exception: " + e.getMessage() + "\n");
}
}
// compare scan range locations
LOG.info("scan range locations: " + locationsStr);
ArrayList<String> expectedLocations = testCase.getSectionContents(Section.SCANRANGELOCATIONS);
if (expectedLocations.size() > 0 && locationsStr != null) {
// Locations' order does not matter.
String result =
TestUtils.compareOutput(
Lists.newArrayList(locationsStr.split("\n")), expectedLocations, false);
if (!result.isEmpty()) {
errorLog.append(
"section " + Section.SCANRANGELOCATIONS + " of query:\n" + query + "\n" + result);
}
actualOutput.append(Section.SCANRANGELOCATIONS.getHeader() + "\n");
actualOutput.append(locationsStr);
// TODO: check that scan range locations are identical in both cases
}
}
/**
* Produces single-node plan for testCase and compares actual plan with expected plan, as well as
* the scan range locations. If testCase contains no expected single-node plan then this function
* is a no-op.
*/
private void testSingleNodePlan(
TestCase testCase, TQueryCtx queryCtx, StringBuilder errorLog, StringBuilder actualOutput)
throws CatalogException {
ArrayList<String> expectedPlan = testCase.getSectionContents(Section.PLAN);
// Test case has no expected single-node plan. Do not test it.
if (expectedPlan == null || expectedPlan.isEmpty()) return;
String query = testCase.getQuery();
String expectedErrorMsg = getExpectedErrorMessage(expectedPlan);
queryCtx.request.getQuery_options().setNum_nodes(1);
queryCtx.request.setStmt(query);
boolean isImplemented = expectedErrorMsg == null;
StringBuilder explainBuilder = new StringBuilder();
TExecRequest execRequest = null;
String locationsStr = null;
actualOutput.append(Section.PLAN.getHeader() + "\n");
try {
execRequest = frontend_.createExecRequest(queryCtx, explainBuilder);
buildMaps(execRequest.query_exec_request);
String explainStr = removeExplainHeader(explainBuilder.toString());
actualOutput.append(explainStr);
if (!isImplemented) {
errorLog.append("query produced PLAN\nquery=" + query + "\nplan=\n" + explainStr);
} else {
LOG.info("single-node plan: " + explainStr);
String result =
TestUtils.compareOutput(Lists.newArrayList(explainStr.split("\n")), expectedPlan, true);
if (!result.isEmpty()) {
errorLog.append(
"section " + Section.PLAN.toString() + " of query:\n" + query + "\n" + result);
}
// Query exec request may not be set for DDL, e.g., CTAS.
if (execRequest.isSetQuery_exec_request()) {
testHdfsPartitionsReferenced(execRequest.query_exec_request, query, errorLog);
locationsStr = PrintScanRangeLocations(execRequest.query_exec_request).toString();
}
}
} catch (ImpalaException e) {
if (e instanceof AnalysisException) {
errorLog.append("query:\n" + query + "\nanalysis error: " + e.getMessage() + "\n");
return;
} else if (e instanceof InternalException) {
errorLog.append("query:\n" + query + "\ninternal error: " + e.getMessage() + "\n");
return;
}
if (e instanceof NotImplementedException) {
handleNotImplException(query, expectedErrorMsg, errorLog, actualOutput, e);
} else if (e instanceof CatalogException) {
// TODO: do we need to rethrow?
throw (CatalogException) e;
} else {
errorLog.append("query:\n" + query + "\nunhandled exception: " + e.getMessage() + "\n");
}
}
// compare scan range locations
LOG.info("scan range locations: " + locationsStr);
ArrayList<String> expectedLocations = testCase.getSectionContents(Section.SCANRANGELOCATIONS);
if (expectedLocations.size() > 0 && locationsStr != null) {
// Locations' order does not matter.
String result =
TestUtils.compareOutput(
Lists.newArrayList(locationsStr.split("\n")), expectedLocations, false);
if (!result.isEmpty()) {
errorLog.append(
"section " + Section.SCANRANGELOCATIONS + " of query:\n" + query + "\n" + result);
}
actualOutput.append(Section.SCANRANGELOCATIONS.getHeader() + "\n");
// Print the locations out sorted since the order is random and messed up
// the diffs. The values in locationStr contains "Node X" labels as well
// as paths.
ArrayList<String> locations = Lists.newArrayList(locationsStr.split("\n"));
ArrayList<String> perNodeLocations = Lists.newArrayList();
for (int i = 0; i < locations.size(); ++i) {
if (locations.get(i).startsWith("NODE")) {
if (!perNodeLocations.isEmpty()) {
Collections.sort(perNodeLocations);
actualOutput.append(Joiner.on("\n").join(perNodeLocations)).append("\n");
perNodeLocations.clear();
}
actualOutput.append(locations.get(i)).append("\n");
} else {
perNodeLocations.add(locations.get(i));
}
}
if (!perNodeLocations.isEmpty()) {
Collections.sort(perNodeLocations);
actualOutput.append(Joiner.on("\n").join(perNodeLocations)).append("\n");
}
// TODO: check that scan range locations are identical in both cases
}
}