public void execQueryOnVirtualTable(
QueryContext queryContext,
Session session,
String query,
LogicalPlan plan,
SubmitQueryResponse.Builder response)
throws Exception {
int maxRow = Integer.MAX_VALUE;
if (plan.getRootBlock().hasNode(NodeType.LIMIT)) {
LimitNode limitNode = plan.getRootBlock().getNode(NodeType.LIMIT);
maxRow = (int) limitNode.getFetchFirstNum();
}
QueryInfo queryInfo =
context
.getQueryJobManager()
.createNewSimpleQuery(
queryContext, session, query, (LogicalRootNode) plan.getRootBlock().getRoot());
NonForwardQueryResultScanner queryResultScanner =
new NonForwardQueryResultSystemScanner(
context, plan, queryInfo.getQueryId(), session.getSessionId(), maxRow);
queryResultScanner.init();
session.addNonForwardQueryResultScanner(queryResultScanner);
response.setState(OK);
response.setQueryId(queryInfo.getQueryId().getProto());
response.setResultType(ResultType.ENCLOSED);
response.setMaxRowNum(maxRow);
response.setTableDesc(queryResultScanner.getTableDesc().getProto());
}
public void execExplain(
Session session,
String query,
LogicalPlan plan,
QueryContext queryContext,
boolean isGlobal,
SubmitQueryResponse.Builder response)
throws Exception {
String explainStr;
boolean isTest = queryContext.getBool(SessionVars.TEST_PLAN_SHAPE_FIX_ENABLED);
if (isTest) {
ExplainPlanPreprocessorForTest preprocessorForTest = new ExplainPlanPreprocessorForTest();
preprocessorForTest.prepareTest(plan);
}
if (isGlobal) {
GlobalPlanner planner = new GlobalPlanner(context.getConf(), context.getCatalog());
MasterPlan masterPlan = compileMasterPlan(plan, queryContext, planner);
if (isTest) {
ExplainGlobalPlanPreprocessorForTest globalPlanPreprocessorForTest =
new ExplainGlobalPlanPreprocessorForTest();
globalPlanPreprocessorForTest.prepareTest(masterPlan);
}
explainStr = masterPlan.toString();
} else {
explainStr = PlannerUtil.buildExplainString(plan.getRootBlock().getRoot());
}
Schema schema = new Schema();
schema.addColumn("explain", TajoDataTypes.Type.TEXT);
RowStoreUtil.RowStoreEncoder encoder = RowStoreUtil.createEncoder(schema);
SerializedResultSet.Builder serializedResBuilder = SerializedResultSet.newBuilder();
VTuple tuple = new VTuple(1);
String[] lines = explainStr.split("\n");
int bytesNum = 0;
for (String line : lines) {
tuple.put(0, DatumFactory.createText(line));
byte[] encodedData = encoder.toBytes(tuple);
bytesNum += encodedData.length;
serializedResBuilder.addSerializedTuples(ByteString.copyFrom(encodedData));
}
serializedResBuilder.setSchema(schema.getProto());
serializedResBuilder.setBytesNum(bytesNum);
QueryInfo queryInfo =
context
.getQueryJobManager()
.createNewSimpleQuery(
queryContext, session, query, (LogicalRootNode) plan.getRootBlock().getRoot());
response.setState(OK);
response.setQueryId(queryInfo.getQueryId().getProto());
response.setResultType(ResultType.ENCLOSED);
response.setResultSet(serializedResBuilder.build());
response.setMaxRowNum(lines.length);
}
public void execNonFromQuery(
QueryContext queryContext,
Session session,
String query,
LogicalPlan plan,
SubmitQueryResponse.Builder responseBuilder)
throws Exception {
LogicalRootNode rootNode = plan.getRootBlock().getRoot();
EvalContext evalContext = new EvalContext();
Target[] targets = plan.getRootBlock().getRawTargets();
if (targets == null) {
throw new TajoInternalError("no targets");
}
try {
// start script executor
startScriptExecutors(queryContext, evalContext, targets);
final VTuple outTuple = new VTuple(targets.length);
for (int i = 0; i < targets.length; i++) {
EvalNode eval = targets[i].getEvalTree();
eval.bind(evalContext, null);
outTuple.put(i, eval.eval(null));
}
boolean isInsert =
rootNode.getChild() != null && rootNode.getChild().getType() == NodeType.INSERT;
if (isInsert) {
InsertNode insertNode = rootNode.getChild();
insertRowValues(queryContext, insertNode, responseBuilder);
} else {
Schema schema = PlannerUtil.targetToSchema(targets);
RowStoreUtil.RowStoreEncoder encoder = RowStoreUtil.createEncoder(schema);
byte[] serializedBytes = encoder.toBytes(outTuple);
SerializedResultSet.Builder serializedResBuilder = SerializedResultSet.newBuilder();
serializedResBuilder.addSerializedTuples(ByteString.copyFrom(serializedBytes));
serializedResBuilder.setSchema(schema.getProto());
serializedResBuilder.setBytesNum(serializedBytes.length);
QueryInfo queryInfo =
context
.getQueryJobManager()
.createNewSimpleQuery(
queryContext, session, query, (LogicalRootNode) plan.getRootBlock().getRoot());
responseBuilder.setState(OK);
responseBuilder.setResultType(ResultType.ENCLOSED);
responseBuilder.setQueryId(queryInfo.getQueryId().getProto());
responseBuilder.setResultSet(serializedResBuilder);
responseBuilder.setMaxRowNum(1);
}
} finally {
// stop script executor
stopScriptExecutors(evalContext);
}
}
public void executeDistributedQuery(
QueryContext queryContext,
Session session,
LogicalPlan plan,
String sql,
String jsonExpr,
SubmitQueryResponse.Builder responseBuilder)
throws Exception {
LogicalRootNode rootNode = plan.getRootBlock().getRoot();
TableDesc tableDesc = PlannerUtil.getTableDesc(catalog, plan.getRootBlock().getRoot());
if (tableDesc != null) {
Tablespace space = TablespaceManager.get(tableDesc.getUri()).get();
FormatProperty formatProperty = space.getFormatProperty(tableDesc.getMeta());
if (!formatProperty.isInsertable()) {
throw new UnsupportedException(
String.format("INSERT operation on %s tablespace", tableDesc.getUri().toString()));
}
space.prepareTable(rootNode.getChild());
}
hookManager.doHooks(queryContext, plan);
QueryManager queryManager = this.context.getQueryJobManager();
QueryInfo queryInfo;
queryInfo = queryManager.scheduleQuery(session, queryContext, sql, jsonExpr, rootNode);
responseBuilder.setState(OK);
responseBuilder.setQueryId(queryInfo.getQueryId().getProto());
responseBuilder.setResultType(ResultType.FETCH);
if (queryInfo.getQueryMasterHost() != null) {
responseBuilder.setQueryMasterHost(queryInfo.getQueryMasterHost());
}
responseBuilder.setQueryMasterPort(queryInfo.getQueryMasterClientPort());
LOG.info(
"Query "
+ queryInfo.getQueryId().toString()
+ ","
+ queryInfo.getSql()
+ ","
+ " is forwarded to "
+ queryInfo.getQueryMasterHost()
+ ":"
+ queryInfo.getQueryMasterPort());
}
public void execSetSession(
Session session, LogicalPlan plan, SubmitQueryResponse.Builder response) {
SetSessionNode setSessionNode = ((LogicalRootNode) plan.getRootBlock().getRoot()).getChild();
final String varName = setSessionNode.getName();
// SET CATALOG 'XXX'
if (varName.equals(SessionVars.CURRENT_DATABASE.name())) {
String databaseName = setSessionNode.getValue();
if (catalog.existDatabase(databaseName)) {
session.selectDatabase(setSessionNode.getValue());
} else {
response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
response.setState(errUndefinedDatabase(databaseName));
}
// others
} else {
if (setSessionNode.hasValue()) {
session.setVariable(varName, setSessionNode.getValue());
} else {
session.removeVariable(varName);
}
}
response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
response.setState(OK);
}
public VerificationState verify(
long broadcastThresholdForCrossJoin, VerificationState state, LogicalPlan plan)
throws TajoException {
Context context = new Context(state, broadcastThresholdForCrossJoin);
visit(context, plan, plan.getRootBlock());
return context.state;
}
public SubmitQueryResponse execute(
QueryContext queryContext, Session session, String sql, String jsonExpr, LogicalPlan plan)
throws Exception {
SubmitQueryResponse.Builder response = SubmitQueryResponse.newBuilder();
response.setUserName(queryContext.get(SessionVars.USERNAME));
LogicalRootNode rootNode = plan.getRootBlock().getRoot();
if (PlannerUtil.checkIfSetSession(rootNode)) {
execSetSession(session, plan, response);
} else if (PlannerUtil.checkIfDDLPlan(rootNode)) {
if (PlannerUtil.isDistExecDDL(rootNode)) {
if (rootNode.getChild().getType() == NodeType.CREATE_INDEX) {
checkIndexExistence(queryContext, (CreateIndexNode) rootNode.getChild());
}
executeDistributedQuery(queryContext, session, plan, sql, jsonExpr, response);
} else {
ddlExecutor.execute(queryContext, plan);
response.setState(OK);
response.setResultType(ResultType.NO_RESULT);
}
} else if (plan.isExplain()) { // explain query
execExplain(session, sql, plan, queryContext, plan.isExplainGlobal(), response);
} else if (PlannerUtil.checkIfQueryTargetIsVirtualTable(plan)) {
execQueryOnVirtualTable(queryContext, session, sql, plan, response);
// Simple query indicates a form of 'select * from tb_name [LIMIT X];'.
} else if (PlannerUtil.checkIfSimpleQuery(plan)) {
execSimpleQuery(queryContext, session, sql, plan, response);
// NonFromQuery indicates a form of 'select a, x+y;'
} else if (PlannerUtil.checkIfNonFromQuery(plan)) {
execNonFromQuery(queryContext, session, sql, plan, response);
} else { // it requires distributed execution. So, the query is forwarded to a query master.
executeDistributedQuery(queryContext, session, plan, sql, jsonExpr, response);
}
response.setSessionVars(ProtoUtil.convertFromMap(session.getAllVariables()));
return response.build();
}
public void execSimpleQuery(
QueryContext queryContext,
Session session,
String query,
LogicalPlan plan,
SubmitQueryResponse.Builder response)
throws Exception {
ScanNode scanNode = plan.getRootBlock().getNode(NodeType.SCAN);
TableDesc desc = scanNode.getTableDesc();
if (desc.hasPartition()) {
scanNode = plan.getRootBlock().getNode(NodeType.PARTITIONS_SCAN);
}
int maxRow = Integer.MAX_VALUE;
if (plan.getRootBlock().hasNode(NodeType.LIMIT)) {
LimitNode limitNode = plan.getRootBlock().getNode(NodeType.LIMIT);
maxRow = (int) limitNode.getFetchFirstNum();
}
if (desc.getStats().getNumRows() == 0) {
desc.getStats().setNumRows(TajoConstants.UNKNOWN_ROW_NUMBER);
}
QueryInfo queryInfo =
context
.getQueryJobManager()
.createNewSimpleQuery(
queryContext, session, query, (LogicalRootNode) plan.getRootBlock().getRoot());
NonForwardQueryResultScanner queryResultScanner =
new NonForwardQueryResultFileScanner(
context.getConf(),
session.getSessionId(),
queryInfo.getQueryId(),
scanNode,
desc,
maxRow);
queryResultScanner.init();
session.addNonForwardQueryResultScanner(queryResultScanner);
response.setState(OK);
response.setQueryId(queryInfo.getQueryId().getProto());
response.setResultType(ResultType.ENCLOSED);
response.setMaxRowNum(maxRow);
response.setTableDesc(desc.getProto());
}
public MasterPlan compileMasterPlan(LogicalPlan plan, QueryContext context, GlobalPlanner planner)
throws Exception {
LogicalRootNode rootNode = plan.getRootBlock().getRoot();
TableDesc tableDesc = PlannerUtil.getTableDesc(planner.getCatalog(), rootNode.getChild());
if (tableDesc != null) {
Tablespace space = TablespaceManager.get(tableDesc.getUri()).get();
space.rewritePlan(context, plan);
}
MasterPlan masterPlan = new MasterPlan(QueryIdFactory.NULL_QUERY_ID, context, plan);
planner.build(context, masterPlan);
return masterPlan;
}