Example #1
0
  public void execQueryOnVirtualTable(
      QueryContext queryContext,
      Session session,
      String query,
      LogicalPlan plan,
      SubmitQueryResponse.Builder response)
      throws Exception {
    int maxRow = Integer.MAX_VALUE;
    if (plan.getRootBlock().hasNode(NodeType.LIMIT)) {
      LimitNode limitNode = plan.getRootBlock().getNode(NodeType.LIMIT);
      maxRow = (int) limitNode.getFetchFirstNum();
    }
    QueryInfo queryInfo =
        context
            .getQueryJobManager()
            .createNewSimpleQuery(
                queryContext, session, query, (LogicalRootNode) plan.getRootBlock().getRoot());

    NonForwardQueryResultScanner queryResultScanner =
        new NonForwardQueryResultSystemScanner(
            context, plan, queryInfo.getQueryId(), session.getSessionId(), maxRow);

    queryResultScanner.init();
    session.addNonForwardQueryResultScanner(queryResultScanner);

    response.setState(OK);
    response.setQueryId(queryInfo.getQueryId().getProto());
    response.setResultType(ResultType.ENCLOSED);
    response.setMaxRowNum(maxRow);
    response.setTableDesc(queryResultScanner.getTableDesc().getProto());
  }
Example #2
0
  public void execExplain(
      Session session,
      String query,
      LogicalPlan plan,
      QueryContext queryContext,
      boolean isGlobal,
      SubmitQueryResponse.Builder response)
      throws Exception {

    String explainStr;
    boolean isTest = queryContext.getBool(SessionVars.TEST_PLAN_SHAPE_FIX_ENABLED);
    if (isTest) {
      ExplainPlanPreprocessorForTest preprocessorForTest = new ExplainPlanPreprocessorForTest();
      preprocessorForTest.prepareTest(plan);
    }

    if (isGlobal) {
      GlobalPlanner planner = new GlobalPlanner(context.getConf(), context.getCatalog());
      MasterPlan masterPlan = compileMasterPlan(plan, queryContext, planner);
      if (isTest) {
        ExplainGlobalPlanPreprocessorForTest globalPlanPreprocessorForTest =
            new ExplainGlobalPlanPreprocessorForTest();
        globalPlanPreprocessorForTest.prepareTest(masterPlan);
      }
      explainStr = masterPlan.toString();
    } else {
      explainStr = PlannerUtil.buildExplainString(plan.getRootBlock().getRoot());
    }

    Schema schema = new Schema();
    schema.addColumn("explain", TajoDataTypes.Type.TEXT);
    RowStoreUtil.RowStoreEncoder encoder = RowStoreUtil.createEncoder(schema);

    SerializedResultSet.Builder serializedResBuilder = SerializedResultSet.newBuilder();

    VTuple tuple = new VTuple(1);
    String[] lines = explainStr.split("\n");
    int bytesNum = 0;
    for (String line : lines) {
      tuple.put(0, DatumFactory.createText(line));
      byte[] encodedData = encoder.toBytes(tuple);
      bytesNum += encodedData.length;
      serializedResBuilder.addSerializedTuples(ByteString.copyFrom(encodedData));
    }
    serializedResBuilder.setSchema(schema.getProto());
    serializedResBuilder.setBytesNum(bytesNum);

    QueryInfo queryInfo =
        context
            .getQueryJobManager()
            .createNewSimpleQuery(
                queryContext, session, query, (LogicalRootNode) plan.getRootBlock().getRoot());

    response.setState(OK);
    response.setQueryId(queryInfo.getQueryId().getProto());
    response.setResultType(ResultType.ENCLOSED);
    response.setResultSet(serializedResBuilder.build());
    response.setMaxRowNum(lines.length);
  }
Example #3
0
  public void execNonFromQuery(
      QueryContext queryContext,
      Session session,
      String query,
      LogicalPlan plan,
      SubmitQueryResponse.Builder responseBuilder)
      throws Exception {
    LogicalRootNode rootNode = plan.getRootBlock().getRoot();

    EvalContext evalContext = new EvalContext();
    Target[] targets = plan.getRootBlock().getRawTargets();
    if (targets == null) {
      throw new TajoInternalError("no targets");
    }
    try {
      // start script executor
      startScriptExecutors(queryContext, evalContext, targets);
      final VTuple outTuple = new VTuple(targets.length);
      for (int i = 0; i < targets.length; i++) {
        EvalNode eval = targets[i].getEvalTree();
        eval.bind(evalContext, null);
        outTuple.put(i, eval.eval(null));
      }
      boolean isInsert =
          rootNode.getChild() != null && rootNode.getChild().getType() == NodeType.INSERT;
      if (isInsert) {
        InsertNode insertNode = rootNode.getChild();
        insertRowValues(queryContext, insertNode, responseBuilder);
      } else {
        Schema schema = PlannerUtil.targetToSchema(targets);
        RowStoreUtil.RowStoreEncoder encoder = RowStoreUtil.createEncoder(schema);
        byte[] serializedBytes = encoder.toBytes(outTuple);
        SerializedResultSet.Builder serializedResBuilder = SerializedResultSet.newBuilder();
        serializedResBuilder.addSerializedTuples(ByteString.copyFrom(serializedBytes));
        serializedResBuilder.setSchema(schema.getProto());
        serializedResBuilder.setBytesNum(serializedBytes.length);

        QueryInfo queryInfo =
            context
                .getQueryJobManager()
                .createNewSimpleQuery(
                    queryContext, session, query, (LogicalRootNode) plan.getRootBlock().getRoot());

        responseBuilder.setState(OK);
        responseBuilder.setResultType(ResultType.ENCLOSED);
        responseBuilder.setQueryId(queryInfo.getQueryId().getProto());
        responseBuilder.setResultSet(serializedResBuilder);
        responseBuilder.setMaxRowNum(1);
      }
    } finally {
      // stop script executor
      stopScriptExecutors(evalContext);
    }
  }
Example #4
0
  public void executeDistributedQuery(
      QueryContext queryContext,
      Session session,
      LogicalPlan plan,
      String sql,
      String jsonExpr,
      SubmitQueryResponse.Builder responseBuilder)
      throws Exception {
    LogicalRootNode rootNode = plan.getRootBlock().getRoot();

    TableDesc tableDesc = PlannerUtil.getTableDesc(catalog, plan.getRootBlock().getRoot());
    if (tableDesc != null) {

      Tablespace space = TablespaceManager.get(tableDesc.getUri()).get();
      FormatProperty formatProperty = space.getFormatProperty(tableDesc.getMeta());

      if (!formatProperty.isInsertable()) {
        throw new UnsupportedException(
            String.format("INSERT operation on %s tablespace", tableDesc.getUri().toString()));
      }

      space.prepareTable(rootNode.getChild());
    }

    hookManager.doHooks(queryContext, plan);

    QueryManager queryManager = this.context.getQueryJobManager();
    QueryInfo queryInfo;

    queryInfo = queryManager.scheduleQuery(session, queryContext, sql, jsonExpr, rootNode);

    responseBuilder.setState(OK);
    responseBuilder.setQueryId(queryInfo.getQueryId().getProto());
    responseBuilder.setResultType(ResultType.FETCH);
    if (queryInfo.getQueryMasterHost() != null) {
      responseBuilder.setQueryMasterHost(queryInfo.getQueryMasterHost());
    }
    responseBuilder.setQueryMasterPort(queryInfo.getQueryMasterClientPort());
    LOG.info(
        "Query "
            + queryInfo.getQueryId().toString()
            + ","
            + queryInfo.getSql()
            + ","
            + " is forwarded to "
            + queryInfo.getQueryMasterHost()
            + ":"
            + queryInfo.getQueryMasterPort());
  }
Example #5
0
  public void execSetSession(
      Session session, LogicalPlan plan, SubmitQueryResponse.Builder response) {
    SetSessionNode setSessionNode = ((LogicalRootNode) plan.getRootBlock().getRoot()).getChild();

    final String varName = setSessionNode.getName();

    // SET CATALOG 'XXX'
    if (varName.equals(SessionVars.CURRENT_DATABASE.name())) {
      String databaseName = setSessionNode.getValue();

      if (catalog.existDatabase(databaseName)) {
        session.selectDatabase(setSessionNode.getValue());
      } else {
        response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
        response.setState(errUndefinedDatabase(databaseName));
      }

      // others
    } else {
      if (setSessionNode.hasValue()) {
        session.setVariable(varName, setSessionNode.getValue());
      } else {
        session.removeVariable(varName);
      }
    }

    response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto());
    response.setState(OK);
  }
 public VerificationState verify(
     long broadcastThresholdForCrossJoin, VerificationState state, LogicalPlan plan)
     throws TajoException {
   Context context = new Context(state, broadcastThresholdForCrossJoin);
   visit(context, plan, plan.getRootBlock());
   return context.state;
 }
Example #7
0
  public SubmitQueryResponse execute(
      QueryContext queryContext, Session session, String sql, String jsonExpr, LogicalPlan plan)
      throws Exception {

    SubmitQueryResponse.Builder response = SubmitQueryResponse.newBuilder();
    response.setUserName(queryContext.get(SessionVars.USERNAME));

    LogicalRootNode rootNode = plan.getRootBlock().getRoot();

    if (PlannerUtil.checkIfSetSession(rootNode)) {
      execSetSession(session, plan, response);

    } else if (PlannerUtil.checkIfDDLPlan(rootNode)) {

      if (PlannerUtil.isDistExecDDL(rootNode)) {
        if (rootNode.getChild().getType() == NodeType.CREATE_INDEX) {
          checkIndexExistence(queryContext, (CreateIndexNode) rootNode.getChild());
        }
        executeDistributedQuery(queryContext, session, plan, sql, jsonExpr, response);
      } else {
        ddlExecutor.execute(queryContext, plan);
        response.setState(OK);
        response.setResultType(ResultType.NO_RESULT);
      }

    } else if (plan.isExplain()) { // explain query
      execExplain(session, sql, plan, queryContext, plan.isExplainGlobal(), response);

    } else if (PlannerUtil.checkIfQueryTargetIsVirtualTable(plan)) {
      execQueryOnVirtualTable(queryContext, session, sql, plan, response);

      // Simple query indicates a form of 'select * from tb_name [LIMIT X];'.
    } else if (PlannerUtil.checkIfSimpleQuery(plan)) {
      execSimpleQuery(queryContext, session, sql, plan, response);

      // NonFromQuery indicates a form of 'select a, x+y;'
    } else if (PlannerUtil.checkIfNonFromQuery(plan)) {
      execNonFromQuery(queryContext, session, sql, plan, response);

    } else { // it requires distributed execution. So, the query is forwarded to a query master.
      executeDistributedQuery(queryContext, session, plan, sql, jsonExpr, response);
    }

    response.setSessionVars(ProtoUtil.convertFromMap(session.getAllVariables()));

    return response.build();
  }
Example #8
0
  public void execSimpleQuery(
      QueryContext queryContext,
      Session session,
      String query,
      LogicalPlan plan,
      SubmitQueryResponse.Builder response)
      throws Exception {
    ScanNode scanNode = plan.getRootBlock().getNode(NodeType.SCAN);
    TableDesc desc = scanNode.getTableDesc();

    if (desc.hasPartition()) {
      scanNode = plan.getRootBlock().getNode(NodeType.PARTITIONS_SCAN);
    }

    int maxRow = Integer.MAX_VALUE;
    if (plan.getRootBlock().hasNode(NodeType.LIMIT)) {
      LimitNode limitNode = plan.getRootBlock().getNode(NodeType.LIMIT);
      maxRow = (int) limitNode.getFetchFirstNum();
    }
    if (desc.getStats().getNumRows() == 0) {
      desc.getStats().setNumRows(TajoConstants.UNKNOWN_ROW_NUMBER);
    }

    QueryInfo queryInfo =
        context
            .getQueryJobManager()
            .createNewSimpleQuery(
                queryContext, session, query, (LogicalRootNode) plan.getRootBlock().getRoot());

    NonForwardQueryResultScanner queryResultScanner =
        new NonForwardQueryResultFileScanner(
            context.getConf(),
            session.getSessionId(),
            queryInfo.getQueryId(),
            scanNode,
            desc,
            maxRow);

    queryResultScanner.init();
    session.addNonForwardQueryResultScanner(queryResultScanner);

    response.setState(OK);
    response.setQueryId(queryInfo.getQueryId().getProto());
    response.setResultType(ResultType.ENCLOSED);
    response.setMaxRowNum(maxRow);
    response.setTableDesc(desc.getProto());
  }
Example #9
0
  public MasterPlan compileMasterPlan(LogicalPlan plan, QueryContext context, GlobalPlanner planner)
      throws Exception {

    LogicalRootNode rootNode = plan.getRootBlock().getRoot();
    TableDesc tableDesc = PlannerUtil.getTableDesc(planner.getCatalog(), rootNode.getChild());

    if (tableDesc != null) {
      Tablespace space = TablespaceManager.get(tableDesc.getUri()).get();
      space.rewritePlan(context, plan);
    }

    MasterPlan masterPlan = new MasterPlan(QueryIdFactory.NULL_QUERY_ID, context, plan);
    planner.build(context, masterPlan);

    return masterPlan;
  }