Example #1
0
 /**
  * Generate an array of random input parameters for a given Statement
  *
  * @param catalog_stmt
  * @return
  */
 protected Object[] makeRandomStatementParameters(Statement catalog_stmt) {
   Object params[] = new Object[catalog_stmt.getParameters().size()];
   for (StmtParameter catalog_param : catalog_stmt.getParameters()) {
     VoltType vtype = VoltType.get(catalog_param.getJavatype());
     params[catalog_param.getIndex()] = VoltTypeUtil.getRandomValue(vtype);
     LOG.debug(
         catalog_param.fullName() + " -> " + params[catalog_param.getIndex()] + " / " + vtype);
   } // FOR
   return (params);
 }
Example #2
0
 private void applyPrefetchableFlags(Database catalog_db) {
   for (Procedure catalog_proc : catalog_db.getProcedures()) {
     boolean proc_prefetchable = false;
     for (Statement statement : catalog_proc.getStatements()) {
       boolean stmt_prefetchable = true;
       for (StmtParameter stmtParam : statement.getParameters()) {
         if (stmtParam.getProcparameter() == null) {
           stmt_prefetchable = false;
           break;
         }
       } // FOR (StmtParameter)
       if (stmt_prefetchable) {
         statement.setPrefetchable(true);
         proc_prefetchable = true;
       }
     } // FOR (Statement)
     if (proc_prefetchable) {
       catalog_proc.setPrefetchable(true);
     }
   } // FOR (Procedure)
 }
Example #3
0
  public static void initSQLStmt(SQLStmt stmt, Statement catStmt) {
    stmt.catStmt = catStmt;
    stmt.numFragGUIDs = catStmt.getFragments().size();
    PlanFragment fragments[] = new PlanFragment[stmt.numFragGUIDs];
    stmt.fragGUIDs = new long[stmt.numFragGUIDs];
    int i = 0;
    for (PlanFragment frag : stmt.catStmt.getFragments()) {
      fragments[i] = frag;
      stmt.fragGUIDs[i] = CatalogUtil.getUniqueIdForFragment(frag);
      i++;
    }

    stmt.numStatementParamJavaTypes = stmt.catStmt.getParameters().size();
    // StmtParameter parameters[] = new StmtParameter[stmt.numStatementParamJavaTypes];
    stmt.statementParamJavaTypes = new byte[stmt.numStatementParamJavaTypes];
    for (StmtParameter param : stmt.catStmt.getParameters()) {
      // parameters[i] = param;
      stmt.statementParamJavaTypes[param.getIndex()] = (byte) param.getJavatype();
      i++;
    }
  }
Example #4
0
  static void compileSingleStmtProcedure(
      VoltCompiler compiler,
      HSQLInterface hsql,
      DatabaseEstimates estimates,
      Catalog catalog,
      Database db,
      ProcedureDescriptor procedureDescriptor)
      throws VoltCompiler.VoltCompilerException {
    final String className = procedureDescriptor.m_className;
    if (className.indexOf('@') != -1) {
      throw compiler.new VoltCompilerException("User procedure names can't contain \"@\".");
    }

    // get the short name of the class (no package if a user procedure)
    // use the Table.<builtin> name (allowing the period) if builtin.
    String shortName = className;
    if (procedureDescriptor.m_builtInStmt == false) {
      String[] parts = className.split("\\.");
      shortName = parts[parts.length - 1];
    }

    // add an entry to the catalog (using the full className)
    final Procedure procedure = db.getProcedures().add(shortName);
    for (String groupName : procedureDescriptor.m_authGroups) {
      final Group group = db.getGroups().get(groupName);
      if (group == null) {
        throw compiler
        .new VoltCompilerException(
            "Procedure "
                + className
                + " allows access by a role "
                + groupName
                + " that does not exist");
      }
      final GroupRef groupRef = procedure.getAuthgroups().add(groupName);
      groupRef.setGroup(group);
    }
    procedure.setClassname(className);
    // sysprocs don't use the procedure compiler
    procedure.setSystemproc(false);
    procedure.setDefaultproc(procedureDescriptor.m_builtInStmt);
    procedure.setHasjava(false);

    // get the annotation
    // first try to get one that has been passed from the compiler
    ProcInfoData info = compiler.getProcInfoOverride(shortName);
    // then check for the usual one in the class itself
    // and create a ProcInfo.Data instance for it
    if (info == null) {
      info = new ProcInfoData();
      if (procedureDescriptor.m_partitionString != null) {
        info.partitionInfo = procedureDescriptor.m_partitionString;
        info.singlePartition = true;
      }
    }
    assert (info != null);

    // ADD THE STATEMENT

    // add the statement to the catalog
    Statement catalogStmt = procedure.getStatements().add(VoltDB.ANON_STMT_NAME);

    // compile the statement
    StatementPartitioning partitioning =
        info.singlePartition ? StatementPartitioning.forceSP() : StatementPartitioning.forceMP();
    // default to FASTER detmode because stmt procs can't feed read output into writes
    StatementCompiler.compileFromSqlTextAndUpdateCatalog(
        compiler,
        hsql,
        catalog,
        db,
        estimates,
        catalogStmt,
        procedureDescriptor.m_singleStmt,
        procedureDescriptor.m_joinOrder,
        DeterminismMode.FASTER,
        partitioning);

    // if the single stmt is not read only, then the proc is not read only
    boolean procHasWriteStmts = (catalogStmt.getReadonly() == false);

    // set the read onlyness of a proc
    procedure.setReadonly(procHasWriteStmts == false);

    int seqs = catalogStmt.getSeqscancount();
    procedure.setHasseqscans(seqs > 0);

    // set procedure parameter types
    CatalogMap<ProcParameter> params = procedure.getParameters();
    CatalogMap<StmtParameter> stmtParams = catalogStmt.getParameters();

    // set the procedure parameter types from the statement parameter types
    int paramCount = 0;
    for (StmtParameter stmtParam : CatalogUtil.getSortedCatalogItems(stmtParams, "index")) {
      // name each parameter "param1", "param2", etc...
      ProcParameter procParam = params.add("param" + String.valueOf(paramCount));
      procParam.setIndex(stmtParam.getIndex());
      procParam.setIsarray(stmtParam.getIsarray());
      procParam.setType(stmtParam.getJavatype());
      paramCount++;
    }

    // parse the procinfo
    procedure.setSinglepartition(info.singlePartition);
    if (info.singlePartition) {
      parsePartitionInfo(compiler, db, procedure, info.partitionInfo);
      if (procedure.getPartitionparameter() >= params.size()) {
        String msg =
            "PartitionInfo parameter not a valid parameter for procedure: "
                + procedure.getClassname();
        throw compiler.new VoltCompilerException(msg);
      }
      // TODO: The planner does not currently validate that a single-statement plan declared as
      // single-partition correctly uses
      // the designated parameter as a partitioning filter, maybe some day.
      // In theory, the PartitioningForStatement would confirm the use of (only) a parameter as a
      // partition key --
      // or if the partition key was determined to be some other hard-coded constant (expression?)
      // it might display a warning
      // message that the passed parameter is assumed to be equal to that constant (expression).
    } else {
      if (partitioning.getCountOfIndependentlyPartitionedTables() == 1) {
        AbstractExpression statementPartitionExpression =
            partitioning.singlePartitioningExpressionForReport();
        if (statementPartitionExpression != null) {
          // The planner has uncovered an overlooked opportunity to run the statement SP.
          String msg = null;
          if (statementPartitionExpression instanceof ParameterValueExpression) {
            msg =
                "This procedure would benefit from setting the attribute 'partitioninfo="
                    + partitioning.getFullColumnName()
                    + ":"
                    + ((ParameterValueExpression) statementPartitionExpression).getParameterIndex()
                    + "'";
          } else {
            String valueDescription = null;
            Object partitionValue = partitioning.getInferredPartitioningValue();
            if (partitionValue == null) {
              // Statement partitioned on a runtime constant. This is likely to be cryptic, but
              // hopefully gets the idea across.
              valueDescription = "of " + statementPartitionExpression.explain("");
            } else {
              valueDescription =
                  partitionValue.toString(); // A simple constant value COULD have been a parameter.
            }
            msg =
                "This procedure would benefit from adding a parameter to be passed the value "
                    + valueDescription
                    + " and setting the attribute 'partitioninfo="
                    + partitioning.getFullColumnName()
                    + ":"
                    + paramCount
                    + "'";
          }
          compiler.addWarn(msg);
        }
      }
    }
  }
Example #5
0
  static String genrateStatementRow(Procedure procedure, Statement statement) {
    StringBuilder sb = new StringBuilder();
    sb.append("        <tr class='primaryrow2'>");

    // name column
    String anchor = (procedure.getTypeName() + "-" + statement.getTypeName()).toLowerCase();
    sb.append(
        "<td style='white-space: nowrap'><i id='p-"
            + anchor
            + "--icon' class='icon-chevron-right'></i> <a href='#' id='p-");
    sb.append(anchor).append("' class='togglex'>");
    sb.append(statement.getTypeName());
    sb.append("</a></td>");

    // sql column
    sb.append("<td><tt>");
    sb.append(statement.getSqltext());
    sb.append("</td></tt>");

    // params column
    sb.append("<td>");
    List<StmtParameter> params =
        CatalogUtil.getSortedCatalogItems(statement.getParameters(), "index");
    List<String> paramTypes = new ArrayList<String>();
    for (StmtParameter param : params) {
      paramTypes.add(VoltType.get((byte) param.getJavatype()).name());
    }
    if (paramTypes.size() == 0) {
      sb.append("<i>None</i>");
    }
    sb.append(StringUtils.join(paramTypes, ", "));
    sb.append("</td>");

    // r/w column
    sb.append("<td>");
    if (statement.getReadonly()) {
      tag(sb, "success", "Read");
    } else {
      tag(sb, "warning", "Write");
    }
    sb.append("</td>");

    // attributes
    sb.append("<td>");

    if (!statement.getIscontentdeterministic() || !statement.getIsorderdeterministic()) {
      tag(sb, "inverse", "Determinism");
    }

    if (statement.getSeqscancount() > 0) {
      tag(sb, "important", "Scans");
    }

    sb.append("</td>");

    sb.append("</tr>\n");

    // BUILD THE DROPDOWN FOR THE PLAN/DETAIL TABLE
    sb.append(
        "<tr class='dropdown2'><td colspan='5' id='p-"
            + procedure.getTypeName().toLowerCase()
            + "-"
            + statement.getTypeName().toLowerCase()
            + "--dropdown'>\n");

    sb.append("<div class='well well-small'><h4>Explain Plan:</h4>\n");
    StatementAnnotation annotation = (StatementAnnotation) statement.getAnnotation();
    if (annotation != null) {
      String plan = annotation.explainPlan;
      plan = plan.replace("\n", "<br/>");
      plan = plan.replace(" ", "&nbsp;");

      for (Table t : annotation.tablesRead) {
        String name = t.getTypeName().toUpperCase();
        String link = "\"<a href='#s-" + t.getTypeName() + "'>" + name + "</a>\"";
        plan = plan.replace("\"" + name + "\"", link);
      }
      for (Table t : annotation.tablesUpdated) {
        String name = t.getTypeName().toUpperCase();
        String link = "\"<a href='#s-" + t.getTypeName() + "'>" + name + "</a>\"";
        plan = plan.replace("\"" + name + "\"", link);
      }
      for (Index i : annotation.indexesUsed) {
        Table t = (Table) i.getParent();
        String name = i.getTypeName().toUpperCase();
        String link =
            "\"<a href='#s-" + t.getTypeName() + "-" + i.getTypeName() + "'>" + name + "</a>\"";
        plan = plan.replace("\"" + name + "\"", link);
      }

      sb.append("<tt>").append(plan).append("</tt>");
    } else {
      sb.append("<i>No SQL explain plan found.</i>\n");
    }
    sb.append("</div>\n");

    sb.append("</td></tr>\n");

    return sb.toString();
  }
  /**
   * Compile and cache the statement and plan and return the final plan graph.
   *
   * @param sql
   * @param paramCount
   */
  public List<AbstractPlanNode> compile(
      String sql,
      int paramCount,
      String joinOrder,
      Object partitionParameter,
      boolean inferSP,
      boolean lockInSP) {
    Statement catalogStmt = proc.getStatements().add("stmt-" + String.valueOf(compileCounter++));
    catalogStmt.setSqltext(sql);
    catalogStmt.setSinglepartition(partitionParameter != null);
    catalogStmt.setBatched(false);
    catalogStmt.setParamnum(paramCount);

    // determine the type of the query
    QueryType qtype = QueryType.SELECT;
    catalogStmt.setReadonly(true);
    if (sql.toLowerCase().startsWith("insert")) {
      qtype = QueryType.INSERT;
      catalogStmt.setReadonly(false);
    }
    if (sql.toLowerCase().startsWith("update")) {
      qtype = QueryType.UPDATE;
      catalogStmt.setReadonly(false);
    }
    if (sql.toLowerCase().startsWith("delete")) {
      qtype = QueryType.DELETE;
      catalogStmt.setReadonly(false);
    }
    catalogStmt.setQuerytype(qtype.getValue());
    // name will look like "basename-stmt-#"
    String name = catalogStmt.getParent().getTypeName() + "-" + catalogStmt.getTypeName();

    DatabaseEstimates estimates = new DatabaseEstimates();
    TrivialCostModel costModel = new TrivialCostModel();
    PartitioningForStatement partitioning =
        new PartitioningForStatement(partitionParameter, inferSP, lockInSP);
    QueryPlanner planner =
        new QueryPlanner(
            catalogStmt.getSqltext(),
            catalogStmt.getTypeName(),
            catalogStmt.getParent().getTypeName(),
            catalog.getClusters().get("cluster"),
            db,
            partitioning,
            hsql,
            estimates,
            false,
            StatementCompiler.DEFAULT_MAX_JOIN_TABLES,
            costModel,
            null,
            joinOrder);

    CompiledPlan plan = null;
    planner.parse();
    plan = planner.plan();
    assert (plan != null);

    // Input Parameters
    // We will need to update the system catalogs with this new information
    // If this is an adhoc query then there won't be any parameters
    for (int i = 0; i < plan.parameters.length; ++i) {
      StmtParameter catalogParam = catalogStmt.getParameters().add(String.valueOf(i));
      catalogParam.setJavatype(plan.parameters[i].getValue());
      catalogParam.setIndex(i);
    }

    // Output Columns
    int index = 0;
    for (SchemaColumn col : plan.columns.getColumns()) {
      Column catColumn = catalogStmt.getOutput_columns().add(String.valueOf(index));
      catColumn.setNullable(false);
      catColumn.setIndex(index);
      catColumn.setName(col.getColumnName());
      catColumn.setType(col.getType().getValue());
      catColumn.setSize(col.getSize());
      index++;
    }

    List<PlanNodeList> nodeLists = new ArrayList<PlanNodeList>();
    nodeLists.add(new PlanNodeList(plan.rootPlanGraph));
    if (plan.subPlanGraph != null) {
      nodeLists.add(new PlanNodeList(plan.subPlanGraph));
    }

    // Store the list of parameters types and indexes in the plan node list.
    List<Pair<Integer, VoltType>> parameters = nodeLists.get(0).getParameters();
    for (int i = 0; i < plan.parameters.length; ++i) {
      Pair<Integer, VoltType> parameter = new Pair<Integer, VoltType>(i, plan.parameters[i]);
      parameters.add(parameter);
    }

    // Now update our catalog information
    // HACK: We're using the node_tree's hashCode() as it's name. It would be really
    //     nice if the Catalog code give us an guid without needing a name first...

    String json = null;
    try {
      JSONObject jobj = new JSONObject(nodeLists.get(0).toJSONString());
      json = jobj.toString(4);
    } catch (JSONException e2) {
      // TODO Auto-generated catch block
      e2.printStackTrace();
      System.exit(-1);
    }

    //
    // We then stick a serialized version of PlanNodeTree into a PlanFragment
    //
    try {
      BuildDirectoryUtils.writeFile("statement-plans", name + "_json.txt", json);
      BuildDirectoryUtils.writeFile(
          "statement-plans", name + ".dot", nodeLists.get(0).toDOTString("name"));
    } catch (Exception e) {
      e.printStackTrace();
    }

    List<AbstractPlanNode> plannodes = new ArrayList<AbstractPlanNode>();
    for (PlanNodeList nodeList : nodeLists) {
      plannodes.add(nodeList.getRootPlanNode());
    }

    m_currentPlan = plan;
    return plannodes;
  }