@VisibleForTesting
 protected static AccessURI extractPartition(ASTNode ast) throws SemanticException {
   for (int i = 0; i < ast.getChildCount(); i++) {
     ASTNode child = (ASTNode) ast.getChild(i);
     if (child.getToken().getType() == HiveParser.TOK_PARTITIONLOCATION
         && child.getChildCount() == 1) {
       return parseURI(BaseSemanticAnalyzer.unescapeSQLString(child.getChild(0).getText()));
     }
   }
   return null;
 }
Example #2
0
  static ASTNode replaceAliases(ASTNode node, int nodePos, Map<String, String> colToTableAlias) {
    if (node == null) {
      return node;
    }

    int nodeType = node.getToken().getType();
    if (nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT) {
      String colName = HQLParser.getColName(node);
      String newAlias = colToTableAlias.get(colName.toLowerCase());

      if (StringUtils.isBlank(newAlias)) {
        return node;
      }

      if (nodeType == HiveParser.DOT) {
        // No need to create a new node, just replace the table name ident
        ASTNode aliasNode = (ASTNode) node.getChild(0);
        ASTNode newAliasIdent = new ASTNode(new CommonToken(HiveParser.Identifier, newAlias));
        aliasNode.setChild(0, newAliasIdent);
      } else {
        // Just a column ref, we need to make it alias.col
        // '.' will become the parent node
        ASTNode dot = new ASTNode(new CommonToken(HiveParser.DOT, "."));
        ASTNode aliasIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, newAlias));
        ASTNode tabRefNode =
            new ASTNode(new CommonToken(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL"));

        tabRefNode.addChild(aliasIdentNode);
        dot.addChild(tabRefNode);

        ASTNode colIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, colName));
        dot.addChild(colIdentNode);

        ASTNode parent = (ASTNode) node.getParent();
        if (parent != null) {
          parent.setChild(nodePos, dot);
        } else {
          return dot;
        }
      }
    } else {
      // recurse down
      for (int i = 0; i < node.getChildCount(); i++) {
        ASTNode child = (ASTNode) node.getChild(i);
        replaceAliases(child, i, colToTableAlias);
      }
    }
    return node;
  }
Example #3
0
  static void updateAliasMap(ASTNode root, CubeQueryContext cubeql) {
    if (root == null) {
      return;
    }

    if (root.getToken().getType() == TOK_SELEXPR) {
      ASTNode alias = HQLParser.findNodeByPath(root, Identifier);
      if (alias != null) {
        cubeql.addExprToAlias(root, alias);
      }
    } else {
      for (int i = 0; i < root.getChildCount(); i++) {
        updateAliasMap((ASTNode) root.getChild(i), cubeql);
      }
    }
  }
Example #4
0
  boolean isCubeMeasure(ASTNode node) {
    String tabname = null;
    String colname;
    int nodeType = node.getToken().getType();
    if (!(nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT)) {
      return false;
    }

    if (nodeType == HiveParser.TOK_TABLE_OR_COL) {
      colname = ((ASTNode) node.getChild(0)).getText();
    } else {
      // node in 'alias.column' format
      ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier);
      ASTNode colIdent = (ASTNode) node.getChild(1);

      colname = colIdent.getText();
      tabname = tabident.getText();
    }

    String msrname = StringUtils.isBlank(tabname) ? colname : tabname + "." + colname;

    return isCubeMeasure(msrname);
  }
    private void resolveClause(CubeQueryContext query, ASTNode node) throws LensException {
      if (node == null) {
        return;
      }

      int nodeType = node.getToken().getType();
      if (nodeType == HiveParser.DOT) {
        String colName = HQLParser.getColName(node).toLowerCase();
        if (!pickedReferences.containsKey(colName)) {
          return;
        }
        // No need to create a new node,
        // replace the table name identifier and column name identifier
        ASTNode tableNode = (ASTNode) node.getChild(0);
        ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier);

        PickedReference refered = getPickedReference(colName, tabident.getText().toLowerCase());
        if (refered == null) {
          return;
        }
        ASTNode newTableNode =
            new ASTNode(
                new CommonToken(HiveParser.Identifier, refered.getChainRef().getChainName()));
        tableNode.setChild(0, newTableNode);

        ASTNode newColumnNode =
            new ASTNode(
                new CommonToken(HiveParser.Identifier, refered.getChainRef().getRefColumn()));
        node.setChild(1, newColumnNode);
      } else {
        // recurse down
        for (int i = 0; i < node.getChildCount(); i++) {
          ASTNode child = (ASTNode) node.getChild(i);
          resolveClause(query, child);
        }
      }
    }
  /**
   * Pre-analyze hook called after compilation and before semantic analysis We extract things for to
   * Database and metadata level operations which are not capture in the input/output entities
   * during semantic analysis. Ideally it should be handled in Hive. We need to move most of these
   * into hive semantic analyzer and then remove it from the access hook.
   */
  @Override
  public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, ASTNode ast)
      throws SemanticException {

    switch (ast.getToken().getType()) {
        // Hive parser doesn't capture the database name in output entity, so we store it here for
        // now
      case HiveParser.TOK_CREATEDATABASE:
      case HiveParser.TOK_ALTERDATABASE_PROPERTIES:
      case HiveParser.TOK_DROPDATABASE:
      case HiveParser.TOK_SWITCHDATABASE:
      case HiveParser.TOK_DESCDATABASE:
        currDB = new Database(BaseSemanticAnalyzer.unescapeIdentifier(ast.getChild(0).getText()));
        break;
      case HiveParser.TOK_CREATETABLE:
      case HiveParser.TOK_CREATEVIEW:
        /*
         * Compiler doesn't create read/write entities for create table.
         * Hence we need extract dbname from db.tab format, if applicable
         */
        currDB = extractDatabase((ASTNode) ast.getChild(0));
        break;
      case HiveParser.TOK_DROPTABLE:
      case HiveParser.TOK_DROPVIEW:
      case HiveParser.TOK_SHOW_CREATETABLE:
      case HiveParser.TOK_ALTERTABLE_SERIALIZER:
      case HiveParser.TOK_ALTERVIEW_ADDPARTS:
      case HiveParser.TOK_ALTERVIEW_DROPPARTS:
      case HiveParser.TOK_ALTERVIEW_PROPERTIES:
      case HiveParser.TOK_ALTERVIEW_RENAME:
      case HiveParser.TOK_CREATEINDEX:
      case HiveParser.TOK_DROPINDEX:
      case HiveParser.TOK_LOCKTABLE:
      case HiveParser.TOK_UNLOCKTABLE:
        currTab = extractTable((ASTNode) ast.getFirstChildWithType(HiveParser.TOK_TABNAME));
        currDB = extractDatabase((ASTNode) ast.getChild(0));
        break;
      case HiveParser.TOK_ALTERINDEX_REBUILD:
        currTab = extractTable((ASTNode) ast.getChild(0)); // type is not TOK_TABNAME
        currDB = extractDatabase((ASTNode) ast.getChild(0));
        break;
      case HiveParser.TOK_SHOW_TABLESTATUS:
        currDB = extractDatabase((ASTNode) ast.getChild(0));
        int children = ast.getChildCount();
        for (int i = 1; i < children; i++) {
          ASTNode child = (ASTNode) ast.getChild(i);
          if (child.getToken().getType() == HiveParser.Identifier) {
            currDB = new Database(child.getText());
            break;
          }
        }
        // loosing the requested privileges for possible wildcard tables, since
        // further authorization will be done at the filter step and those unwanted will
        // eventually be filtered out from the output
        currTab = Table.ALL;
        break;
      case HiveParser.TOK_ALTERTABLE_RENAME:
      case HiveParser.TOK_ALTERTABLE_PROPERTIES:
      case HiveParser.TOK_ALTERTABLE_DROPPARTS:
      case HiveParser.TOK_ALTERTABLE_RENAMECOL:
      case HiveParser.TOK_ALTERTABLE_ADDCOLS:
      case HiveParser.TOK_ALTERTABLE_REPLACECOLS:
      case HiveParser.TOK_SHOW_TBLPROPERTIES:
      case HiveParser.TOK_SHOWINDEXES:
      case HiveParser.TOK_SHOWPARTITIONS:
        // token name TOK_TABNAME is not properly set in this case
        currTab = extractTable((ASTNode) ast.getChild(0));
        currDB = extractDatabase((ASTNode) ast.getChild(0));
        break;
      case HiveParser.TOK_MSCK:
        // token name TOK_TABNAME is not properly set in this case and child(0) does
        // not contain the table name.
        // TODO: Fix Hive to capture the table and DB name
        currOutTab = extractTable((ASTNode) ast.getChild(1));
        currOutDB = extractDatabase((ASTNode) ast.getChild(0));
        break;
      case HiveParser.TOK_ALTERTABLE_ADDPARTS:
        /*
         * Compiler doesn't create read/write entities for create table.
         * Hence we need extract dbname from db.tab format, if applicable
         */
        currTab = extractTable((ASTNode) ast.getChild(0));
        currDB = extractDatabase((ASTNode) ast.getChild(0));
        partitionURI = extractPartition(ast);
        break;
      case HiveParser.TOK_CREATEFUNCTION:
        String udfClassName = BaseSemanticAnalyzer.unescapeSQLString(ast.getChild(1).getText());
        try {
          CodeSource udfSrc =
              Class.forName(udfClassName, true, Utilities.getSessionSpecifiedClassLoader())
                  .getProtectionDomain()
                  .getCodeSource();
          if (udfSrc == null) {
            throw new SemanticException("Could not resolve the jar for UDF class " + udfClassName);
          }
          String udfJar = udfSrc.getLocation().getPath();
          if (udfJar == null || udfJar.isEmpty()) {
            throw new SemanticException(
                "Could not find the jar for UDF class " + udfClassName + "to validate privileges");
          }
          udfURI = parseURI(udfSrc.getLocation().toString(), true);
        } catch (ClassNotFoundException e) {
          throw new SemanticException("Error retrieving udf class:" + e.getMessage(), e);
        }
        // create/drop function is allowed with any database
        currDB = Database.ALL;
        break;
      case HiveParser.TOK_DROPFUNCTION:
        // create/drop function is allowed with any database
        currDB = Database.ALL;
        break;

      case HiveParser.TOK_LOAD:
        String dbName =
            BaseSemanticAnalyzer.unescapeIdentifier(
                ast.getChild(1).getChild(0).getChild(0).getText());
        currDB = new Database(dbName);
        break;
      case HiveParser.TOK_DESCTABLE:
        currDB = getCanonicalDb();
        // For DESCRIBE FORMATTED/EXTENDED ast will have an additional child node with value
        // "FORMATTED/EXTENDED".
        isDescTableBasic = (ast.getChildCount() == 1);
        break;
      case HiveParser.TOK_TRUNCATETABLE:
        // SENTRY-826:
        // Truncate empty partitioned table should throw SemanticException only if the
        // user does not have permission.
        // In postAnalyze, currOutDB and currOutTbl will be added into outputHierarchy
        // which will be validated in the hiveAuthzBinding.authorize method.
        Preconditions.checkArgument(ast.getChildCount() == 1);
        // childcount is 1 for table without partition, 2 for table with partitions
        Preconditions.checkArgument(ast.getChild(0).getChildCount() >= 1);
        Preconditions.checkArgument(ast.getChild(0).getChild(0).getChildCount() == 1);
        currOutDB = extractDatabase((ASTNode) ast.getChild(0));
        currOutTab = extractTable((ASTNode) ast.getChild(0).getChild(0).getChild(0));
        break;
      default:
        currDB = getCanonicalDb();
        break;
    }
    return ast;
  }
Example #7
0
  @Override
  public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, ASTNode ast)
      throws SemanticException {

    this.ast = ast;
    switch (ast.getToken().getType()) {

        // HCat wants to intercept following tokens and special-handle them.
      case HiveParser.TOK_CREATETABLE:
        hook = new CreateTableHook();
        return hook.preAnalyze(context, ast);

      case HiveParser.TOK_CREATEDATABASE:
        hook = new CreateDatabaseHook();
        return hook.preAnalyze(context, ast);

      case HiveParser.TOK_ALTERTABLE_PARTITION:
        if (((ASTNode) ast.getChild(1)).getToken().getType()
            == HiveParser.TOK_ALTERTABLE_FILEFORMAT) {
          return ast;
        } else if (((ASTNode) ast.getChild(1)).getToken().getType()
            == HiveParser.TOK_ALTERTABLE_MERGEFILES) {
          // unsupported
          throw new SemanticException("Operation not supported.");
        } else {
          return ast;
        }

        // HCat will allow these operations to be performed.
        // Database DDL
      case HiveParser.TOK_SHOWDATABASES:
      case HiveParser.TOK_DROPDATABASE:
      case HiveParser.TOK_SWITCHDATABASE:
      case HiveParser.TOK_DESCDATABASE:
      case HiveParser.TOK_ALTERDATABASE_PROPERTIES:

        // Index DDL
      case HiveParser.TOK_ALTERINDEX_PROPERTIES:
      case HiveParser.TOK_CREATEINDEX:
      case HiveParser.TOK_DROPINDEX:
      case HiveParser.TOK_SHOWINDEXES:

        // View DDL
        // "alter view add partition" does not work because of the nature of implementation
        // of the DDL in hive. Hive will internally invoke another Driver on the select statement,
        // and HCat does not let "select" statement through. I cannot find a way to get around it
        // without modifying hive code. So just leave it unsupported.
        // case HiveParser.TOK_ALTERVIEW_ADDPARTS:
      case HiveParser.TOK_ALTERVIEW_DROPPARTS:
      case HiveParser.TOK_ALTERVIEW_PROPERTIES:
      case HiveParser.TOK_ALTERVIEW_RENAME:
      case HiveParser.TOK_CREATEVIEW:
      case HiveParser.TOK_DROPVIEW:

        // Authorization DDL
      case HiveParser.TOK_CREATEROLE:
      case HiveParser.TOK_DROPROLE:
      case HiveParser.TOK_GRANT_ROLE:
      case HiveParser.TOK_GRANT_WITH_OPTION:
      case HiveParser.TOK_GRANT:
      case HiveParser.TOK_REVOKE_ROLE:
      case HiveParser.TOK_REVOKE:
      case HiveParser.TOK_SHOW_GRANT:
      case HiveParser.TOK_SHOW_ROLE_GRANT:

        // Misc DDL
      case HiveParser.TOK_LOCKTABLE:
      case HiveParser.TOK_UNLOCKTABLE:
      case HiveParser.TOK_SHOWLOCKS:
      case HiveParser.TOK_DESCFUNCTION:
      case HiveParser.TOK_SHOWFUNCTIONS:
      case HiveParser.TOK_EXPLAIN:

        // Table DDL
      case HiveParser.TOK_ALTERTABLE_ADDPARTS:
      case HiveParser.TOK_ALTERTABLE_ADDCOLS:
      case HiveParser.TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION:
      case HiveParser.TOK_ALTERTABLE_SERDEPROPERTIES:
      case HiveParser.TOK_ALTERTABLE_CLUSTER_SORT:
      case HiveParser.TOK_ALTERTABLE_DROPPARTS:
      case HiveParser.TOK_ALTERTABLE_PROPERTIES:
      case HiveParser.TOK_ALTERTABLE_RENAME:
      case HiveParser.TOK_ALTERTABLE_RENAMECOL:
      case HiveParser.TOK_ALTERTABLE_REPLACECOLS:
      case HiveParser.TOK_ALTERTABLE_SERIALIZER:
      case HiveParser.TOK_ALTERTABLE_TOUCH:
      case HiveParser.TOK_DESCTABLE:
      case HiveParser.TOK_DROPTABLE:
      case HiveParser.TOK_SHOW_TABLESTATUS:
      case HiveParser.TOK_SHOWPARTITIONS:
      case HiveParser.TOK_SHOWTABLES:
        return ast;

        // In all other cases, throw an exception. Its a white-list of allowed operations.
      default:
        throw new SemanticException("Operation not supported.");
    }
  }
Example #8
0
  @Override
  public void postAnalyze(
      HiveSemanticAnalyzerHookContext context, List<Task<? extends Serializable>> rootTasks)
      throws SemanticException {

    try {

      switch (ast.getToken().getType()) {
        case HiveParser.TOK_CREATETABLE:
        case HiveParser.TOK_CREATEDATABASE:
        case HiveParser.TOK_ALTERTABLE_PARTITION:

          // HCat will allow these operations to be performed.
          // Database DDL
        case HiveParser.TOK_SHOWDATABASES:
        case HiveParser.TOK_DROPDATABASE:
        case HiveParser.TOK_SWITCHDATABASE:
        case HiveParser.TOK_DESCDATABASE:
        case HiveParser.TOK_ALTERDATABASE_PROPERTIES:

          // Index DDL
        case HiveParser.TOK_ALTERINDEX_PROPERTIES:
        case HiveParser.TOK_CREATEINDEX:
        case HiveParser.TOK_DROPINDEX:
        case HiveParser.TOK_SHOWINDEXES:

          // View DDL
          // case HiveParser.TOK_ALTERVIEW_ADDPARTS:
        case HiveParser.TOK_ALTERVIEW_DROPPARTS:
        case HiveParser.TOK_ALTERVIEW_PROPERTIES:
        case HiveParser.TOK_ALTERVIEW_RENAME:
        case HiveParser.TOK_CREATEVIEW:
        case HiveParser.TOK_DROPVIEW:

          // Authorization DDL
        case HiveParser.TOK_CREATEROLE:
        case HiveParser.TOK_DROPROLE:
        case HiveParser.TOK_GRANT_ROLE:
        case HiveParser.TOK_GRANT_WITH_OPTION:
        case HiveParser.TOK_GRANT:
        case HiveParser.TOK_REVOKE_ROLE:
        case HiveParser.TOK_REVOKE:
        case HiveParser.TOK_SHOW_GRANT:
        case HiveParser.TOK_SHOW_ROLE_GRANT:

          // Misc DDL
        case HiveParser.TOK_LOCKTABLE:
        case HiveParser.TOK_UNLOCKTABLE:
        case HiveParser.TOK_SHOWLOCKS:
        case HiveParser.TOK_DESCFUNCTION:
        case HiveParser.TOK_SHOWFUNCTIONS:
        case HiveParser.TOK_EXPLAIN:

          // Table DDL
        case HiveParser.TOK_ALTERTABLE_ADDPARTS:
        case HiveParser.TOK_ALTERTABLE_ADDCOLS:
        case HiveParser.TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION:
        case HiveParser.TOK_ALTERTABLE_SERDEPROPERTIES:
        case HiveParser.TOK_ALTERTABLE_CLUSTER_SORT:
        case HiveParser.TOK_ALTERTABLE_DROPPARTS:
        case HiveParser.TOK_ALTERTABLE_PROPERTIES:
        case HiveParser.TOK_ALTERTABLE_RENAME:
        case HiveParser.TOK_ALTERTABLE_RENAMECOL:
        case HiveParser.TOK_ALTERTABLE_REPLACECOLS:
        case HiveParser.TOK_ALTERTABLE_SERIALIZER:
        case HiveParser.TOK_ALTERTABLE_TOUCH:
        case HiveParser.TOK_DESCTABLE:
        case HiveParser.TOK_DROPTABLE:
        case HiveParser.TOK_SHOW_TABLESTATUS:
        case HiveParser.TOK_SHOWPARTITIONS:
        case HiveParser.TOK_SHOWTABLES:
          break;

        default:
          throw new HCatException(
              ErrorType.ERROR_INTERNAL_EXCEPTION, "Unexpected token: " + ast.getToken());
      }

      authorizeDDL(context, rootTasks);

    } catch (HCatException e) {
      throw new SemanticException(e);
    } catch (HiveException e) {
      throw new SemanticException(e);
    }

    if (hook != null) {
      hook.postAnalyze(context, rootTasks);
    }
  }