@VisibleForTesting protected static AccessURI extractPartition(ASTNode ast) throws SemanticException { for (int i = 0; i < ast.getChildCount(); i++) { ASTNode child = (ASTNode) ast.getChild(i); if (child.getToken().getType() == HiveParser.TOK_PARTITIONLOCATION && child.getChildCount() == 1) { return parseURI(BaseSemanticAnalyzer.unescapeSQLString(child.getChild(0).getText())); } } return null; }
static ASTNode replaceAliases(ASTNode node, int nodePos, Map<String, String> colToTableAlias) { if (node == null) { return node; } int nodeType = node.getToken().getType(); if (nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT) { String colName = HQLParser.getColName(node); String newAlias = colToTableAlias.get(colName.toLowerCase()); if (StringUtils.isBlank(newAlias)) { return node; } if (nodeType == HiveParser.DOT) { // No need to create a new node, just replace the table name ident ASTNode aliasNode = (ASTNode) node.getChild(0); ASTNode newAliasIdent = new ASTNode(new CommonToken(HiveParser.Identifier, newAlias)); aliasNode.setChild(0, newAliasIdent); } else { // Just a column ref, we need to make it alias.col // '.' will become the parent node ASTNode dot = new ASTNode(new CommonToken(HiveParser.DOT, ".")); ASTNode aliasIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, newAlias)); ASTNode tabRefNode = new ASTNode(new CommonToken(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL")); tabRefNode.addChild(aliasIdentNode); dot.addChild(tabRefNode); ASTNode colIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, colName)); dot.addChild(colIdentNode); ASTNode parent = (ASTNode) node.getParent(); if (parent != null) { parent.setChild(nodePos, dot); } else { return dot; } } } else { // recurse down for (int i = 0; i < node.getChildCount(); i++) { ASTNode child = (ASTNode) node.getChild(i); replaceAliases(child, i, colToTableAlias); } } return node; }
static void updateAliasMap(ASTNode root, CubeQueryContext cubeql) { if (root == null) { return; } if (root.getToken().getType() == TOK_SELEXPR) { ASTNode alias = HQLParser.findNodeByPath(root, Identifier); if (alias != null) { cubeql.addExprToAlias(root, alias); } } else { for (int i = 0; i < root.getChildCount(); i++) { updateAliasMap((ASTNode) root.getChild(i), cubeql); } } }
boolean isCubeMeasure(ASTNode node) { String tabname = null; String colname; int nodeType = node.getToken().getType(); if (!(nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT)) { return false; } if (nodeType == HiveParser.TOK_TABLE_OR_COL) { colname = ((ASTNode) node.getChild(0)).getText(); } else { // node in 'alias.column' format ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier); ASTNode colIdent = (ASTNode) node.getChild(1); colname = colIdent.getText(); tabname = tabident.getText(); } String msrname = StringUtils.isBlank(tabname) ? colname : tabname + "." + colname; return isCubeMeasure(msrname); }
private void resolveClause(CubeQueryContext query, ASTNode node) throws LensException { if (node == null) { return; } int nodeType = node.getToken().getType(); if (nodeType == HiveParser.DOT) { String colName = HQLParser.getColName(node).toLowerCase(); if (!pickedReferences.containsKey(colName)) { return; } // No need to create a new node, // replace the table name identifier and column name identifier ASTNode tableNode = (ASTNode) node.getChild(0); ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier); PickedReference refered = getPickedReference(colName, tabident.getText().toLowerCase()); if (refered == null) { return; } ASTNode newTableNode = new ASTNode( new CommonToken(HiveParser.Identifier, refered.getChainRef().getChainName())); tableNode.setChild(0, newTableNode); ASTNode newColumnNode = new ASTNode( new CommonToken(HiveParser.Identifier, refered.getChainRef().getRefColumn())); node.setChild(1, newColumnNode); } else { // recurse down for (int i = 0; i < node.getChildCount(); i++) { ASTNode child = (ASTNode) node.getChild(i); resolveClause(query, child); } } }
/** * Pre-analyze hook called after compilation and before semantic analysis We extract things for to * Database and metadata level operations which are not capture in the input/output entities * during semantic analysis. Ideally it should be handled in Hive. We need to move most of these * into hive semantic analyzer and then remove it from the access hook. */ @Override public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, ASTNode ast) throws SemanticException { switch (ast.getToken().getType()) { // Hive parser doesn't capture the database name in output entity, so we store it here for // now case HiveParser.TOK_CREATEDATABASE: case HiveParser.TOK_ALTERDATABASE_PROPERTIES: case HiveParser.TOK_DROPDATABASE: case HiveParser.TOK_SWITCHDATABASE: case HiveParser.TOK_DESCDATABASE: currDB = new Database(BaseSemanticAnalyzer.unescapeIdentifier(ast.getChild(0).getText())); break; case HiveParser.TOK_CREATETABLE: case HiveParser.TOK_CREATEVIEW: /* * Compiler doesn't create read/write entities for create table. * Hence we need extract dbname from db.tab format, if applicable */ currDB = extractDatabase((ASTNode) ast.getChild(0)); break; case HiveParser.TOK_DROPTABLE: case HiveParser.TOK_DROPVIEW: case HiveParser.TOK_SHOW_CREATETABLE: case HiveParser.TOK_ALTERTABLE_SERIALIZER: case HiveParser.TOK_ALTERVIEW_ADDPARTS: case HiveParser.TOK_ALTERVIEW_DROPPARTS: case HiveParser.TOK_ALTERVIEW_PROPERTIES: case HiveParser.TOK_ALTERVIEW_RENAME: case HiveParser.TOK_CREATEINDEX: case HiveParser.TOK_DROPINDEX: case HiveParser.TOK_LOCKTABLE: case HiveParser.TOK_UNLOCKTABLE: currTab = extractTable((ASTNode) ast.getFirstChildWithType(HiveParser.TOK_TABNAME)); currDB = extractDatabase((ASTNode) ast.getChild(0)); break; case HiveParser.TOK_ALTERINDEX_REBUILD: currTab = extractTable((ASTNode) ast.getChild(0)); // type is not TOK_TABNAME currDB = extractDatabase((ASTNode) ast.getChild(0)); break; case HiveParser.TOK_SHOW_TABLESTATUS: currDB = extractDatabase((ASTNode) ast.getChild(0)); int children = ast.getChildCount(); for (int i = 1; i < children; i++) { ASTNode child = (ASTNode) ast.getChild(i); if (child.getToken().getType() == HiveParser.Identifier) { currDB = new Database(child.getText()); break; } } // loosing the requested privileges for possible wildcard tables, since // further authorization will be done at the filter step and those unwanted will // eventually be filtered out from the output currTab = Table.ALL; break; case HiveParser.TOK_ALTERTABLE_RENAME: case HiveParser.TOK_ALTERTABLE_PROPERTIES: case HiveParser.TOK_ALTERTABLE_DROPPARTS: case HiveParser.TOK_ALTERTABLE_RENAMECOL: case HiveParser.TOK_ALTERTABLE_ADDCOLS: case HiveParser.TOK_ALTERTABLE_REPLACECOLS: case HiveParser.TOK_SHOW_TBLPROPERTIES: case HiveParser.TOK_SHOWINDEXES: case HiveParser.TOK_SHOWPARTITIONS: // token name TOK_TABNAME is not properly set in this case currTab = extractTable((ASTNode) ast.getChild(0)); currDB = extractDatabase((ASTNode) ast.getChild(0)); break; case HiveParser.TOK_MSCK: // token name TOK_TABNAME is not properly set in this case and child(0) does // not contain the table name. // TODO: Fix Hive to capture the table and DB name currOutTab = extractTable((ASTNode) ast.getChild(1)); currOutDB = extractDatabase((ASTNode) ast.getChild(0)); break; case HiveParser.TOK_ALTERTABLE_ADDPARTS: /* * Compiler doesn't create read/write entities for create table. * Hence we need extract dbname from db.tab format, if applicable */ currTab = extractTable((ASTNode) ast.getChild(0)); currDB = extractDatabase((ASTNode) ast.getChild(0)); partitionURI = extractPartition(ast); break; case HiveParser.TOK_CREATEFUNCTION: String udfClassName = BaseSemanticAnalyzer.unescapeSQLString(ast.getChild(1).getText()); try { CodeSource udfSrc = Class.forName(udfClassName, true, Utilities.getSessionSpecifiedClassLoader()) .getProtectionDomain() .getCodeSource(); if (udfSrc == null) { throw new SemanticException("Could not resolve the jar for UDF class " + udfClassName); } String udfJar = udfSrc.getLocation().getPath(); if (udfJar == null || udfJar.isEmpty()) { throw new SemanticException( "Could not find the jar for UDF class " + udfClassName + "to validate privileges"); } udfURI = parseURI(udfSrc.getLocation().toString(), true); } catch (ClassNotFoundException e) { throw new SemanticException("Error retrieving udf class:" + e.getMessage(), e); } // create/drop function is allowed with any database currDB = Database.ALL; break; case HiveParser.TOK_DROPFUNCTION: // create/drop function is allowed with any database currDB = Database.ALL; break; case HiveParser.TOK_LOAD: String dbName = BaseSemanticAnalyzer.unescapeIdentifier( ast.getChild(1).getChild(0).getChild(0).getText()); currDB = new Database(dbName); break; case HiveParser.TOK_DESCTABLE: currDB = getCanonicalDb(); // For DESCRIBE FORMATTED/EXTENDED ast will have an additional child node with value // "FORMATTED/EXTENDED". isDescTableBasic = (ast.getChildCount() == 1); break; case HiveParser.TOK_TRUNCATETABLE: // SENTRY-826: // Truncate empty partitioned table should throw SemanticException only if the // user does not have permission. // In postAnalyze, currOutDB and currOutTbl will be added into outputHierarchy // which will be validated in the hiveAuthzBinding.authorize method. Preconditions.checkArgument(ast.getChildCount() == 1); // childcount is 1 for table without partition, 2 for table with partitions Preconditions.checkArgument(ast.getChild(0).getChildCount() >= 1); Preconditions.checkArgument(ast.getChild(0).getChild(0).getChildCount() == 1); currOutDB = extractDatabase((ASTNode) ast.getChild(0)); currOutTab = extractTable((ASTNode) ast.getChild(0).getChild(0).getChild(0)); break; default: currDB = getCanonicalDb(); break; } return ast; }
@Override public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, ASTNode ast) throws SemanticException { this.ast = ast; switch (ast.getToken().getType()) { // HCat wants to intercept following tokens and special-handle them. case HiveParser.TOK_CREATETABLE: hook = new CreateTableHook(); return hook.preAnalyze(context, ast); case HiveParser.TOK_CREATEDATABASE: hook = new CreateDatabaseHook(); return hook.preAnalyze(context, ast); case HiveParser.TOK_ALTERTABLE_PARTITION: if (((ASTNode) ast.getChild(1)).getToken().getType() == HiveParser.TOK_ALTERTABLE_FILEFORMAT) { return ast; } else if (((ASTNode) ast.getChild(1)).getToken().getType() == HiveParser.TOK_ALTERTABLE_MERGEFILES) { // unsupported throw new SemanticException("Operation not supported."); } else { return ast; } // HCat will allow these operations to be performed. // Database DDL case HiveParser.TOK_SHOWDATABASES: case HiveParser.TOK_DROPDATABASE: case HiveParser.TOK_SWITCHDATABASE: case HiveParser.TOK_DESCDATABASE: case HiveParser.TOK_ALTERDATABASE_PROPERTIES: // Index DDL case HiveParser.TOK_ALTERINDEX_PROPERTIES: case HiveParser.TOK_CREATEINDEX: case HiveParser.TOK_DROPINDEX: case HiveParser.TOK_SHOWINDEXES: // View DDL // "alter view add partition" does not work because of the nature of implementation // of the DDL in hive. Hive will internally invoke another Driver on the select statement, // and HCat does not let "select" statement through. I cannot find a way to get around it // without modifying hive code. So just leave it unsupported. // case HiveParser.TOK_ALTERVIEW_ADDPARTS: case HiveParser.TOK_ALTERVIEW_DROPPARTS: case HiveParser.TOK_ALTERVIEW_PROPERTIES: case HiveParser.TOK_ALTERVIEW_RENAME: case HiveParser.TOK_CREATEVIEW: case HiveParser.TOK_DROPVIEW: // Authorization DDL case HiveParser.TOK_CREATEROLE: case HiveParser.TOK_DROPROLE: case HiveParser.TOK_GRANT_ROLE: case HiveParser.TOK_GRANT_WITH_OPTION: case HiveParser.TOK_GRANT: case HiveParser.TOK_REVOKE_ROLE: case HiveParser.TOK_REVOKE: case HiveParser.TOK_SHOW_GRANT: case HiveParser.TOK_SHOW_ROLE_GRANT: // Misc DDL case HiveParser.TOK_LOCKTABLE: case HiveParser.TOK_UNLOCKTABLE: case HiveParser.TOK_SHOWLOCKS: case HiveParser.TOK_DESCFUNCTION: case HiveParser.TOK_SHOWFUNCTIONS: case HiveParser.TOK_EXPLAIN: // Table DDL case HiveParser.TOK_ALTERTABLE_ADDPARTS: case HiveParser.TOK_ALTERTABLE_ADDCOLS: case HiveParser.TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION: case HiveParser.TOK_ALTERTABLE_SERDEPROPERTIES: case HiveParser.TOK_ALTERTABLE_CLUSTER_SORT: case HiveParser.TOK_ALTERTABLE_DROPPARTS: case HiveParser.TOK_ALTERTABLE_PROPERTIES: case HiveParser.TOK_ALTERTABLE_RENAME: case HiveParser.TOK_ALTERTABLE_RENAMECOL: case HiveParser.TOK_ALTERTABLE_REPLACECOLS: case HiveParser.TOK_ALTERTABLE_SERIALIZER: case HiveParser.TOK_ALTERTABLE_TOUCH: case HiveParser.TOK_DESCTABLE: case HiveParser.TOK_DROPTABLE: case HiveParser.TOK_SHOW_TABLESTATUS: case HiveParser.TOK_SHOWPARTITIONS: case HiveParser.TOK_SHOWTABLES: return ast; // In all other cases, throw an exception. Its a white-list of allowed operations. default: throw new SemanticException("Operation not supported."); } }
@Override public void postAnalyze( HiveSemanticAnalyzerHookContext context, List<Task<? extends Serializable>> rootTasks) throws SemanticException { try { switch (ast.getToken().getType()) { case HiveParser.TOK_CREATETABLE: case HiveParser.TOK_CREATEDATABASE: case HiveParser.TOK_ALTERTABLE_PARTITION: // HCat will allow these operations to be performed. // Database DDL case HiveParser.TOK_SHOWDATABASES: case HiveParser.TOK_DROPDATABASE: case HiveParser.TOK_SWITCHDATABASE: case HiveParser.TOK_DESCDATABASE: case HiveParser.TOK_ALTERDATABASE_PROPERTIES: // Index DDL case HiveParser.TOK_ALTERINDEX_PROPERTIES: case HiveParser.TOK_CREATEINDEX: case HiveParser.TOK_DROPINDEX: case HiveParser.TOK_SHOWINDEXES: // View DDL // case HiveParser.TOK_ALTERVIEW_ADDPARTS: case HiveParser.TOK_ALTERVIEW_DROPPARTS: case HiveParser.TOK_ALTERVIEW_PROPERTIES: case HiveParser.TOK_ALTERVIEW_RENAME: case HiveParser.TOK_CREATEVIEW: case HiveParser.TOK_DROPVIEW: // Authorization DDL case HiveParser.TOK_CREATEROLE: case HiveParser.TOK_DROPROLE: case HiveParser.TOK_GRANT_ROLE: case HiveParser.TOK_GRANT_WITH_OPTION: case HiveParser.TOK_GRANT: case HiveParser.TOK_REVOKE_ROLE: case HiveParser.TOK_REVOKE: case HiveParser.TOK_SHOW_GRANT: case HiveParser.TOK_SHOW_ROLE_GRANT: // Misc DDL case HiveParser.TOK_LOCKTABLE: case HiveParser.TOK_UNLOCKTABLE: case HiveParser.TOK_SHOWLOCKS: case HiveParser.TOK_DESCFUNCTION: case HiveParser.TOK_SHOWFUNCTIONS: case HiveParser.TOK_EXPLAIN: // Table DDL case HiveParser.TOK_ALTERTABLE_ADDPARTS: case HiveParser.TOK_ALTERTABLE_ADDCOLS: case HiveParser.TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION: case HiveParser.TOK_ALTERTABLE_SERDEPROPERTIES: case HiveParser.TOK_ALTERTABLE_CLUSTER_SORT: case HiveParser.TOK_ALTERTABLE_DROPPARTS: case HiveParser.TOK_ALTERTABLE_PROPERTIES: case HiveParser.TOK_ALTERTABLE_RENAME: case HiveParser.TOK_ALTERTABLE_RENAMECOL: case HiveParser.TOK_ALTERTABLE_REPLACECOLS: case HiveParser.TOK_ALTERTABLE_SERIALIZER: case HiveParser.TOK_ALTERTABLE_TOUCH: case HiveParser.TOK_DESCTABLE: case HiveParser.TOK_DROPTABLE: case HiveParser.TOK_SHOW_TABLESTATUS: case HiveParser.TOK_SHOWPARTITIONS: case HiveParser.TOK_SHOWTABLES: break; default: throw new HCatException( ErrorType.ERROR_INTERNAL_EXCEPTION, "Unexpected token: " + ast.getToken()); } authorizeDDL(context, rootTasks); } catch (HCatException e) { throw new SemanticException(e); } catch (HiveException e) { throw new SemanticException(e); } if (hook != null) { hook.postAnalyze(context, rootTasks); } }