@Override
public Object process(
Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs)
throws SemanticException {
JoinTypeCheckCtx ctx = (JoinTypeCheckCtx) procCtx;
if (ctx.getError() != null) {
return null;
}
ASTNode expr = (ASTNode) nd;
ASTNode parent = stack.size() > 1 ? (ASTNode) stack.get(stack.size() - 2) : null;
if (expr.getType() != HiveParser.TOK_TABLE_OR_COL) {
ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr), expr);
return null;
}
assert (expr.getChildCount() == 1);
String tableOrCol = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText());
boolean qualifiedAccess = (parent != null && parent.getType() == HiveParser.DOT);
ColumnInfo colInfo = null;
if (!qualifiedAccess) {
colInfo = getColInfo(ctx, null, tableOrCol, expr);
// It's a column.
return new ExprNodeColumnDesc(colInfo);
} else if (hasTableAlias(ctx, tableOrCol, expr)) {
return null;
} else {
// Qualified column access for which table was not found
throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(expr));
}
}
private Table extractTable(ASTNode ast) throws SemanticException {
String tableName = BaseSemanticAnalyzer.getUnescapedName(ast);
if (tableName.contains(".")) {
return new Table(tableName.split("\\.")[1]);
} else {
return new Table(tableName);
}
}
private Database extractDatabase(ASTNode ast) throws SemanticException {
String tableName = BaseSemanticAnalyzer.getUnescapedName(ast);
if (tableName.contains(".")) {
return new Database(tableName.split("\\.")[0]);
} else {
return getCanonicalDb();
}
}
@VisibleForTesting
protected static AccessURI extractPartition(ASTNode ast) throws SemanticException {
for (int i = 0; i < ast.getChildCount(); i++) {
ASTNode child = (ASTNode) ast.getChild(i);
if (child.getToken().getType() == HiveParser.TOK_PARTITIONLOCATION
&& child.getChildCount() == 1) {
return parseURI(BaseSemanticAnalyzer.unescapeSQLString(child.getChild(0).getText()));
}
}
return null;
}
@Override
protected ExprNodeColumnDesc processQualifiedColRef(
TypeCheckCtx ctx, ASTNode expr, Object... nodeOutputs) throws SemanticException {
String tableAlias =
BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getChild(0).getText());
// NOTE: tableAlias must be a valid non-ambiguous table alias,
// because we've checked that in TOK_TABLE_OR_COL's process method.
ColumnInfo colInfo =
getColInfo(
(JoinTypeCheckCtx) ctx,
tableAlias,
((ExprNodeConstantDesc) nodeOutputs[1]).getValue().toString(),
expr);
if (colInfo == null) {
ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1)), expr);
return null;
}
return new ExprNodeColumnDesc(
colInfo.getType(), colInfo.getInternalName(), tableAlias, colInfo.getIsVirtualCol());
}
/**
* Pre-analyze hook called after compilation and before semantic analysis We extract things for to
* Database and metadata level operations which are not capture in the input/output entities
* during semantic analysis. Ideally it should be handled in Hive. We need to move most of these
* into hive semantic analyzer and then remove it from the access hook.
*/
@Override
public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, ASTNode ast)
throws SemanticException {
switch (ast.getToken().getType()) {
// Hive parser doesn't capture the database name in output entity, so we store it here for
// now
case HiveParser.TOK_CREATEDATABASE:
case HiveParser.TOK_ALTERDATABASE_PROPERTIES:
case HiveParser.TOK_DROPDATABASE:
case HiveParser.TOK_SWITCHDATABASE:
case HiveParser.TOK_DESCDATABASE:
currDB = new Database(BaseSemanticAnalyzer.unescapeIdentifier(ast.getChild(0).getText()));
break;
case HiveParser.TOK_CREATETABLE:
case HiveParser.TOK_CREATEVIEW:
/*
* Compiler doesn't create read/write entities for create table.
* Hence we need extract dbname from db.tab format, if applicable
*/
currDB = extractDatabase((ASTNode) ast.getChild(0));
break;
case HiveParser.TOK_DROPTABLE:
case HiveParser.TOK_DROPVIEW:
case HiveParser.TOK_SHOW_CREATETABLE:
case HiveParser.TOK_ALTERTABLE_SERIALIZER:
case HiveParser.TOK_ALTERVIEW_ADDPARTS:
case HiveParser.TOK_ALTERVIEW_DROPPARTS:
case HiveParser.TOK_ALTERVIEW_PROPERTIES:
case HiveParser.TOK_ALTERVIEW_RENAME:
case HiveParser.TOK_CREATEINDEX:
case HiveParser.TOK_DROPINDEX:
case HiveParser.TOK_LOCKTABLE:
case HiveParser.TOK_UNLOCKTABLE:
currTab = extractTable((ASTNode) ast.getFirstChildWithType(HiveParser.TOK_TABNAME));
currDB = extractDatabase((ASTNode) ast.getChild(0));
break;
case HiveParser.TOK_ALTERINDEX_REBUILD:
currTab = extractTable((ASTNode) ast.getChild(0)); // type is not TOK_TABNAME
currDB = extractDatabase((ASTNode) ast.getChild(0));
break;
case HiveParser.TOK_SHOW_TABLESTATUS:
currDB = extractDatabase((ASTNode) ast.getChild(0));
int children = ast.getChildCount();
for (int i = 1; i < children; i++) {
ASTNode child = (ASTNode) ast.getChild(i);
if (child.getToken().getType() == HiveParser.Identifier) {
currDB = new Database(child.getText());
break;
}
}
// loosing the requested privileges for possible wildcard tables, since
// further authorization will be done at the filter step and those unwanted will
// eventually be filtered out from the output
currTab = Table.ALL;
break;
case HiveParser.TOK_ALTERTABLE_RENAME:
case HiveParser.TOK_ALTERTABLE_PROPERTIES:
case HiveParser.TOK_ALTERTABLE_DROPPARTS:
case HiveParser.TOK_ALTERTABLE_RENAMECOL:
case HiveParser.TOK_ALTERTABLE_ADDCOLS:
case HiveParser.TOK_ALTERTABLE_REPLACECOLS:
case HiveParser.TOK_SHOW_TBLPROPERTIES:
case HiveParser.TOK_SHOWINDEXES:
case HiveParser.TOK_SHOWPARTITIONS:
// token name TOK_TABNAME is not properly set in this case
currTab = extractTable((ASTNode) ast.getChild(0));
currDB = extractDatabase((ASTNode) ast.getChild(0));
break;
case HiveParser.TOK_MSCK:
// token name TOK_TABNAME is not properly set in this case and child(0) does
// not contain the table name.
// TODO: Fix Hive to capture the table and DB name
currOutTab = extractTable((ASTNode) ast.getChild(1));
currOutDB = extractDatabase((ASTNode) ast.getChild(0));
break;
case HiveParser.TOK_ALTERTABLE_ADDPARTS:
/*
* Compiler doesn't create read/write entities for create table.
* Hence we need extract dbname from db.tab format, if applicable
*/
currTab = extractTable((ASTNode) ast.getChild(0));
currDB = extractDatabase((ASTNode) ast.getChild(0));
partitionURI = extractPartition(ast);
break;
case HiveParser.TOK_CREATEFUNCTION:
String udfClassName = BaseSemanticAnalyzer.unescapeSQLString(ast.getChild(1).getText());
try {
CodeSource udfSrc =
Class.forName(udfClassName, true, Utilities.getSessionSpecifiedClassLoader())
.getProtectionDomain()
.getCodeSource();
if (udfSrc == null) {
throw new SemanticException("Could not resolve the jar for UDF class " + udfClassName);
}
String udfJar = udfSrc.getLocation().getPath();
if (udfJar == null || udfJar.isEmpty()) {
throw new SemanticException(
"Could not find the jar for UDF class " + udfClassName + "to validate privileges");
}
udfURI = parseURI(udfSrc.getLocation().toString(), true);
} catch (ClassNotFoundException e) {
throw new SemanticException("Error retrieving udf class:" + e.getMessage(), e);
}
// create/drop function is allowed with any database
currDB = Database.ALL;
break;
case HiveParser.TOK_DROPFUNCTION:
// create/drop function is allowed with any database
currDB = Database.ALL;
break;
case HiveParser.TOK_LOAD:
String dbName =
BaseSemanticAnalyzer.unescapeIdentifier(
ast.getChild(1).getChild(0).getChild(0).getText());
currDB = new Database(dbName);
break;
case HiveParser.TOK_DESCTABLE:
currDB = getCanonicalDb();
// For DESCRIBE FORMATTED/EXTENDED ast will have an additional child node with value
// "FORMATTED/EXTENDED".
isDescTableBasic = (ast.getChildCount() == 1);
break;
case HiveParser.TOK_TRUNCATETABLE:
// SENTRY-826:
// Truncate empty partitioned table should throw SemanticException only if the
// user does not have permission.
// In postAnalyze, currOutDB and currOutTbl will be added into outputHierarchy
// which will be validated in the hiveAuthzBinding.authorize method.
Preconditions.checkArgument(ast.getChildCount() == 1);
// childcount is 1 for table without partition, 2 for table with partitions
Preconditions.checkArgument(ast.getChild(0).getChildCount() >= 1);
Preconditions.checkArgument(ast.getChild(0).getChild(0).getChildCount() == 1);
currOutDB = extractDatabase((ASTNode) ast.getChild(0));
currOutTab = extractTable((ASTNode) ast.getChild(0).getChild(0).getChild(0));
break;
default:
currDB = getCanonicalDb();
break;
}
return ast;
}
public void validate() throws SemanticException {
if ((this.getCols() == null) || (this.getCols().size() == 0)) {
// for now make sure that serde exists
if (StringUtils.isEmpty(this.getSerName())
|| !SerDeUtils.shouldGetColsFromSerDe(this.getSerName())) {
throw new SemanticException(ErrorMsg.INVALID_TBL_DDL_SERDE.getMsg());
}
return;
}
if (this.getStorageHandler() == null) {
try {
Class<?> origin = Class.forName(this.getOutputFormat(), true, JavaUtils.getClassLoader());
Class<? extends HiveOutputFormat> replaced =
HiveFileFormatUtils.getOutputFormatSubstitute(origin);
if (replaced == null) {
throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE.getMsg());
}
} catch (ClassNotFoundException e) {
throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE.getMsg());
}
}
List<String> colNames = ParseUtils.validateColumnNameUniqueness(this.getCols());
if (this.getBucketCols() != null) {
// all columns in cluster and sort are valid columns
Iterator<String> bucketCols = this.getBucketCols().iterator();
while (bucketCols.hasNext()) {
String bucketCol = bucketCols.next();
boolean found = false;
Iterator<String> colNamesIter = colNames.iterator();
while (colNamesIter.hasNext()) {
String colName = colNamesIter.next();
if (bucketCol.equalsIgnoreCase(colName)) {
found = true;
break;
}
}
if (!found) {
throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg());
}
}
}
if (this.getSortCols() != null) {
// all columns in cluster and sort are valid columns
Iterator<Order> sortCols = this.getSortCols().iterator();
while (sortCols.hasNext()) {
String sortCol = sortCols.next().getCol();
boolean found = false;
Iterator<String> colNamesIter = colNames.iterator();
while (colNamesIter.hasNext()) {
String colName = colNamesIter.next();
if (sortCol.equalsIgnoreCase(colName)) {
found = true;
break;
}
}
if (!found) {
throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg());
}
}
}
if (this.getPartCols() != null) {
// there is no overlap between columns and partitioning columns
Iterator<FieldSchema> partColsIter = this.getPartCols().iterator();
while (partColsIter.hasNext()) {
FieldSchema fs = partColsIter.next();
String partCol = fs.getName();
PrimitiveObjectInspectorUtils.PrimitiveTypeEntry pte =
PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(fs.getType());
if (null == pte) {
throw new SemanticException(
ErrorMsg.PARTITION_COLUMN_NON_PRIMITIVE.getMsg()
+ " Found "
+ partCol
+ " of type: "
+ fs.getType());
}
Iterator<String> colNamesIter = colNames.iterator();
while (colNamesIter.hasNext()) {
String colName = BaseSemanticAnalyzer.unescapeIdentifier(colNamesIter.next());
if (partCol.equalsIgnoreCase(colName)) {
throw new SemanticException(ErrorMsg.COLUMN_REPEATED_IN_PARTITIONING_COLS.getMsg());
}
}
}
}
/* Validate skewed information. */
ValidationUtility.validateSkewedInformation(
colNames, this.getSkewedColNames(), this.getSkewedColValues());
}