private static ASTNode replaceAlias(final ASTNode expr, final CubeQueryContext cubeql)
throws SemanticException {
ASTNode finalAST = HQLParser.copyAST(expr);
HQLParser.bft(
finalAST,
new ASTNodeVisitor() {
@Override
public void visit(TreeNode visited) {
ASTNode node = visited.getNode();
ASTNode parent = null;
if (visited.getParent() != null) {
parent = visited.getParent().getNode();
}
if (node.getToken().getType() == TOK_TABLE_OR_COL
&& (parent != null && parent.getToken().getType() == DOT)) {
ASTNode current = (ASTNode) node.getChild(0);
if (current.getToken().getType() == Identifier) {
String tableName = current.getToken().getText().toLowerCase();
String alias = cubeql.getAliasForTableName(tableName);
if (!alias.equalsIgnoreCase(tableName)) {
node.setChild(0, new ASTNode(new CommonToken(HiveParser.Identifier, alias)));
}
}
}
}
});
return finalAST;
}
public CubeQueryContext(ASTNode ast, QB qb, Configuration queryConf, HiveConf metastoreConf)
throws LensException {
this.ast = ast;
this.qb = qb;
this.conf = queryConf;
this.clauseName = getClause();
this.timeRanges = new ArrayList<TimeRange>();
try {
metastoreClient = CubeMetastoreClient.getInstance(metastoreConf);
} catch (HiveException e) {
throw new LensException(e);
}
if (qb.getParseInfo().getWhrForClause(clauseName) != null) {
this.whereAST = qb.getParseInfo().getWhrForClause(clauseName);
}
if (qb.getParseInfo().getHavingForClause(clauseName) != null) {
this.havingAST = qb.getParseInfo().getHavingForClause(clauseName);
}
if (qb.getParseInfo().getOrderByForClause(clauseName) != null) {
this.orderByAST = qb.getParseInfo().getOrderByForClause(clauseName);
}
if (qb.getParseInfo().getGroupByForClause(clauseName) != null) {
this.groupByAST = qb.getParseInfo().getGroupByForClause(clauseName);
}
if (qb.getParseInfo().getSelForClause(clauseName) != null) {
this.selectAST = qb.getParseInfo().getSelForClause(clauseName);
}
for (ASTNode aggrTree : qb.getParseInfo().getAggregationExprsForClause(clauseName).values()) {
String aggr = HQLParser.getString(aggrTree);
aggregateExprs.add(aggr);
}
extractMetaTables();
}
boolean hasAggregates() {
for (ExprSpecContext esc : allExprs) {
if (HQLParser.hasAggregate(esc.finalAST)) {
return true;
}
}
return false;
}
public String getInsertClause() {
String insertString = "";
ASTNode destTree = qb.getParseInfo().getDestForClause(clauseName);
if (destTree != null
&& ((ASTNode) (destTree.getChild(0))).getToken().getType() != TOK_TMP_FILE) {
insertString =
"INSERT OVERWRITE" + HQLParser.getString(qb.getParseInfo().getDestForClause(clauseName));
}
return insertString;
}
private void resolveClause(CubeQueryContext query, ASTNode node) throws LensException {
if (node == null) {
return;
}
int nodeType = node.getToken().getType();
if (nodeType == HiveParser.DOT) {
String colName = HQLParser.getColName(node).toLowerCase();
if (!pickedReferences.containsKey(colName)) {
return;
}
// No need to create a new node,
// replace the table name identifier and column name identifier
ASTNode tableNode = (ASTNode) node.getChild(0);
ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier);
PickedReference refered = getPickedReference(colName, tabident.getText().toLowerCase());
if (refered == null) {
return;
}
ASTNode newTableNode =
new ASTNode(
new CommonToken(HiveParser.Identifier, refered.getChainRef().getChainName()));
tableNode.setChild(0, newTableNode);
ASTNode newColumnNode =
new ASTNode(
new CommonToken(HiveParser.Identifier, refered.getChainRef().getRefColumn()));
node.setChild(1, newColumnNode);
} else {
// recurse down
for (int i = 0; i < node.getChildCount(); i++) {
ASTNode child = (ASTNode) node.getChild(i);
resolveClause(query, child);
}
}
}
static ASTNode replaceAliases(ASTNode node, int nodePos, Map<String, String> colToTableAlias) {
if (node == null) {
return node;
}
int nodeType = node.getToken().getType();
if (nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT) {
String colName = HQLParser.getColName(node);
String newAlias = colToTableAlias.get(colName.toLowerCase());
if (StringUtils.isBlank(newAlias)) {
return node;
}
if (nodeType == HiveParser.DOT) {
// No need to create a new node, just replace the table name ident
ASTNode aliasNode = (ASTNode) node.getChild(0);
ASTNode newAliasIdent = new ASTNode(new CommonToken(HiveParser.Identifier, newAlias));
aliasNode.setChild(0, newAliasIdent);
} else {
// Just a column ref, we need to make it alias.col
// '.' will become the parent node
ASTNode dot = new ASTNode(new CommonToken(HiveParser.DOT, "."));
ASTNode aliasIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, newAlias));
ASTNode tabRefNode =
new ASTNode(new CommonToken(HiveParser.TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL"));
tabRefNode.addChild(aliasIdentNode);
dot.addChild(tabRefNode);
ASTNode colIdentNode = new ASTNode(new CommonToken(HiveParser.Identifier, colName));
dot.addChild(colIdentNode);
ASTNode parent = (ASTNode) node.getParent();
if (parent != null) {
parent.setChild(nodePos, dot);
} else {
return dot;
}
}
} else {
// recurse down
for (int i = 0; i < node.getChildCount(); i++) {
ASTNode child = (ASTNode) node.getChild(i);
replaceAliases(child, i, colToTableAlias);
}
}
return node;
}
static void updateAliasMap(ASTNode root, CubeQueryContext cubeql) {
if (root == null) {
return;
}
if (root.getToken().getType() == TOK_SELEXPR) {
ASTNode alias = HQLParser.findNodeByPath(root, Identifier);
if (alias != null) {
cubeql.addExprToAlias(root, alias);
}
} else {
for (int i = 0; i < root.getChildCount(); i++) {
updateAliasMap((ASTNode) root.getChild(i), cubeql);
}
}
}
private void addRangeClauses(CandidateFact fact) throws LensException {
if (fact != null) {
// resolve timerange positions and replace it by corresponding where clause
for (TimeRange range : getTimeRanges()) {
for (Map.Entry<String, String> entry :
fact.getRangeToStorageWhereMap().get(range).entrySet()) {
String table = entry.getKey();
String rangeWhere = entry.getValue();
if (!StringUtils.isBlank(rangeWhere)) {
ASTNode rangeAST = HQLParser.parseExpr(rangeWhere);
range.getParent().setChild(range.getChildIndex(), rangeAST);
}
fact.getStorgeWhereClauseMap().put(table, getWhereTree());
}
}
}
}
private static void replaceColumnInAST(
ASTNode expr, final String toReplace, final ASTNode columnAST) throws SemanticException {
if (expr == null) {
return;
}
// Traverse the tree and resolve expression columns
HQLParser.bft(
expr,
new ASTNodeVisitor() {
@Override
public void visit(TreeNode visited) throws SemanticException {
ASTNode node = visited.getNode();
int childcount = node.getChildCount();
for (int i = 0; i < childcount; i++) {
ASTNode current = (ASTNode) node.getChild(i);
if (current.getToken().getType() == TOK_TABLE_OR_COL
&& (node != null && node.getToken().getType() != DOT)) {
// Take child ident.totext
ASTNode ident = (ASTNode) current.getChild(0);
String column = ident.getText().toLowerCase();
if (toReplace.equals(column)) {
node.setChild(i, HQLParser.copyAST(columnAST));
}
} else if (current.getToken().getType() == DOT) {
// This is for the case where column name is prefixed by table name
// or table alias
// For example 'select fact.id, dim2.id ...'
// Right child is the column name, left child.ident is table name
ASTNode tabident = HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier);
ASTNode colIdent = (ASTNode) current.getChild(1);
String column = colIdent.getText().toLowerCase();
if (toReplace.equals(column)) {
node.setChild(i, HQLParser.copyAST(columnAST));
}
}
}
}
});
}
boolean isCubeMeasure(ASTNode node) {
String tabname = null;
String colname;
int nodeType = node.getToken().getType();
if (!(nodeType == HiveParser.TOK_TABLE_OR_COL || nodeType == HiveParser.DOT)) {
return false;
}
if (nodeType == HiveParser.TOK_TABLE_OR_COL) {
colname = ((ASTNode) node.getChild(0)).getText();
} else {
// node in 'alias.column' format
ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier);
ASTNode colIdent = (ASTNode) node.getChild(1);
colname = colIdent.getText();
tabname = tabident.getText();
}
String msrname = StringUtils.isBlank(tabname) ? colname : tabname + "." + colname;
return isCubeMeasure(msrname);
}
private void addAllNestedExpressions(
CubeQueryContext cubeql,
ExprSpecContext baseEsc,
AbstractBaseTable baseTable,
Set<ExprSpecContext> nestedExpressions,
Set<String> exprCols)
throws SemanticException {
for (String col : exprCols) {
Set<ExprSpecContext> replacedExpressions = new LinkedHashSet<ExprSpecContext>();
for (ExprSpec es : baseTable.getExpressionByName(col).getExpressionSpecs()) {
ASTNode finalAST = HQLParser.copyAST(baseEsc.getFinalAST());
replaceColumnInAST(finalAST, col, es.getASTNode());
ExprSpecContext replacedESC = new ExprSpecContext(baseEsc, es, finalAST, cubeql);
nestedExpressions.add(replacedESC);
replacedExpressions.add(replacedESC);
}
Set<String> remaining = new LinkedHashSet<String>(exprCols);
remaining.remove(col);
for (ExprSpecContext replacedESC : replacedExpressions) {
addAllNestedExpressions(cubeql, replacedESC, baseTable, nestedExpressions, remaining);
}
}
}
private void replaceAST(final CubeQueryContext cubeql, ASTNode node) throws SemanticException {
if (node == null) {
return;
}
// Traverse the tree and resolve expression columns
HQLParser.bft(
node,
new ASTNodeVisitor() {
@Override
public void visit(TreeNode visited) throws SemanticException {
ASTNode node = visited.getNode();
int childcount = node.getChildCount();
for (int i = 0; i < childcount; i++) {
ASTNode current = (ASTNode) node.getChild(i);
if (current.getToken().getType() == DOT) {
// This is for the case where column name is prefixed by table name
// or table alias
// For example 'select fact.id, dim2.id ...'
// Right child is the column name, left child.ident is table name
ASTNode tabident =
HQLParser.findNodeByPath(current, TOK_TABLE_OR_COL, Identifier);
ASTNode colIdent = (ASTNode) current.getChild(1);
String column = colIdent.getText().toLowerCase();
if (pickedExpressions.containsKey(column)) {
PickedExpression expr =
getPickedExpression(column, tabident.getText().toLowerCase());
if (expr != null) {
node.setChild(i, replaceAlias(expr.pickedCtx.finalAST, cubeql));
}
}
}
}
}
});
}
public void addExprToAlias(ASTNode expr, ASTNode alias) {
exprToAlias.put(HQLParser.getString(expr).trim(), alias.getText().toLowerCase());
}
public String toString() {
return HQLParser.getString(finalAST);
}
public String getSelectTree() {
return HQLParser.getString(selectAST);
}
public String getWhereTree() {
if (whereAST != null) {
return HQLParser.getString(whereAST);
}
return null;
}
public String getGroupByTree() {
if (groupByAST != null) {
return HQLParser.getString(groupByAST);
}
return null;
}
public String getHavingTree() {
if (havingAST != null) {
return HQLParser.getString(havingAST);
}
return null;
}
public String getOrderByTree() {
if (orderByAST != null) {
return HQLParser.getString(orderByAST);
}
return null;
}