/**
* destination table : a table whose columns are getting queried intermediate table : a table
* which is only used as a link between cube and destination table
*
* @param alias
* @param tblName
* @param isOptional pass false when it's a destination table pass true when it's an intermediate
* table when join chain destination is being added, this will be false.
* @param isChainedDimension pass true when you're adding the dimension as a joinchain
* destination, pass false when this table is mentioned by name in the user query
* @return true if added
* @throws LensException
*/
private boolean addQueriedTable(
String alias, String tblName, boolean isOptional, boolean isChainedDimension)
throws LensException {
alias = alias.toLowerCase();
if (cubeTbls.containsKey(alias)) {
return true;
}
try {
if (metastoreClient.isCube(tblName)) {
if (cube != null) {
if (!cube.getName().equalsIgnoreCase(tblName)) {
throw new LensException(
LensCubeErrorCode.MORE_THAN_ONE_CUBE.getLensErrorInfo(), cube.getName(), tblName);
}
}
cube = metastoreClient.getCube(tblName);
cubeTbls.put(alias, (AbstractCubeTable) cube);
} else if (metastoreClient.isDimension(tblName)) {
Dimension dim = metastoreClient.getDimension(tblName);
if (!isOptional) {
dimensions.add(dim);
}
if (!isChainedDimension) {
nonChainedDimensions.add(dim);
}
cubeTbls.put(alias, dim);
} else {
return false;
}
} catch (HiveException e) {
return false;
}
return true;
}
public Set<Dimension> rewriteDenormctx(
CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery, boolean replaceFact)
throws LensException {
Set<Dimension> refTbls = new HashSet<>();
if (!tableToRefCols.isEmpty()) {
// pick referenced columns for fact
if (cfact != null) {
pickColumnsForTable(cfact.getName());
}
// pick referenced columns for dimensions
if (dimsToQuery != null && !dimsToQuery.isEmpty()) {
for (CandidateDim cdim : dimsToQuery.values()) {
pickColumnsForTable(cdim.getName());
}
}
// Replace picked reference in all the base trees
replaceReferencedColumns(cfact, replaceFact);
// Add the picked references to dimsToQuery
for (PickedReference picked : pickedRefs) {
if (isPickedFor(picked, cfact, dimsToQuery)) {
refTbls.add(
(Dimension) cubeql.getCubeTableForAlias(picked.getChainRef().getChainName()));
cubeql.addColumnsQueried(
picked.getChainRef().getChainName(), picked.getChainRef().getRefColumn());
}
}
}
return refTbls;
}
private String getStorageStringWithAlias(
CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, String alias) {
if (cubeTbls.get(alias) instanceof CubeInterface) {
return fact.getStorageString(alias);
} else {
return dimsToQuery.get(cubeTbls.get(alias)).getStorageString(alias);
}
}
private void getQLString(
QBJoinTree joinTree,
StringBuilder builder,
CandidateFact fact,
Map<Dimension, CandidateDim> dimsToQuery)
throws LensException {
List<String> joiningTables = new ArrayList<>();
if (joinTree.getBaseSrc()[0] == null) {
if (joinTree.getJoinSrc() != null) {
getQLString(joinTree.getJoinSrc(), builder, fact, dimsToQuery);
}
} else { // (joinTree.getBaseSrc()[0] != null){
String alias = joinTree.getBaseSrc()[0].toLowerCase();
builder.append(getStorageStringWithAlias(fact, dimsToQuery, alias));
joiningTables.add(alias);
}
if (joinTree.getJoinCond() != null) {
builder.append(JoinUtils.getJoinTypeStr(joinTree.getJoinCond()[0].getJoinType()));
builder.append(" JOIN ");
}
if (joinTree.getBaseSrc()[1] == null) {
if (joinTree.getJoinSrc() != null) {
getQLString(joinTree.getJoinSrc(), builder, fact, dimsToQuery);
}
} else { // (joinTree.getBaseSrc()[1] != null){
String alias = joinTree.getBaseSrc()[1].toLowerCase();
builder.append(getStorageStringWithAlias(fact, dimsToQuery, alias));
joiningTables.add(alias);
}
String joinCond = joinConds.get(joinTree);
if (joinCond != null) {
builder.append(" ON ");
builder.append(joinCond);
// joining tables will contain all tables involved in joins.
// we need to push storage filters for Dimensions into join conditions, thus the following
// code
// takes care of the same.
for (String joiningTable : joiningTables) {
if (cubeTbls.get(joiningTable) instanceof Dimension) {
DimOnlyHQLContext.appendWhereClause(
builder, getWhereClauseWithAlias(dimsToQuery, joiningTable), true);
dimsToQuery.get(cubeTbls.get(joiningTable)).setWhereClauseAdded(joiningTable);
}
}
} else {
throw new LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo());
}
}
private Map<Dimension, CandidateDim> pickCandidateDimsToQuery(Set<Dimension> dimensions)
throws LensException {
Map<Dimension, CandidateDim> dimsToQuery = new HashMap<Dimension, CandidateDim>();
if (!dimensions.isEmpty()) {
for (Dimension dim : dimensions) {
if (candidateDims.get(dim) != null && candidateDims.get(dim).size() > 0) {
CandidateDim cdim = candidateDims.get(dim).iterator().next();
log.info(
"Available candidate dims are:{}, picking up {} for querying.",
candidateDims.get(dim),
cdim.dimtable);
dimsToQuery.put(dim, cdim);
} else {
String reason = "";
if (dimPruningMsgs.get(dim) != null && !dimPruningMsgs.get(dim).isEmpty()) {
ByteArrayOutputStream out = null;
try {
ObjectMapper mapper = new ObjectMapper();
out = new ByteArrayOutputStream();
mapper.writeValue(out, dimPruningMsgs.get(dim).getJsonObject());
reason = out.toString("UTF-8");
} catch (Exception e) {
throw new LensException("Error writing dim pruning messages", e);
} finally {
if (out != null) {
try {
out.close();
} catch (IOException e) {
throw new LensException(e);
}
}
}
}
log.error(
"Query rewrite failed due to NO_CANDIDATE_DIM_AVAILABLE, Cause {}",
dimPruningMsgs.get(dim).toJsonObject());
throw new NoCandidateDimAvailableException(dimPruningMsgs.get(dim));
}
}
}
return dimsToQuery;
}
// checks if the reference if picked for facts and dimsToQuery passed
private boolean isPickedFor(
PickedReference picked, CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery) {
if (cfact != null && picked.pickedFor.equalsIgnoreCase(cfact.getName())) {
return true;
}
if (dimsToQuery != null) {
for (CandidateDim cdim : dimsToQuery.values()) {
if (picked.pickedFor.equalsIgnoreCase(cdim.getName())) {
return true;
}
}
}
return false;
}
public String getAliasForTableName(String tableName) {
for (String alias : qb.getTabAliases()) {
String table = qb.getTabNameForAlias(alias);
if (table != null && table.equalsIgnoreCase(tableName)) {
return alias;
}
}
// get alias from cubeTbls
for (Map.Entry<String, AbstractCubeTable> cubeTblEntry : cubeTbls.entrySet()) {
if (cubeTblEntry.getValue().getName().equalsIgnoreCase(tableName)) {
return cubeTblEntry.getKey();
}
}
return tableName.toLowerCase();
}
private void addOptionalDimTable(
String alias,
CandidateTable candidate,
boolean isRequiredInJoin,
String cubeCol,
boolean isRef,
String tableAlias,
String... cols)
throws LensException {
alias = alias.toLowerCase();
if (!addQueriedTable(alias, true)) {
throw new LensException(LensCubeErrorCode.QUERIED_TABLE_NOT_FOUND.getLensErrorInfo(), alias);
}
Dimension dim = (Dimension) cubeTbls.get(alias);
Aliased<Dimension> aliasedDim = Aliased.create(dim, alias);
OptionalDimCtx optDim = optionalDimensionMap.get(aliasedDim);
if (optDim == null) {
optDim = new OptionalDimCtx();
optionalDimensionMap.put(aliasedDim, optDim);
}
if (cols != null && candidate != null) {
for (String col : cols) {
optDim.colQueried.add(col);
}
optDim.requiredForCandidates.add(candidate);
}
if (cubeCol != null) {
if (isRef) {
updateRefColDim(cubeCol, aliasedDim);
} else {
updateExprColDim(tableAlias, cubeCol, aliasedDim);
}
}
if (!optDim.isRequiredInJoinChain) {
optDim.isRequiredInJoinChain = isRequiredInJoin;
}
if (log.isDebugEnabled()) {
log.debug(
"Adding optional dimension:{} optDim:{} {} isRef:{}",
aliasedDim,
optDim,
(cubeCol == null ? "" : " for column:" + cubeCol),
isRef);
}
}
private boolean addJoinChain(String alias, boolean isOptional) throws LensException {
boolean retVal = false;
String aliasLowerCaseStr = alias.toLowerCase();
JoinChain joinchain = null;
if (getCube() != null) {
JoinChain chainByName = getCube().getChainByName(aliasLowerCaseStr);
if (chainByName != null) {
joinchain = chainByName;
retVal = true;
}
}
if (!retVal) {
for (Dimension table : dimensions) {
JoinChain chainByName = table.getChainByName(aliasLowerCaseStr);
if (chainByName != null) {
joinchain = chainByName;
retVal = true;
break;
}
}
}
if (retVal) {
joinchains.put(aliasLowerCaseStr, new JoinChain(joinchain));
String destTable = joinchain.getDestTable();
boolean added = addQueriedTable(alias, destTable, isOptional, true);
if (!added) {
log.info("Queried tables do not exist. Missing tables:{}", destTable);
throw new LensException(LensCubeErrorCode.NEITHER_CUBE_NOR_DIMENSION.getLensErrorInfo());
}
log.info("Added join chain for {}", destTable);
return true;
}
return retVal;
}
String getQBFromString(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery)
throws LensException {
String fromString;
if (getJoinAST() == null) {
if (cube != null) {
if (dimensions.size() > 0) {
throw new LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo());
}
fromString = fact.getStorageString(getAliasForTableName(cube.getName()));
} else {
if (dimensions.size() != 1) {
throw new LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo());
}
Dimension dim = dimensions.iterator().next();
fromString = dimsToQuery.get(dim).getStorageString(getAliasForTableName(dim.getName()));
}
} else {
StringBuilder builder = new StringBuilder();
getQLString(qb.getQbJoinTree(), builder, fact, dimsToQuery);
fromString = builder.toString();
}
return fromString;
}
public String toHQL() throws LensException {
Set<CandidateFact> cfacts = pickCandidateFactToQuery();
Map<Dimension, CandidateDim> dimsToQuery = pickCandidateDimsToQuery(dimensions);
log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery);
if (autoJoinCtx != null) {
// prune join paths for picked fact and dimensions
autoJoinCtx.pruneAllPaths(cube, cfacts, dimsToQuery);
}
Map<CandidateFact, Set<Dimension>> factDimMap = new HashMap<>();
if (cfacts != null) {
if (cfacts.size() > 1) {
// copy ASTs for each fact
for (CandidateFact cfact : cfacts) {
cfact.copyASTs(this);
factDimMap.put(cfact, new HashSet<>(dimsToQuery.keySet()));
}
}
for (CandidateFact fact : cfacts) {
addRangeClauses(fact);
}
}
// pick dimension tables required during expression expansion for the picked fact and dimensions
Set<Dimension> exprDimensions = new HashSet<Dimension>();
if (cfacts != null) {
for (CandidateFact cfact : cfacts) {
Set<Dimension> factExprDimTables =
exprCtx.rewriteExprCtx(cfact, dimsToQuery, cfacts.size() > 1 ? cfact : this);
exprDimensions.addAll(factExprDimTables);
if (cfacts.size() > 1) {
factDimMap.get(cfact).addAll(factExprDimTables);
}
}
if (cfacts.size() > 1) {
havingAST = MultiFactHQLContext.pushDownHaving(havingAST, this, cfacts);
}
} else {
// dim only query
exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, this));
}
dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions));
log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery);
// pick denorm tables for the picked fact and dimensions
Set<Dimension> denormTables = new HashSet<Dimension>();
if (cfacts != null) {
for (CandidateFact cfact : cfacts) {
Set<Dimension> factDenormTables =
deNormCtx.rewriteDenormctx(cfact, dimsToQuery, cfacts.size() > 1);
denormTables.addAll(factDenormTables);
if (cfacts.size() > 1) {
factDimMap.get(cfact).addAll(factDenormTables);
}
}
} else {
denormTables.addAll(deNormCtx.rewriteDenormctx(null, dimsToQuery, false));
}
dimsToQuery.putAll(pickCandidateDimsToQuery(denormTables));
log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery);
// Prune join paths once denorm tables are picked
if (autoJoinCtx != null) {
// prune join paths for picked fact and dimensions
autoJoinCtx.pruneAllPaths(cube, cfacts, dimsToQuery);
}
if (autoJoinCtx != null) {
// add optional dims from Join resolver
Set<Dimension> joiningTables = new HashSet<Dimension>();
if (cfacts != null && cfacts.size() > 1) {
for (CandidateFact cfact : cfacts) {
Set<Dimension> factJoiningTables =
autoJoinCtx.pickOptionalTables(cfact, factDimMap.get(cfact), this);
factDimMap.get(cfact).addAll(factJoiningTables);
joiningTables.addAll(factJoiningTables);
}
} else {
joiningTables.addAll(autoJoinCtx.pickOptionalTables(null, dimsToQuery.keySet(), this));
}
dimsToQuery.putAll(pickCandidateDimsToQuery(joiningTables));
}
log.info("Picked Fact:{} dimsToQuery: {}", cfacts, dimsToQuery);
pickedDimTables = dimsToQuery.values();
pickedFacts = cfacts;
if (cfacts != null) {
if (cfacts.size() > 1) {
// Update ASTs for each fact
for (CandidateFact cfact : cfacts) {
cfact.updateASTs(this);
}
whereAST =
MultiFactHQLContext.convertHavingToWhere(
havingAST, this, cfacts, new DefaultAliasDecider());
}
}
hqlContext = createHQLContext(cfacts, dimsToQuery, factDimMap);
return hqlContext.toHQL();
}
private String getWhereClauseWithAlias(Map<Dimension, CandidateDim> dimsToQuery, String alias) {
return StorageUtil.getWhereClause(dimsToQuery.get(cubeTbls.get(alias)), alias);
}
public AbstractCubeTable getCubeTableForAlias(String alias) {
return cubeTbls.get(alias);
}
public void addExprToAlias(ASTNode expr, ASTNode alias) {
exprToAlias.put(HQLParser.getString(expr).trim(), alias.getText().toLowerCase());
}
public void setJoinCond(QBJoinTree qb, String cond) {
joinConds.put(qb, cond);
}
public String getAlias(String expr) {
return exprToAlias.get(expr);
}