/** * destination table : a table whose columns are getting queried intermediate table : a table * which is only used as a link between cube and destination table * * @param alias * @param tblName * @param isOptional pass false when it's a destination table pass true when it's an intermediate * table when join chain destination is being added, this will be false. * @param isChainedDimension pass true when you're adding the dimension as a joinchain * destination, pass false when this table is mentioned by name in the user query * @return true if added * @throws LensException */ private boolean addQueriedTable( String alias, String tblName, boolean isOptional, boolean isChainedDimension) throws LensException { alias = alias.toLowerCase(); if (cubeTbls.containsKey(alias)) { return true; } try { if (metastoreClient.isCube(tblName)) { if (cube != null) { if (!cube.getName().equalsIgnoreCase(tblName)) { throw new LensException( LensCubeErrorCode.MORE_THAN_ONE_CUBE.getLensErrorInfo(), cube.getName(), tblName); } } cube = metastoreClient.getCube(tblName); cubeTbls.put(alias, (AbstractCubeTable) cube); } else if (metastoreClient.isDimension(tblName)) { Dimension dim = metastoreClient.getDimension(tblName); if (!isOptional) { dimensions.add(dim); } if (!isChainedDimension) { nonChainedDimensions.add(dim); } cubeTbls.put(alias, dim); } else { return false; } } catch (HiveException e) { return false; } return true; }
public Set<Dimension> rewriteDenormctx( CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery, boolean replaceFact) throws LensException { Set<Dimension> refTbls = new HashSet<>(); if (!tableToRefCols.isEmpty()) { // pick referenced columns for fact if (cfact != null) { pickColumnsForTable(cfact.getName()); } // pick referenced columns for dimensions if (dimsToQuery != null && !dimsToQuery.isEmpty()) { for (CandidateDim cdim : dimsToQuery.values()) { pickColumnsForTable(cdim.getName()); } } // Replace picked reference in all the base trees replaceReferencedColumns(cfact, replaceFact); // Add the picked references to dimsToQuery for (PickedReference picked : pickedRefs) { if (isPickedFor(picked, cfact, dimsToQuery)) { refTbls.add( (Dimension) cubeql.getCubeTableForAlias(picked.getChainRef().getChainName())); cubeql.addColumnsQueried( picked.getChainRef().getChainName(), picked.getChainRef().getRefColumn()); } } } return refTbls; }
private String getStorageStringWithAlias( CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, String alias) { if (cubeTbls.get(alias) instanceof CubeInterface) { return fact.getStorageString(alias); } else { return dimsToQuery.get(cubeTbls.get(alias)).getStorageString(alias); } }
private void getQLString( QBJoinTree joinTree, StringBuilder builder, CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery) throws LensException { List<String> joiningTables = new ArrayList<>(); if (joinTree.getBaseSrc()[0] == null) { if (joinTree.getJoinSrc() != null) { getQLString(joinTree.getJoinSrc(), builder, fact, dimsToQuery); } } else { // (joinTree.getBaseSrc()[0] != null){ String alias = joinTree.getBaseSrc()[0].toLowerCase(); builder.append(getStorageStringWithAlias(fact, dimsToQuery, alias)); joiningTables.add(alias); } if (joinTree.getJoinCond() != null) { builder.append(JoinUtils.getJoinTypeStr(joinTree.getJoinCond()[0].getJoinType())); builder.append(" JOIN "); } if (joinTree.getBaseSrc()[1] == null) { if (joinTree.getJoinSrc() != null) { getQLString(joinTree.getJoinSrc(), builder, fact, dimsToQuery); } } else { // (joinTree.getBaseSrc()[1] != null){ String alias = joinTree.getBaseSrc()[1].toLowerCase(); builder.append(getStorageStringWithAlias(fact, dimsToQuery, alias)); joiningTables.add(alias); } String joinCond = joinConds.get(joinTree); if (joinCond != null) { builder.append(" ON "); builder.append(joinCond); // joining tables will contain all tables involved in joins. // we need to push storage filters for Dimensions into join conditions, thus the following // code // takes care of the same. for (String joiningTable : joiningTables) { if (cubeTbls.get(joiningTable) instanceof Dimension) { DimOnlyHQLContext.appendWhereClause( builder, getWhereClauseWithAlias(dimsToQuery, joiningTable), true); dimsToQuery.get(cubeTbls.get(joiningTable)).setWhereClauseAdded(joiningTable); } } } else { throw new LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo()); } }
private Map<Dimension, CandidateDim> pickCandidateDimsToQuery(Set<Dimension> dimensions) throws LensException { Map<Dimension, CandidateDim> dimsToQuery = new HashMap<Dimension, CandidateDim>(); if (!dimensions.isEmpty()) { for (Dimension dim : dimensions) { if (candidateDims.get(dim) != null && candidateDims.get(dim).size() > 0) { CandidateDim cdim = candidateDims.get(dim).iterator().next(); log.info( "Available candidate dims are:{}, picking up {} for querying.", candidateDims.get(dim), cdim.dimtable); dimsToQuery.put(dim, cdim); } else { String reason = ""; if (dimPruningMsgs.get(dim) != null && !dimPruningMsgs.get(dim).isEmpty()) { ByteArrayOutputStream out = null; try { ObjectMapper mapper = new ObjectMapper(); out = new ByteArrayOutputStream(); mapper.writeValue(out, dimPruningMsgs.get(dim).getJsonObject()); reason = out.toString("UTF-8"); } catch (Exception e) { throw new LensException("Error writing dim pruning messages", e); } finally { if (out != null) { try { out.close(); } catch (IOException e) { throw new LensException(e); } } } } log.error( "Query rewrite failed due to NO_CANDIDATE_DIM_AVAILABLE, Cause {}", dimPruningMsgs.get(dim).toJsonObject()); throw new NoCandidateDimAvailableException(dimPruningMsgs.get(dim)); } } } return dimsToQuery; }
// checks if the reference if picked for facts and dimsToQuery passed private boolean isPickedFor( PickedReference picked, CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery) { if (cfact != null && picked.pickedFor.equalsIgnoreCase(cfact.getName())) { return true; } if (dimsToQuery != null) { for (CandidateDim cdim : dimsToQuery.values()) { if (picked.pickedFor.equalsIgnoreCase(cdim.getName())) { return true; } } } return false; }
public String getAliasForTableName(String tableName) { for (String alias : qb.getTabAliases()) { String table = qb.getTabNameForAlias(alias); if (table != null && table.equalsIgnoreCase(tableName)) { return alias; } } // get alias from cubeTbls for (Map.Entry<String, AbstractCubeTable> cubeTblEntry : cubeTbls.entrySet()) { if (cubeTblEntry.getValue().getName().equalsIgnoreCase(tableName)) { return cubeTblEntry.getKey(); } } return tableName.toLowerCase(); }
private void addOptionalDimTable( String alias, CandidateTable candidate, boolean isRequiredInJoin, String cubeCol, boolean isRef, String tableAlias, String... cols) throws LensException { alias = alias.toLowerCase(); if (!addQueriedTable(alias, true)) { throw new LensException(LensCubeErrorCode.QUERIED_TABLE_NOT_FOUND.getLensErrorInfo(), alias); } Dimension dim = (Dimension) cubeTbls.get(alias); Aliased<Dimension> aliasedDim = Aliased.create(dim, alias); OptionalDimCtx optDim = optionalDimensionMap.get(aliasedDim); if (optDim == null) { optDim = new OptionalDimCtx(); optionalDimensionMap.put(aliasedDim, optDim); } if (cols != null && candidate != null) { for (String col : cols) { optDim.colQueried.add(col); } optDim.requiredForCandidates.add(candidate); } if (cubeCol != null) { if (isRef) { updateRefColDim(cubeCol, aliasedDim); } else { updateExprColDim(tableAlias, cubeCol, aliasedDim); } } if (!optDim.isRequiredInJoinChain) { optDim.isRequiredInJoinChain = isRequiredInJoin; } if (log.isDebugEnabled()) { log.debug( "Adding optional dimension:{} optDim:{} {} isRef:{}", aliasedDim, optDim, (cubeCol == null ? "" : " for column:" + cubeCol), isRef); } }
private boolean addJoinChain(String alias, boolean isOptional) throws LensException { boolean retVal = false; String aliasLowerCaseStr = alias.toLowerCase(); JoinChain joinchain = null; if (getCube() != null) { JoinChain chainByName = getCube().getChainByName(aliasLowerCaseStr); if (chainByName != null) { joinchain = chainByName; retVal = true; } } if (!retVal) { for (Dimension table : dimensions) { JoinChain chainByName = table.getChainByName(aliasLowerCaseStr); if (chainByName != null) { joinchain = chainByName; retVal = true; break; } } } if (retVal) { joinchains.put(aliasLowerCaseStr, new JoinChain(joinchain)); String destTable = joinchain.getDestTable(); boolean added = addQueriedTable(alias, destTable, isOptional, true); if (!added) { log.info("Queried tables do not exist. Missing tables:{}", destTable); throw new LensException(LensCubeErrorCode.NEITHER_CUBE_NOR_DIMENSION.getLensErrorInfo()); } log.info("Added join chain for {}", destTable); return true; } return retVal; }
String getQBFromString(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery) throws LensException { String fromString; if (getJoinAST() == null) { if (cube != null) { if (dimensions.size() > 0) { throw new LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo()); } fromString = fact.getStorageString(getAliasForTableName(cube.getName())); } else { if (dimensions.size() != 1) { throw new LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo()); } Dimension dim = dimensions.iterator().next(); fromString = dimsToQuery.get(dim).getStorageString(getAliasForTableName(dim.getName())); } } else { StringBuilder builder = new StringBuilder(); getQLString(qb.getQbJoinTree(), builder, fact, dimsToQuery); fromString = builder.toString(); } return fromString; }
public String toHQL() throws LensException { Set<CandidateFact> cfacts = pickCandidateFactToQuery(); Map<Dimension, CandidateDim> dimsToQuery = pickCandidateDimsToQuery(dimensions); log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery); if (autoJoinCtx != null) { // prune join paths for picked fact and dimensions autoJoinCtx.pruneAllPaths(cube, cfacts, dimsToQuery); } Map<CandidateFact, Set<Dimension>> factDimMap = new HashMap<>(); if (cfacts != null) { if (cfacts.size() > 1) { // copy ASTs for each fact for (CandidateFact cfact : cfacts) { cfact.copyASTs(this); factDimMap.put(cfact, new HashSet<>(dimsToQuery.keySet())); } } for (CandidateFact fact : cfacts) { addRangeClauses(fact); } } // pick dimension tables required during expression expansion for the picked fact and dimensions Set<Dimension> exprDimensions = new HashSet<Dimension>(); if (cfacts != null) { for (CandidateFact cfact : cfacts) { Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(cfact, dimsToQuery, cfacts.size() > 1 ? cfact : this); exprDimensions.addAll(factExprDimTables); if (cfacts.size() > 1) { factDimMap.get(cfact).addAll(factExprDimTables); } } if (cfacts.size() > 1) { havingAST = MultiFactHQLContext.pushDownHaving(havingAST, this, cfacts); } } else { // dim only query exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, this)); } dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions)); log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery); // pick denorm tables for the picked fact and dimensions Set<Dimension> denormTables = new HashSet<Dimension>(); if (cfacts != null) { for (CandidateFact cfact : cfacts) { Set<Dimension> factDenormTables = deNormCtx.rewriteDenormctx(cfact, dimsToQuery, cfacts.size() > 1); denormTables.addAll(factDenormTables); if (cfacts.size() > 1) { factDimMap.get(cfact).addAll(factDenormTables); } } } else { denormTables.addAll(deNormCtx.rewriteDenormctx(null, dimsToQuery, false)); } dimsToQuery.putAll(pickCandidateDimsToQuery(denormTables)); log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery); // Prune join paths once denorm tables are picked if (autoJoinCtx != null) { // prune join paths for picked fact and dimensions autoJoinCtx.pruneAllPaths(cube, cfacts, dimsToQuery); } if (autoJoinCtx != null) { // add optional dims from Join resolver Set<Dimension> joiningTables = new HashSet<Dimension>(); if (cfacts != null && cfacts.size() > 1) { for (CandidateFact cfact : cfacts) { Set<Dimension> factJoiningTables = autoJoinCtx.pickOptionalTables(cfact, factDimMap.get(cfact), this); factDimMap.get(cfact).addAll(factJoiningTables); joiningTables.addAll(factJoiningTables); } } else { joiningTables.addAll(autoJoinCtx.pickOptionalTables(null, dimsToQuery.keySet(), this)); } dimsToQuery.putAll(pickCandidateDimsToQuery(joiningTables)); } log.info("Picked Fact:{} dimsToQuery: {}", cfacts, dimsToQuery); pickedDimTables = dimsToQuery.values(); pickedFacts = cfacts; if (cfacts != null) { if (cfacts.size() > 1) { // Update ASTs for each fact for (CandidateFact cfact : cfacts) { cfact.updateASTs(this); } whereAST = MultiFactHQLContext.convertHavingToWhere( havingAST, this, cfacts, new DefaultAliasDecider()); } } hqlContext = createHQLContext(cfacts, dimsToQuery, factDimMap); return hqlContext.toHQL(); }
private String getWhereClauseWithAlias(Map<Dimension, CandidateDim> dimsToQuery, String alias) { return StorageUtil.getWhereClause(dimsToQuery.get(cubeTbls.get(alias)), alias); }
public AbstractCubeTable getCubeTableForAlias(String alias) { return cubeTbls.get(alias); }
public void addExprToAlias(ASTNode expr, ASTNode alias) { exprToAlias.put(HQLParser.getString(expr).trim(), alias.getText().toLowerCase()); }
public void setJoinCond(QBJoinTree qb, String cond) { joinConds.put(qb, cond); }
public String getAlias(String expr) { return exprToAlias.get(expr); }