Esempio n. 1
0
 /**
  * destination table : a table whose columns are getting queried intermediate table : a table
  * which is only used as a link between cube and destination table
  *
  * @param alias
  * @param tblName
  * @param isOptional pass false when it's a destination table pass true when it's an intermediate
  *     table when join chain destination is being added, this will be false.
  * @param isChainedDimension pass true when you're adding the dimension as a joinchain
  *     destination, pass false when this table is mentioned by name in the user query
  * @return true if added
  * @throws LensException
  */
 private boolean addQueriedTable(
     String alias, String tblName, boolean isOptional, boolean isChainedDimension)
     throws LensException {
   alias = alias.toLowerCase();
   if (cubeTbls.containsKey(alias)) {
     return true;
   }
   try {
     if (metastoreClient.isCube(tblName)) {
       if (cube != null) {
         if (!cube.getName().equalsIgnoreCase(tblName)) {
           throw new LensException(
               LensCubeErrorCode.MORE_THAN_ONE_CUBE.getLensErrorInfo(), cube.getName(), tblName);
         }
       }
       cube = metastoreClient.getCube(tblName);
       cubeTbls.put(alias, (AbstractCubeTable) cube);
     } else if (metastoreClient.isDimension(tblName)) {
       Dimension dim = metastoreClient.getDimension(tblName);
       if (!isOptional) {
         dimensions.add(dim);
       }
       if (!isChainedDimension) {
         nonChainedDimensions.add(dim);
       }
       cubeTbls.put(alias, dim);
     } else {
       return false;
     }
   } catch (HiveException e) {
     return false;
   }
   return true;
 }
    public Set<Dimension> rewriteDenormctx(
        CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery, boolean replaceFact)
        throws LensException {
      Set<Dimension> refTbls = new HashSet<>();

      if (!tableToRefCols.isEmpty()) {
        // pick referenced columns for fact
        if (cfact != null) {
          pickColumnsForTable(cfact.getName());
        }
        // pick referenced columns for dimensions
        if (dimsToQuery != null && !dimsToQuery.isEmpty()) {
          for (CandidateDim cdim : dimsToQuery.values()) {
            pickColumnsForTable(cdim.getName());
          }
        }
        // Replace picked reference in all the base trees
        replaceReferencedColumns(cfact, replaceFact);

        // Add the picked references to dimsToQuery
        for (PickedReference picked : pickedRefs) {
          if (isPickedFor(picked, cfact, dimsToQuery)) {
            refTbls.add(
                (Dimension) cubeql.getCubeTableForAlias(picked.getChainRef().getChainName()));
            cubeql.addColumnsQueried(
                picked.getChainRef().getChainName(), picked.getChainRef().getRefColumn());
          }
        }
      }
      return refTbls;
    }
Esempio n. 3
0
 private String getStorageStringWithAlias(
     CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, String alias) {
   if (cubeTbls.get(alias) instanceof CubeInterface) {
     return fact.getStorageString(alias);
   } else {
     return dimsToQuery.get(cubeTbls.get(alias)).getStorageString(alias);
   }
 }
Esempio n. 4
0
  private void getQLString(
      QBJoinTree joinTree,
      StringBuilder builder,
      CandidateFact fact,
      Map<Dimension, CandidateDim> dimsToQuery)
      throws LensException {
    List<String> joiningTables = new ArrayList<>();
    if (joinTree.getBaseSrc()[0] == null) {
      if (joinTree.getJoinSrc() != null) {
        getQLString(joinTree.getJoinSrc(), builder, fact, dimsToQuery);
      }
    } else { // (joinTree.getBaseSrc()[0] != null){
      String alias = joinTree.getBaseSrc()[0].toLowerCase();
      builder.append(getStorageStringWithAlias(fact, dimsToQuery, alias));
      joiningTables.add(alias);
    }
    if (joinTree.getJoinCond() != null) {
      builder.append(JoinUtils.getJoinTypeStr(joinTree.getJoinCond()[0].getJoinType()));
      builder.append(" JOIN ");
    }
    if (joinTree.getBaseSrc()[1] == null) {
      if (joinTree.getJoinSrc() != null) {
        getQLString(joinTree.getJoinSrc(), builder, fact, dimsToQuery);
      }
    } else { // (joinTree.getBaseSrc()[1] != null){
      String alias = joinTree.getBaseSrc()[1].toLowerCase();
      builder.append(getStorageStringWithAlias(fact, dimsToQuery, alias));
      joiningTables.add(alias);
    }

    String joinCond = joinConds.get(joinTree);
    if (joinCond != null) {
      builder.append(" ON ");
      builder.append(joinCond);
      // joining tables will contain all tables involved in joins.
      // we need to push storage filters for Dimensions into join conditions, thus the following
      // code
      // takes care of the same.
      for (String joiningTable : joiningTables) {
        if (cubeTbls.get(joiningTable) instanceof Dimension) {
          DimOnlyHQLContext.appendWhereClause(
              builder, getWhereClauseWithAlias(dimsToQuery, joiningTable), true);
          dimsToQuery.get(cubeTbls.get(joiningTable)).setWhereClauseAdded(joiningTable);
        }
      }
    } else {
      throw new LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo());
    }
  }
Esempio n. 5
0
 private Map<Dimension, CandidateDim> pickCandidateDimsToQuery(Set<Dimension> dimensions)
     throws LensException {
   Map<Dimension, CandidateDim> dimsToQuery = new HashMap<Dimension, CandidateDim>();
   if (!dimensions.isEmpty()) {
     for (Dimension dim : dimensions) {
       if (candidateDims.get(dim) != null && candidateDims.get(dim).size() > 0) {
         CandidateDim cdim = candidateDims.get(dim).iterator().next();
         log.info(
             "Available candidate dims are:{}, picking up {} for querying.",
             candidateDims.get(dim),
             cdim.dimtable);
         dimsToQuery.put(dim, cdim);
       } else {
         String reason = "";
         if (dimPruningMsgs.get(dim) != null && !dimPruningMsgs.get(dim).isEmpty()) {
           ByteArrayOutputStream out = null;
           try {
             ObjectMapper mapper = new ObjectMapper();
             out = new ByteArrayOutputStream();
             mapper.writeValue(out, dimPruningMsgs.get(dim).getJsonObject());
             reason = out.toString("UTF-8");
           } catch (Exception e) {
             throw new LensException("Error writing dim pruning messages", e);
           } finally {
             if (out != null) {
               try {
                 out.close();
               } catch (IOException e) {
                 throw new LensException(e);
               }
             }
           }
         }
         log.error(
             "Query rewrite failed due to NO_CANDIDATE_DIM_AVAILABLE, Cause {}",
             dimPruningMsgs.get(dim).toJsonObject());
         throw new NoCandidateDimAvailableException(dimPruningMsgs.get(dim));
       }
     }
   }
   return dimsToQuery;
 }
 // checks if the reference if picked for facts and dimsToQuery passed
 private boolean isPickedFor(
     PickedReference picked, CandidateFact cfact, Map<Dimension, CandidateDim> dimsToQuery) {
   if (cfact != null && picked.pickedFor.equalsIgnoreCase(cfact.getName())) {
     return true;
   }
   if (dimsToQuery != null) {
     for (CandidateDim cdim : dimsToQuery.values()) {
       if (picked.pickedFor.equalsIgnoreCase(cdim.getName())) {
         return true;
       }
     }
   }
   return false;
 }
Esempio n. 7
0
 public String getAliasForTableName(String tableName) {
   for (String alias : qb.getTabAliases()) {
     String table = qb.getTabNameForAlias(alias);
     if (table != null && table.equalsIgnoreCase(tableName)) {
       return alias;
     }
   }
   // get alias from cubeTbls
   for (Map.Entry<String, AbstractCubeTable> cubeTblEntry : cubeTbls.entrySet()) {
     if (cubeTblEntry.getValue().getName().equalsIgnoreCase(tableName)) {
       return cubeTblEntry.getKey();
     }
   }
   return tableName.toLowerCase();
 }
Esempio n. 8
0
 private void addOptionalDimTable(
     String alias,
     CandidateTable candidate,
     boolean isRequiredInJoin,
     String cubeCol,
     boolean isRef,
     String tableAlias,
     String... cols)
     throws LensException {
   alias = alias.toLowerCase();
   if (!addQueriedTable(alias, true)) {
     throw new LensException(LensCubeErrorCode.QUERIED_TABLE_NOT_FOUND.getLensErrorInfo(), alias);
   }
   Dimension dim = (Dimension) cubeTbls.get(alias);
   Aliased<Dimension> aliasedDim = Aliased.create(dim, alias);
   OptionalDimCtx optDim = optionalDimensionMap.get(aliasedDim);
   if (optDim == null) {
     optDim = new OptionalDimCtx();
     optionalDimensionMap.put(aliasedDim, optDim);
   }
   if (cols != null && candidate != null) {
     for (String col : cols) {
       optDim.colQueried.add(col);
     }
     optDim.requiredForCandidates.add(candidate);
   }
   if (cubeCol != null) {
     if (isRef) {
       updateRefColDim(cubeCol, aliasedDim);
     } else {
       updateExprColDim(tableAlias, cubeCol, aliasedDim);
     }
   }
   if (!optDim.isRequiredInJoinChain) {
     optDim.isRequiredInJoinChain = isRequiredInJoin;
   }
   if (log.isDebugEnabled()) {
     log.debug(
         "Adding optional dimension:{} optDim:{} {} isRef:{}",
         aliasedDim,
         optDim,
         (cubeCol == null ? "" : " for column:" + cubeCol),
         isRef);
   }
 }
Esempio n. 9
0
  private boolean addJoinChain(String alias, boolean isOptional) throws LensException {
    boolean retVal = false;
    String aliasLowerCaseStr = alias.toLowerCase();
    JoinChain joinchain = null;

    if (getCube() != null) {
      JoinChain chainByName = getCube().getChainByName(aliasLowerCaseStr);
      if (chainByName != null) {
        joinchain = chainByName;
        retVal = true;
      }
    }

    if (!retVal) {
      for (Dimension table : dimensions) {
        JoinChain chainByName = table.getChainByName(aliasLowerCaseStr);
        if (chainByName != null) {
          joinchain = chainByName;
          retVal = true;
          break;
        }
      }
    }

    if (retVal) {
      joinchains.put(aliasLowerCaseStr, new JoinChain(joinchain));
      String destTable = joinchain.getDestTable();
      boolean added = addQueriedTable(alias, destTable, isOptional, true);
      if (!added) {
        log.info("Queried tables do not exist. Missing tables:{}", destTable);
        throw new LensException(LensCubeErrorCode.NEITHER_CUBE_NOR_DIMENSION.getLensErrorInfo());
      }
      log.info("Added join chain for {}", destTable);
      return true;
    }

    return retVal;
  }
Esempio n. 10
0
 String getQBFromString(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery)
     throws LensException {
   String fromString;
   if (getJoinAST() == null) {
     if (cube != null) {
       if (dimensions.size() > 0) {
         throw new LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo());
       }
       fromString = fact.getStorageString(getAliasForTableName(cube.getName()));
     } else {
       if (dimensions.size() != 1) {
         throw new LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo());
       }
       Dimension dim = dimensions.iterator().next();
       fromString = dimsToQuery.get(dim).getStorageString(getAliasForTableName(dim.getName()));
     }
   } else {
     StringBuilder builder = new StringBuilder();
     getQLString(qb.getQbJoinTree(), builder, fact, dimsToQuery);
     fromString = builder.toString();
   }
   return fromString;
 }
Esempio n. 11
0
  public String toHQL() throws LensException {
    Set<CandidateFact> cfacts = pickCandidateFactToQuery();
    Map<Dimension, CandidateDim> dimsToQuery = pickCandidateDimsToQuery(dimensions);
    log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery);
    if (autoJoinCtx != null) {
      // prune join paths for picked fact and dimensions
      autoJoinCtx.pruneAllPaths(cube, cfacts, dimsToQuery);
    }

    Map<CandidateFact, Set<Dimension>> factDimMap = new HashMap<>();
    if (cfacts != null) {
      if (cfacts.size() > 1) {
        // copy ASTs for each fact
        for (CandidateFact cfact : cfacts) {
          cfact.copyASTs(this);
          factDimMap.put(cfact, new HashSet<>(dimsToQuery.keySet()));
        }
      }
      for (CandidateFact fact : cfacts) {
        addRangeClauses(fact);
      }
    }

    // pick dimension tables required during expression expansion for the picked fact and dimensions
    Set<Dimension> exprDimensions = new HashSet<Dimension>();
    if (cfacts != null) {
      for (CandidateFact cfact : cfacts) {
        Set<Dimension> factExprDimTables =
            exprCtx.rewriteExprCtx(cfact, dimsToQuery, cfacts.size() > 1 ? cfact : this);
        exprDimensions.addAll(factExprDimTables);
        if (cfacts.size() > 1) {
          factDimMap.get(cfact).addAll(factExprDimTables);
        }
      }
      if (cfacts.size() > 1) {
        havingAST = MultiFactHQLContext.pushDownHaving(havingAST, this, cfacts);
      }
    } else {
      // dim only query
      exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, this));
    }
    dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions));
    log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery);

    // pick denorm tables for the picked fact and dimensions
    Set<Dimension> denormTables = new HashSet<Dimension>();
    if (cfacts != null) {
      for (CandidateFact cfact : cfacts) {
        Set<Dimension> factDenormTables =
            deNormCtx.rewriteDenormctx(cfact, dimsToQuery, cfacts.size() > 1);
        denormTables.addAll(factDenormTables);
        if (cfacts.size() > 1) {
          factDimMap.get(cfact).addAll(factDenormTables);
        }
      }
    } else {
      denormTables.addAll(deNormCtx.rewriteDenormctx(null, dimsToQuery, false));
    }
    dimsToQuery.putAll(pickCandidateDimsToQuery(denormTables));
    log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery);
    // Prune join paths once denorm tables are picked
    if (autoJoinCtx != null) {
      // prune join paths for picked fact and dimensions
      autoJoinCtx.pruneAllPaths(cube, cfacts, dimsToQuery);
    }
    if (autoJoinCtx != null) {
      // add optional dims from Join resolver
      Set<Dimension> joiningTables = new HashSet<Dimension>();
      if (cfacts != null && cfacts.size() > 1) {
        for (CandidateFact cfact : cfacts) {
          Set<Dimension> factJoiningTables =
              autoJoinCtx.pickOptionalTables(cfact, factDimMap.get(cfact), this);
          factDimMap.get(cfact).addAll(factJoiningTables);
          joiningTables.addAll(factJoiningTables);
        }
      } else {
        joiningTables.addAll(autoJoinCtx.pickOptionalTables(null, dimsToQuery.keySet(), this));
      }
      dimsToQuery.putAll(pickCandidateDimsToQuery(joiningTables));
    }
    log.info("Picked Fact:{} dimsToQuery: {}", cfacts, dimsToQuery);
    pickedDimTables = dimsToQuery.values();
    pickedFacts = cfacts;
    if (cfacts != null) {
      if (cfacts.size() > 1) {
        // Update ASTs for each fact
        for (CandidateFact cfact : cfacts) {
          cfact.updateASTs(this);
        }
        whereAST =
            MultiFactHQLContext.convertHavingToWhere(
                havingAST, this, cfacts, new DefaultAliasDecider());
      }
    }
    hqlContext = createHQLContext(cfacts, dimsToQuery, factDimMap);
    return hqlContext.toHQL();
  }
Esempio n. 12
0
 private String getWhereClauseWithAlias(Map<Dimension, CandidateDim> dimsToQuery, String alias) {
   return StorageUtil.getWhereClause(dimsToQuery.get(cubeTbls.get(alias)), alias);
 }
Esempio n. 13
0
 public AbstractCubeTable getCubeTableForAlias(String alias) {
   return cubeTbls.get(alias);
 }
Esempio n. 14
0
 public void addExprToAlias(ASTNode expr, ASTNode alias) {
   exprToAlias.put(HQLParser.getString(expr).trim(), alias.getText().toLowerCase());
 }
Esempio n. 15
0
 public void setJoinCond(QBJoinTree qb, String cond) {
   joinConds.put(qb, cond);
 }
Esempio n. 16
0
 public String getAlias(String expr) {
   return exprToAlias.get(expr);
 }