Пример #1
0
  // Append a flattened version of this plan node, including all children, to 'container'.
  private void treeToThriftHelper(TPlan container) {
    TPlanNode msg = new TPlanNode();
    msg.node_id = id_.asInt();
    msg.limit = limit_;

    TExecStats estimatedStats = new TExecStats();
    estimatedStats.setCardinality(cardinality_);
    estimatedStats.setMemory_used(perHostMemCost_);
    msg.setLabel(getDisplayLabel());
    msg.setLabel_detail(getDisplayLabelDetail());
    msg.setEstimated_stats(estimatedStats);

    msg.setRow_tuples(Lists.<Integer>newArrayListWithCapacity(tupleIds_.size()));
    msg.setNullable_tuples(Lists.<Boolean>newArrayListWithCapacity(tupleIds_.size()));
    for (TupleId tid : tupleIds_) {
      msg.addToRow_tuples(tid.asInt());
      msg.addToNullable_tuples(nullableTupleIds_.contains(tid));
    }
    for (Expr e : conjuncts_) {
      msg.addToConjuncts(e.treeToThrift());
    }
    toThrift(msg);
    container.addToNodes(msg);
    // For the purpose of the BE consider ExchangeNodes to have no children.
    if (this instanceof ExchangeNode) {
      msg.num_children = 0;
      return;
    } else {
      msg.num_children = children_.size();
      for (PlanNode child : children_) {
        child.treeToThriftHelper(container);
      }
    }
  }
Пример #2
0
 /**
  * The input cardinality is the sum of output cardinalities of its children. For scan nodes the
  * input cardinality is the expected number of rows scanned.
  */
 public long getInputCardinality() {
   long sum = 0;
   for (PlanNode p : children_) {
     long tmp = p.getCardinality();
     if (tmp == -1) return -1;
     sum = addCardinalities(sum, tmp);
   }
   return sum;
 }
Пример #3
0
  /** Create SortInfo, including sort tuple, to sort entire input row on sortExprs. */
  private SortInfo createSortInfo(
      PlanNode input, List<Expr> sortExprs, List<Boolean> isAsc, List<Boolean> nullsFirst) {
    // create tuple for sort output = the entire materialized input in a single tuple
    TupleDescriptor sortTupleDesc = analyzer_.getDescTbl().createTupleDescriptor("sort-tuple");
    ExprSubstitutionMap sortSmap = new ExprSubstitutionMap();
    List<Expr> sortSlotExprs = Lists.newArrayList();
    sortTupleDesc.setIsMaterialized(true);
    for (TupleId tid : input.getTupleIds()) {
      TupleDescriptor tupleDesc = analyzer_.getTupleDesc(tid);
      for (SlotDescriptor inputSlotDesc : tupleDesc.getSlots()) {
        if (!inputSlotDesc.isMaterialized()) continue;
        SlotDescriptor sortSlotDesc = analyzer_.copySlotDescriptor(inputSlotDesc, sortTupleDesc);
        // all output slots need to be materialized
        sortSlotDesc.setIsMaterialized(true);
        sortSmap.put(new SlotRef(inputSlotDesc), new SlotRef(sortSlotDesc));
        sortSlotExprs.add(new SlotRef(inputSlotDesc));
      }
    }

    SortInfo sortInfo =
        new SortInfo(Expr.substituteList(sortExprs, sortSmap, analyzer_, false), isAsc, nullsFirst);
    LOG.trace("sortinfo exprs: " + Expr.debugString(sortInfo.getOrderingExprs()));
    sortInfo.setMaterializedTupleInfo(sortTupleDesc, sortSlotExprs);
    return sortInfo;
  }
Пример #4
0
  @Override
  public void getMaterializedIds(List<SlotId> ids) {
    super.getMaterializedIds(ids);

    // we indirectly reference all grouping slots (because we write them)
    // so they're all materialized.
    aggInfo.getRefdSlots(ids);
  }
Пример #5
0
  /**
   * Return plan tree that augments 'root' with plan nodes that implement single-node evaluation of
   * the AnalyticExprs in analyticInfo. This plan takes into account a possible hash partition of
   * its input on 'groupingExprs'; if this is non-null, it returns in 'inputPartitionExprs' a subset
   * of the grouping exprs which should be used for the aggregate hash partitioning during the
   * parallelization of 'root'. TODO: when generating sort orders for the sort groups, optimize the
   * ordering of the partition exprs (so that subsequent sort operations see the input sorted on a
   * prefix of their required sort exprs) TODO: when merging sort groups, recognize equivalent exprs
   * (using the equivalence classes) rather than looking for expr equality
   */
  public PlanNode createSingleNodePlan(
      PlanNode root, List<Expr> groupingExprs, List<Expr> inputPartitionExprs)
      throws ImpalaException {
    List<WindowGroup> windowGroups = collectWindowGroups();
    for (int i = 0; i < windowGroups.size(); ++i) {
      windowGroups.get(i).init(analyzer_, "wg-" + i);
    }
    List<SortGroup> sortGroups = collectSortGroups(windowGroups);
    mergeSortGroups(sortGroups);
    for (SortGroup g : sortGroups) {
      g.init();
    }
    List<PartitionGroup> partitionGroups = collectPartitionGroups(sortGroups);
    mergePartitionGroups(partitionGroups, root.getNumNodes());
    orderGroups(partitionGroups);
    if (groupingExprs != null) {
      Preconditions.checkNotNull(inputPartitionExprs);
      computeInputPartitionExprs(
          partitionGroups, groupingExprs, root.getNumNodes(), inputPartitionExprs);
    }

    for (PartitionGroup partitionGroup : partitionGroups) {
      for (int i = 0; i < partitionGroup.sortGroups.size(); ++i) {
        root =
            createSortGroupPlan(
                root,
                partitionGroup.sortGroups.get(i),
                i == 0 ? partitionGroup.partitionByExprs : null);
      }
    }

    // create equiv classes for newly added slots
    analyzer_.createIdentityEquivClasses();

    return root;
  }
Пример #6
0
  /**
   * Create plan tree for the entire sort group, including all contained window groups. Marks the
   * SortNode as requiring its input to be partitioned if partitionExprs is not null (partitionExprs
   * represent the data partition of the entire partition group of which this sort group is a part).
   */
  private PlanNode createSortGroupPlan(
      PlanNode root, SortGroup sortGroup, List<Expr> partitionExprs) throws ImpalaException {
    List<Expr> partitionByExprs = sortGroup.partitionByExprs;
    List<OrderByElement> orderByElements = sortGroup.orderByElements;
    ExprSubstitutionMap sortSmap = null;
    TupleId sortTupleId = null;
    TupleDescriptor bufferedTupleDesc = null;
    // map from input to buffered tuple
    ExprSubstitutionMap bufferedSmap = new ExprSubstitutionMap();

    // sort on partition by (pb) + order by (ob) exprs and create pb/ob predicates
    if (!partitionByExprs.isEmpty() || !orderByElements.isEmpty()) {
      // first sort on partitionExprs (direction doesn't matter)
      List<Expr> sortExprs = Lists.newArrayList(partitionByExprs);
      List<Boolean> isAsc =
          Lists.newArrayList(Collections.nCopies(sortExprs.size(), new Boolean(true)));
      // TODO: utilize a direction and nulls/first last that has benefit
      // for subsequent sort groups
      List<Boolean> nullsFirst =
          Lists.newArrayList(Collections.nCopies(sortExprs.size(), new Boolean(true)));

      // then sort on orderByExprs
      for (OrderByElement orderByElement : sortGroup.orderByElements) {
        sortExprs.add(orderByElement.getExpr());
        isAsc.add(orderByElement.isAsc());
        nullsFirst.add(orderByElement.getNullsFirstParam());
      }

      SortInfo sortInfo = createSortInfo(root, sortExprs, isAsc, nullsFirst);
      SortNode sortNode = new SortNode(idGenerator_.getNextId(), root, sortInfo, false, 0);

      // if this sort group does not have partitioning exprs, we want the sort
      // to be executed like a regular distributed sort
      if (!partitionByExprs.isEmpty()) sortNode.setIsAnalyticSort(true);

      if (partitionExprs != null) {
        // create required input partition
        DataPartition inputPartition = DataPartition.UNPARTITIONED;
        if (!partitionExprs.isEmpty()) {
          inputPartition = new DataPartition(TPartitionType.HASH_PARTITIONED, partitionExprs);
        }
        sortNode.setInputPartition(inputPartition);
      }

      root = sortNode;
      root.init(analyzer_);
      sortSmap = sortNode.getOutputSmap();

      // create bufferedTupleDesc and bufferedSmap
      sortTupleId = sortNode.tupleIds_.get(0);
      bufferedTupleDesc = analyzer_.getDescTbl().copyTupleDescriptor(sortTupleId, "buffered-tuple");
      LOG.trace("desctbl: " + analyzer_.getDescTbl().debugString());

      List<SlotDescriptor> inputSlots = analyzer_.getTupleDesc(sortTupleId).getSlots();
      List<SlotDescriptor> bufferedSlots = bufferedTupleDesc.getSlots();
      for (int i = 0; i < inputSlots.size(); ++i) {
        bufferedSmap.put(new SlotRef(inputSlots.get(i)), new SlotRef(bufferedSlots.get(i)));
      }
    }

    // create one AnalyticEvalNode per window group
    for (WindowGroup windowGroup : sortGroup.windowGroups) {
      // Create partition-by (pb) and order-by (ob) less-than predicates between the
      // input tuple (the output of the preceding sort) and a buffered tuple that is
      // identical to the input tuple. We need a different tuple descriptor for the
      // buffered tuple because the generated predicates should compare two different
      // tuple instances from the same input stream (i.e., the predicates should be
      // evaluated over a row that is composed of the input and the buffered tuple).

      // we need to remap the pb/ob exprs to a) the sort output, b) our buffer of the
      // sort input
      Expr partitionByEq = null;
      if (!windowGroup.partitionByExprs.isEmpty()) {
        partitionByEq =
            createNullMatchingEquals(
                Expr.substituteList(windowGroup.partitionByExprs, sortSmap, analyzer_, false),
                sortTupleId,
                bufferedSmap);
        LOG.trace("partitionByEq: " + partitionByEq.debugString());
      }
      Expr orderByEq = null;
      if (!windowGroup.orderByElements.isEmpty()) {
        orderByEq =
            createNullMatchingEquals(
                OrderByElement.getOrderByExprs(
                    OrderByElement.substitute(windowGroup.orderByElements, sortSmap, analyzer_)),
                sortTupleId,
                bufferedSmap);
        LOG.trace("orderByEq: " + orderByEq.debugString());
      }

      root =
          new AnalyticEvalNode(
              idGenerator_.getNextId(),
              root,
              stmtTupleIds_,
              windowGroup.analyticFnCalls,
              windowGroup.partitionByExprs,
              windowGroup.orderByElements,
              windowGroup.window,
              analyticInfo_.getOutputTupleDesc(),
              windowGroup.physicalIntermediateTuple,
              windowGroup.physicalOutputTuple,
              windowGroup.logicalToPhysicalSmap,
              partitionByEq,
              orderByEq,
              bufferedTupleDesc);
      root.init(analyzer_);
    }
    return root;
  }