// Append a flattened version of this plan node, including all children, to 'container'.
private void treeToThriftHelper(TPlan container) {
TPlanNode msg = new TPlanNode();
msg.node_id = id_.asInt();
msg.limit = limit_;
TExecStats estimatedStats = new TExecStats();
estimatedStats.setCardinality(cardinality_);
estimatedStats.setMemory_used(perHostMemCost_);
msg.setLabel(getDisplayLabel());
msg.setLabel_detail(getDisplayLabelDetail());
msg.setEstimated_stats(estimatedStats);
msg.setRow_tuples(Lists.<Integer>newArrayListWithCapacity(tupleIds_.size()));
msg.setNullable_tuples(Lists.<Boolean>newArrayListWithCapacity(tupleIds_.size()));
for (TupleId tid : tupleIds_) {
msg.addToRow_tuples(tid.asInt());
msg.addToNullable_tuples(nullableTupleIds_.contains(tid));
}
for (Expr e : conjuncts_) {
msg.addToConjuncts(e.treeToThrift());
}
toThrift(msg);
container.addToNodes(msg);
// For the purpose of the BE consider ExchangeNodes to have no children.
if (this instanceof ExchangeNode) {
msg.num_children = 0;
return;
} else {
msg.num_children = children_.size();
for (PlanNode child : children_) {
child.treeToThriftHelper(container);
}
}
}
/**
* The input cardinality is the sum of output cardinalities of its children. For scan nodes the
* input cardinality is the expected number of rows scanned.
*/
public long getInputCardinality() {
long sum = 0;
for (PlanNode p : children_) {
long tmp = p.getCardinality();
if (tmp == -1) return -1;
sum = addCardinalities(sum, tmp);
}
return sum;
}
/** Create SortInfo, including sort tuple, to sort entire input row on sortExprs. */
private SortInfo createSortInfo(
PlanNode input, List<Expr> sortExprs, List<Boolean> isAsc, List<Boolean> nullsFirst) {
// create tuple for sort output = the entire materialized input in a single tuple
TupleDescriptor sortTupleDesc = analyzer_.getDescTbl().createTupleDescriptor("sort-tuple");
ExprSubstitutionMap sortSmap = new ExprSubstitutionMap();
List<Expr> sortSlotExprs = Lists.newArrayList();
sortTupleDesc.setIsMaterialized(true);
for (TupleId tid : input.getTupleIds()) {
TupleDescriptor tupleDesc = analyzer_.getTupleDesc(tid);
for (SlotDescriptor inputSlotDesc : tupleDesc.getSlots()) {
if (!inputSlotDesc.isMaterialized()) continue;
SlotDescriptor sortSlotDesc = analyzer_.copySlotDescriptor(inputSlotDesc, sortTupleDesc);
// all output slots need to be materialized
sortSlotDesc.setIsMaterialized(true);
sortSmap.put(new SlotRef(inputSlotDesc), new SlotRef(sortSlotDesc));
sortSlotExprs.add(new SlotRef(inputSlotDesc));
}
}
SortInfo sortInfo =
new SortInfo(Expr.substituteList(sortExprs, sortSmap, analyzer_, false), isAsc, nullsFirst);
LOG.trace("sortinfo exprs: " + Expr.debugString(sortInfo.getOrderingExprs()));
sortInfo.setMaterializedTupleInfo(sortTupleDesc, sortSlotExprs);
return sortInfo;
}
@Override
public void getMaterializedIds(List<SlotId> ids) {
super.getMaterializedIds(ids);
// we indirectly reference all grouping slots (because we write them)
// so they're all materialized.
aggInfo.getRefdSlots(ids);
}
/**
* Return plan tree that augments 'root' with plan nodes that implement single-node evaluation of
* the AnalyticExprs in analyticInfo. This plan takes into account a possible hash partition of
* its input on 'groupingExprs'; if this is non-null, it returns in 'inputPartitionExprs' a subset
* of the grouping exprs which should be used for the aggregate hash partitioning during the
* parallelization of 'root'. TODO: when generating sort orders for the sort groups, optimize the
* ordering of the partition exprs (so that subsequent sort operations see the input sorted on a
* prefix of their required sort exprs) TODO: when merging sort groups, recognize equivalent exprs
* (using the equivalence classes) rather than looking for expr equality
*/
public PlanNode createSingleNodePlan(
PlanNode root, List<Expr> groupingExprs, List<Expr> inputPartitionExprs)
throws ImpalaException {
List<WindowGroup> windowGroups = collectWindowGroups();
for (int i = 0; i < windowGroups.size(); ++i) {
windowGroups.get(i).init(analyzer_, "wg-" + i);
}
List<SortGroup> sortGroups = collectSortGroups(windowGroups);
mergeSortGroups(sortGroups);
for (SortGroup g : sortGroups) {
g.init();
}
List<PartitionGroup> partitionGroups = collectPartitionGroups(sortGroups);
mergePartitionGroups(partitionGroups, root.getNumNodes());
orderGroups(partitionGroups);
if (groupingExprs != null) {
Preconditions.checkNotNull(inputPartitionExprs);
computeInputPartitionExprs(
partitionGroups, groupingExprs, root.getNumNodes(), inputPartitionExprs);
}
for (PartitionGroup partitionGroup : partitionGroups) {
for (int i = 0; i < partitionGroup.sortGroups.size(); ++i) {
root =
createSortGroupPlan(
root,
partitionGroup.sortGroups.get(i),
i == 0 ? partitionGroup.partitionByExprs : null);
}
}
// create equiv classes for newly added slots
analyzer_.createIdentityEquivClasses();
return root;
}
/**
* Create plan tree for the entire sort group, including all contained window groups. Marks the
* SortNode as requiring its input to be partitioned if partitionExprs is not null (partitionExprs
* represent the data partition of the entire partition group of which this sort group is a part).
*/
private PlanNode createSortGroupPlan(
PlanNode root, SortGroup sortGroup, List<Expr> partitionExprs) throws ImpalaException {
List<Expr> partitionByExprs = sortGroup.partitionByExprs;
List<OrderByElement> orderByElements = sortGroup.orderByElements;
ExprSubstitutionMap sortSmap = null;
TupleId sortTupleId = null;
TupleDescriptor bufferedTupleDesc = null;
// map from input to buffered tuple
ExprSubstitutionMap bufferedSmap = new ExprSubstitutionMap();
// sort on partition by (pb) + order by (ob) exprs and create pb/ob predicates
if (!partitionByExprs.isEmpty() || !orderByElements.isEmpty()) {
// first sort on partitionExprs (direction doesn't matter)
List<Expr> sortExprs = Lists.newArrayList(partitionByExprs);
List<Boolean> isAsc =
Lists.newArrayList(Collections.nCopies(sortExprs.size(), new Boolean(true)));
// TODO: utilize a direction and nulls/first last that has benefit
// for subsequent sort groups
List<Boolean> nullsFirst =
Lists.newArrayList(Collections.nCopies(sortExprs.size(), new Boolean(true)));
// then sort on orderByExprs
for (OrderByElement orderByElement : sortGroup.orderByElements) {
sortExprs.add(orderByElement.getExpr());
isAsc.add(orderByElement.isAsc());
nullsFirst.add(orderByElement.getNullsFirstParam());
}
SortInfo sortInfo = createSortInfo(root, sortExprs, isAsc, nullsFirst);
SortNode sortNode = new SortNode(idGenerator_.getNextId(), root, sortInfo, false, 0);
// if this sort group does not have partitioning exprs, we want the sort
// to be executed like a regular distributed sort
if (!partitionByExprs.isEmpty()) sortNode.setIsAnalyticSort(true);
if (partitionExprs != null) {
// create required input partition
DataPartition inputPartition = DataPartition.UNPARTITIONED;
if (!partitionExprs.isEmpty()) {
inputPartition = new DataPartition(TPartitionType.HASH_PARTITIONED, partitionExprs);
}
sortNode.setInputPartition(inputPartition);
}
root = sortNode;
root.init(analyzer_);
sortSmap = sortNode.getOutputSmap();
// create bufferedTupleDesc and bufferedSmap
sortTupleId = sortNode.tupleIds_.get(0);
bufferedTupleDesc = analyzer_.getDescTbl().copyTupleDescriptor(sortTupleId, "buffered-tuple");
LOG.trace("desctbl: " + analyzer_.getDescTbl().debugString());
List<SlotDescriptor> inputSlots = analyzer_.getTupleDesc(sortTupleId).getSlots();
List<SlotDescriptor> bufferedSlots = bufferedTupleDesc.getSlots();
for (int i = 0; i < inputSlots.size(); ++i) {
bufferedSmap.put(new SlotRef(inputSlots.get(i)), new SlotRef(bufferedSlots.get(i)));
}
}
// create one AnalyticEvalNode per window group
for (WindowGroup windowGroup : sortGroup.windowGroups) {
// Create partition-by (pb) and order-by (ob) less-than predicates between the
// input tuple (the output of the preceding sort) and a buffered tuple that is
// identical to the input tuple. We need a different tuple descriptor for the
// buffered tuple because the generated predicates should compare two different
// tuple instances from the same input stream (i.e., the predicates should be
// evaluated over a row that is composed of the input and the buffered tuple).
// we need to remap the pb/ob exprs to a) the sort output, b) our buffer of the
// sort input
Expr partitionByEq = null;
if (!windowGroup.partitionByExprs.isEmpty()) {
partitionByEq =
createNullMatchingEquals(
Expr.substituteList(windowGroup.partitionByExprs, sortSmap, analyzer_, false),
sortTupleId,
bufferedSmap);
LOG.trace("partitionByEq: " + partitionByEq.debugString());
}
Expr orderByEq = null;
if (!windowGroup.orderByElements.isEmpty()) {
orderByEq =
createNullMatchingEquals(
OrderByElement.getOrderByExprs(
OrderByElement.substitute(windowGroup.orderByElements, sortSmap, analyzer_)),
sortTupleId,
bufferedSmap);
LOG.trace("orderByEq: " + orderByEq.debugString());
}
root =
new AnalyticEvalNode(
idGenerator_.getNextId(),
root,
stmtTupleIds_,
windowGroup.analyticFnCalls,
windowGroup.partitionByExprs,
windowGroup.orderByElements,
windowGroup.window,
analyticInfo_.getOutputTupleDesc(),
windowGroup.physicalIntermediateTuple,
windowGroup.physicalOutputTuple,
windowGroup.logicalToPhysicalSmap,
partitionByEq,
orderByEq,
bufferedTupleDesc);
root.init(analyzer_);
}
return root;
}
