@Override protected List<RequestedGlobalProperties> createPossibleGlobalProperties() { RequestedGlobalProperties props = new RequestedGlobalProperties(); if (customPartitioner == null) { props.setAnyPartitioning(this.keys); } else { props.setCustomPartitioned(this.keys, this.customPartitioner); } return Collections.singletonList(props); }
@Override public void computeInterestingPropertiesForInputs(CostEstimator estimator) { // get what we inherit and what is preserved by our user code final InterestingProperties props = getInterestingProperties().filterByCodeAnnotations(this, 0); // add all properties relevant to this node for (OperatorDescriptorSingle dps : getPossibleProperties()) { for (RequestedGlobalProperties gp : dps.getPossibleGlobalProperties()) { if (gp.getPartitioning().isPartitionedOnKey()) { // make sure that among the same partitioning types, we do not push anything down that has // fewer key fields for (RequestedGlobalProperties contained : props.getGlobalProperties()) { if (contained.getPartitioning() == gp.getPartitioning() && gp.getPartitionedFields().isValidSubset(contained.getPartitionedFields())) { props.getGlobalProperties().remove(contained); break; } } } props.addGlobalProperties(gp); } for (RequestedLocalProperties lp : dps.getPossibleLocalProperties()) { props.addLocalProperties(lp); } } this.inConn.setInterestingProperties(props); for (DagConnection conn : getBroadcastConnections()) { conn.setInterestingProperties(new InterestingProperties()); } }
@Override public List<PlanNode> getAlternativePlans(CostEstimator estimator) { // check if we have a cached version if (this.cachedPlans != null) { return this.cachedPlans; } boolean childrenSkippedDueToReplicatedInput = false; // calculate alternative sub-plans for predecessor final List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator); final Set<RequestedGlobalProperties> intGlobal = this.inConn.getInterestingProperties().getGlobalProperties(); // calculate alternative sub-plans for broadcast inputs final List<Set<? extends NamedChannel>> broadcastPlanChannels = new ArrayList<Set<? extends NamedChannel>>(); List<DagConnection> broadcastConnections = getBroadcastConnections(); List<String> broadcastConnectionNames = getBroadcastConnectionNames(); for (int i = 0; i < broadcastConnections.size(); i++) { DagConnection broadcastConnection = broadcastConnections.get(i); String broadcastConnectionName = broadcastConnectionNames.get(i); List<PlanNode> broadcastPlanCandidates = broadcastConnection.getSource().getAlternativePlans(estimator); // wrap the plan candidates in named channels HashSet<NamedChannel> broadcastChannels = new HashSet<NamedChannel>(broadcastPlanCandidates.size()); for (PlanNode plan : broadcastPlanCandidates) { NamedChannel c = new NamedChannel(broadcastConnectionName, plan); DataExchangeMode exMode = DataExchangeMode.select( broadcastConnection.getDataExchangeMode(), ShipStrategyType.BROADCAST, broadcastConnection.isBreakingPipeline()); c.setShipStrategy(ShipStrategyType.BROADCAST, exMode); broadcastChannels.add(c); } broadcastPlanChannels.add(broadcastChannels); } final RequestedGlobalProperties[] allValidGlobals; { Set<RequestedGlobalProperties> pairs = new HashSet<RequestedGlobalProperties>(); for (OperatorDescriptorSingle ods : getPossibleProperties()) { pairs.addAll(ods.getPossibleGlobalProperties()); } allValidGlobals = pairs.toArray(new RequestedGlobalProperties[pairs.size()]); } final ArrayList<PlanNode> outputPlans = new ArrayList<PlanNode>(); final ExecutionMode executionMode = this.inConn.getDataExchangeMode(); final int parallelism = getParallelism(); final int inParallelism = getPredecessorNode().getParallelism(); final boolean parallelismChange = inParallelism != parallelism; final boolean breaksPipeline = this.inConn.isBreakingPipeline(); // create all candidates for (PlanNode child : subPlans) { if (child.getGlobalProperties().isFullyReplicated()) { // fully replicated input is always locally forwarded if the parallelism is not changed if (parallelismChange) { // can not continue with this child childrenSkippedDueToReplicatedInput = true; continue; } else { this.inConn.setShipStrategy(ShipStrategyType.FORWARD); } } if (this.inConn.getShipStrategy() == null) { // pick the strategy ourselves for (RequestedGlobalProperties igps : intGlobal) { final Channel c = new Channel(child, this.inConn.getMaterializationMode()); igps.parameterizeChannel(c, parallelismChange, executionMode, breaksPipeline); // if the parallelism changed, make sure that we cancel out properties, unless the // ship strategy preserves/establishes them even under changing parallelisms if (parallelismChange && !c.getShipStrategy().isNetworkStrategy()) { c.getGlobalProperties().reset(); } // check whether we meet any of the accepted properties // we may remove this check, when we do a check to not inherit // requested global properties that are incompatible with all possible // requested properties for (RequestedGlobalProperties rgps : allValidGlobals) { if (rgps.isMetBy(c.getGlobalProperties())) { c.setRequiredGlobalProps(rgps); addLocalCandidates(c, broadcastPlanChannels, igps, outputPlans, estimator); break; } } } } else { // hint fixed the strategy final Channel c = new Channel(child, this.inConn.getMaterializationMode()); final ShipStrategyType shipStrategy = this.inConn.getShipStrategy(); final DataExchangeMode exMode = DataExchangeMode.select(executionMode, shipStrategy, breaksPipeline); if (this.keys != null) { c.setShipStrategy(shipStrategy, this.keys.toFieldList(), exMode); } else { c.setShipStrategy(shipStrategy, exMode); } if (parallelismChange) { c.adjustGlobalPropertiesForFullParallelismChange(); } // check whether we meet any of the accepted properties for (RequestedGlobalProperties rgps : allValidGlobals) { if (rgps.isMetBy(c.getGlobalProperties())) { addLocalCandidates(c, broadcastPlanChannels, rgps, outputPlans, estimator); break; } } } } if (outputPlans.isEmpty()) { if (childrenSkippedDueToReplicatedInput) { throw new CompilerException( "No plan meeting the requirements could be created @ " + this + ". Most likely reason: Invalid use of replicated input."); } else { throw new CompilerException( "No plan meeting the requirements could be created @ " + this + ". Most likely reason: Too restrictive plan hints."); } } // cost and prune the plans for (PlanNode node : outputPlans) { estimator.costOperator(node); } prunePlanAlternatives(outputPlans); outputPlans.trimToSize(); this.cachedPlans = outputPlans; return outputPlans; }