protected void addLocalCandidates(
Channel template,
List<Set<? extends NamedChannel>> broadcastPlanChannels,
RequestedGlobalProperties rgps,
List<PlanNode> target,
CostEstimator estimator) {
for (RequestedLocalProperties ilp :
this.inConn.getInterestingProperties().getLocalProperties()) {
final Channel in = template.clone();
ilp.parameterizeChannel(in);
// instantiate a candidate, if the instantiated local properties meet one possible local
// property set
outer:
for (OperatorDescriptorSingle dps : getPossibleProperties()) {
for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
if (ilps.isMetBy(in.getLocalProperties())) {
in.setRequiredLocalProps(ilps);
instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
break outer;
}
}
}
}
}
protected void instantiateCandidate(
OperatorDescriptorSingle dps,
Channel in,
List<Set<? extends NamedChannel>> broadcastPlanChannels,
List<PlanNode> target,
CostEstimator estimator,
RequestedGlobalProperties globPropsReq,
RequestedLocalProperties locPropsReq) {
final PlanNode inputSource = in.getSource();
for (List<NamedChannel> broadcastChannelsCombination :
Sets.cartesianProduct(broadcastPlanChannels)) {
boolean validCombination = true;
boolean requiresPipelinebreaker = false;
// check whether the broadcast inputs use the same plan candidate at the branching point
for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
NamedChannel nc = broadcastChannelsCombination.get(i);
PlanNode bcSource = nc.getSource();
// check branch compatibility against input
if (!areBranchCompatible(bcSource, inputSource)) {
validCombination = false;
break;
}
// check branch compatibility against all other broadcast variables
for (int k = 0; k < i; k++) {
PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
if (!areBranchCompatible(bcSource, otherBcSource)) {
validCombination = false;
break;
}
}
// check if there is a common predecessor and whether there is a dam on the way to all
// common predecessors
if (in.isOnDynamicPath() && this.hereJoinedBranches != null) {
for (OptimizerNode brancher : this.hereJoinedBranches) {
PlanNode candAtBrancher = in.getSource().getCandidateAtBranchPoint(brancher);
if (candAtBrancher == null) {
// closed branch between two broadcast variables
continue;
}
SourceAndDamReport res = in.getSource().hasDamOnPathDownTo(candAtBrancher);
if (res == NOT_FOUND) {
throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
} else if (res == FOUND_SOURCE) {
requiresPipelinebreaker = true;
break;
} else if (res == FOUND_SOURCE_AND_DAM) {
// good
} else {
throw new CompilerException();
}
}
}
}
if (!validCombination) {
continue;
}
if (requiresPipelinebreaker) {
in.setTempMode(in.getTempMode().makePipelineBreaker());
}
final SingleInputPlanNode node = dps.instantiate(in, this);
node.setBroadcastInputs(broadcastChannelsCombination);
// compute how the strategy affects the properties
GlobalProperties gProps = in.getGlobalProperties().clone();
LocalProperties lProps = in.getLocalProperties().clone();
gProps = dps.computeGlobalProperties(gProps);
lProps = dps.computeLocalProperties(lProps);
// filter by the user code field copies
gProps =
gProps.filterBySemanticProperties(getSemanticPropertiesForGlobalPropertyFiltering(), 0);
lProps =
lProps.filterBySemanticProperties(getSemanticPropertiesForLocalPropertyFiltering(), 0);
// apply
node.initProperties(gProps, lProps);
node.updatePropertiesWithUniqueSets(getUniqueFields());
target.add(node);
}
}
@Override
public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
// check if we have a cached version
if (this.cachedPlans != null) {
return this.cachedPlans;
}
boolean childrenSkippedDueToReplicatedInput = false;
// calculate alternative sub-plans for predecessor
final List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator);
final Set<RequestedGlobalProperties> intGlobal =
this.inConn.getInterestingProperties().getGlobalProperties();
// calculate alternative sub-plans for broadcast inputs
final List<Set<? extends NamedChannel>> broadcastPlanChannels =
new ArrayList<Set<? extends NamedChannel>>();
List<DagConnection> broadcastConnections = getBroadcastConnections();
List<String> broadcastConnectionNames = getBroadcastConnectionNames();
for (int i = 0; i < broadcastConnections.size(); i++) {
DagConnection broadcastConnection = broadcastConnections.get(i);
String broadcastConnectionName = broadcastConnectionNames.get(i);
List<PlanNode> broadcastPlanCandidates =
broadcastConnection.getSource().getAlternativePlans(estimator);
// wrap the plan candidates in named channels
HashSet<NamedChannel> broadcastChannels =
new HashSet<NamedChannel>(broadcastPlanCandidates.size());
for (PlanNode plan : broadcastPlanCandidates) {
NamedChannel c = new NamedChannel(broadcastConnectionName, plan);
DataExchangeMode exMode =
DataExchangeMode.select(
broadcastConnection.getDataExchangeMode(),
ShipStrategyType.BROADCAST,
broadcastConnection.isBreakingPipeline());
c.setShipStrategy(ShipStrategyType.BROADCAST, exMode);
broadcastChannels.add(c);
}
broadcastPlanChannels.add(broadcastChannels);
}
final RequestedGlobalProperties[] allValidGlobals;
{
Set<RequestedGlobalProperties> pairs = new HashSet<RequestedGlobalProperties>();
for (OperatorDescriptorSingle ods : getPossibleProperties()) {
pairs.addAll(ods.getPossibleGlobalProperties());
}
allValidGlobals = pairs.toArray(new RequestedGlobalProperties[pairs.size()]);
}
final ArrayList<PlanNode> outputPlans = new ArrayList<PlanNode>();
final ExecutionMode executionMode = this.inConn.getDataExchangeMode();
final int parallelism = getParallelism();
final int inParallelism = getPredecessorNode().getParallelism();
final boolean parallelismChange = inParallelism != parallelism;
final boolean breaksPipeline = this.inConn.isBreakingPipeline();
// create all candidates
for (PlanNode child : subPlans) {
if (child.getGlobalProperties().isFullyReplicated()) {
// fully replicated input is always locally forwarded if the parallelism is not changed
if (parallelismChange) {
// can not continue with this child
childrenSkippedDueToReplicatedInput = true;
continue;
} else {
this.inConn.setShipStrategy(ShipStrategyType.FORWARD);
}
}
if (this.inConn.getShipStrategy() == null) {
// pick the strategy ourselves
for (RequestedGlobalProperties igps : intGlobal) {
final Channel c = new Channel(child, this.inConn.getMaterializationMode());
igps.parameterizeChannel(c, parallelismChange, executionMode, breaksPipeline);
// if the parallelism changed, make sure that we cancel out properties, unless the
// ship strategy preserves/establishes them even under changing parallelisms
if (parallelismChange && !c.getShipStrategy().isNetworkStrategy()) {
c.getGlobalProperties().reset();
}
// check whether we meet any of the accepted properties
// we may remove this check, when we do a check to not inherit
// requested global properties that are incompatible with all possible
// requested properties
for (RequestedGlobalProperties rgps : allValidGlobals) {
if (rgps.isMetBy(c.getGlobalProperties())) {
c.setRequiredGlobalProps(rgps);
addLocalCandidates(c, broadcastPlanChannels, igps, outputPlans, estimator);
break;
}
}
}
} else {
// hint fixed the strategy
final Channel c = new Channel(child, this.inConn.getMaterializationMode());
final ShipStrategyType shipStrategy = this.inConn.getShipStrategy();
final DataExchangeMode exMode =
DataExchangeMode.select(executionMode, shipStrategy, breaksPipeline);
if (this.keys != null) {
c.setShipStrategy(shipStrategy, this.keys.toFieldList(), exMode);
} else {
c.setShipStrategy(shipStrategy, exMode);
}
if (parallelismChange) {
c.adjustGlobalPropertiesForFullParallelismChange();
}
// check whether we meet any of the accepted properties
for (RequestedGlobalProperties rgps : allValidGlobals) {
if (rgps.isMetBy(c.getGlobalProperties())) {
addLocalCandidates(c, broadcastPlanChannels, rgps, outputPlans, estimator);
break;
}
}
}
}
if (outputPlans.isEmpty()) {
if (childrenSkippedDueToReplicatedInput) {
throw new CompilerException(
"No plan meeting the requirements could be created @ "
+ this
+ ". Most likely reason: Invalid use of replicated input.");
} else {
throw new CompilerException(
"No plan meeting the requirements could be created @ "
+ this
+ ". Most likely reason: Too restrictive plan hints.");
}
}
// cost and prune the plans
for (PlanNode node : outputPlans) {
estimator.costOperator(node);
}
prunePlanAlternatives(outputPlans);
outputPlans.trimToSize();
this.cachedPlans = outputPlans;
return outputPlans;
}
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
// adjust a sort (changes grouping, so it must be for this driver to combining sort
if (in.getLocalStrategy() == LocalStrategy.SORT) {
if (!in.getLocalStrategyKeys().isValidUnorderedPrefix(this.keys)) {
throw new RuntimeException("Bug: Inconsistent sort for group strategy.");
}
in.setLocalStrategy(
LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
}
return new SingleInputPlanNode(
node,
"Reduce(" + node.getOperator().getName() + ")",
in,
DriverStrategy.SORTED_GROUP_REDUCE,
this.keyList);
} else {
// non forward case. all local properties are killed anyways, so we can safely plug in a
// combiner
Channel toCombiner = new Channel(in.getSource());
toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
// create an input node for combine with same parallelism as input node
GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
combinerNode.setParallelism(in.getSource().getParallelism());
SingleInputPlanNode combiner =
new SingleInputPlanNode(
combinerNode,
"Combine(" + node.getOperator().getName() + ")",
toCombiner,
DriverStrategy.SORTED_GROUP_COMBINE);
combiner.setCosts(new Costs(0, 0));
combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
// set sorting comparator key info
combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
// set grouping comparator key info
combiner.setDriverKeyInfo(this.keyList, 1);
Channel toReducer = new Channel(combiner);
toReducer.setShipStrategy(
in.getShipStrategy(),
in.getShipStrategyKeys(),
in.getShipStrategySortOrder(),
in.getDataExchangeMode());
if (in.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
toReducer.setDataDistribution(in.getDataDistribution());
}
toReducer.setLocalStrategy(
LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
return new SingleInputPlanNode(
node,
"Reduce (" + node.getOperator().getName() + ")",
toReducer,
DriverStrategy.SORTED_GROUP_REDUCE,
this.keyList);
}
}