@Override
  public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
    if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
      // adjust a sort (changes grouping, so it must be for this driver to combining sort
      if (in.getLocalStrategy() == LocalStrategy.SORT) {
        if (!in.getLocalStrategyKeys().isValidUnorderedPrefix(this.keys)) {
          throw new RuntimeException("Bug: Inconsistent sort for group strategy.");
        }
        in.setLocalStrategy(
            LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
      }
      return new SingleInputPlanNode(
          node,
          "Reduce(" + node.getOperator().getName() + ")",
          in,
          DriverStrategy.SORTED_GROUP_REDUCE,
          this.keyList);
    } else {
      // non forward case. all local properties are killed anyways, so we can safely plug in a
      // combiner
      Channel toCombiner = new Channel(in.getSource());
      toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);

      // create an input node for combine with same parallelism as input node
      GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
      combinerNode.setParallelism(in.getSource().getParallelism());

      SingleInputPlanNode combiner =
          new SingleInputPlanNode(
              combinerNode,
              "Combine(" + node.getOperator().getName() + ")",
              toCombiner,
              DriverStrategy.SORTED_GROUP_COMBINE);
      combiner.setCosts(new Costs(0, 0));
      combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
      // set sorting comparator key info
      combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
      // set grouping comparator key info
      combiner.setDriverKeyInfo(this.keyList, 1);

      Channel toReducer = new Channel(combiner);
      toReducer.setShipStrategy(
          in.getShipStrategy(),
          in.getShipStrategyKeys(),
          in.getShipStrategySortOrder(),
          in.getDataExchangeMode());
      if (in.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
        toReducer.setDataDistribution(in.getDataDistribution());
      }
      toReducer.setLocalStrategy(
          LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());

      return new SingleInputPlanNode(
          node,
          "Reduce (" + node.getOperator().getName() + ")",
          toReducer,
          DriverStrategy.SORTED_GROUP_REDUCE,
          this.keyList);
    }
  }
Пример #2
0
  @Override
  public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
    // check if we have a cached version
    if (this.cachedPlans != null) {
      return this.cachedPlans;
    }

    boolean childrenSkippedDueToReplicatedInput = false;

    // calculate alternative sub-plans for predecessor
    final List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator);
    final Set<RequestedGlobalProperties> intGlobal =
        this.inConn.getInterestingProperties().getGlobalProperties();

    // calculate alternative sub-plans for broadcast inputs
    final List<Set<? extends NamedChannel>> broadcastPlanChannels =
        new ArrayList<Set<? extends NamedChannel>>();
    List<DagConnection> broadcastConnections = getBroadcastConnections();
    List<String> broadcastConnectionNames = getBroadcastConnectionNames();

    for (int i = 0; i < broadcastConnections.size(); i++) {
      DagConnection broadcastConnection = broadcastConnections.get(i);
      String broadcastConnectionName = broadcastConnectionNames.get(i);
      List<PlanNode> broadcastPlanCandidates =
          broadcastConnection.getSource().getAlternativePlans(estimator);

      // wrap the plan candidates in named channels
      HashSet<NamedChannel> broadcastChannels =
          new HashSet<NamedChannel>(broadcastPlanCandidates.size());
      for (PlanNode plan : broadcastPlanCandidates) {
        NamedChannel c = new NamedChannel(broadcastConnectionName, plan);
        DataExchangeMode exMode =
            DataExchangeMode.select(
                broadcastConnection.getDataExchangeMode(),
                ShipStrategyType.BROADCAST,
                broadcastConnection.isBreakingPipeline());
        c.setShipStrategy(ShipStrategyType.BROADCAST, exMode);
        broadcastChannels.add(c);
      }
      broadcastPlanChannels.add(broadcastChannels);
    }

    final RequestedGlobalProperties[] allValidGlobals;
    {
      Set<RequestedGlobalProperties> pairs = new HashSet<RequestedGlobalProperties>();
      for (OperatorDescriptorSingle ods : getPossibleProperties()) {
        pairs.addAll(ods.getPossibleGlobalProperties());
      }
      allValidGlobals = pairs.toArray(new RequestedGlobalProperties[pairs.size()]);
    }
    final ArrayList<PlanNode> outputPlans = new ArrayList<PlanNode>();

    final ExecutionMode executionMode = this.inConn.getDataExchangeMode();

    final int parallelism = getParallelism();
    final int inParallelism = getPredecessorNode().getParallelism();

    final boolean parallelismChange = inParallelism != parallelism;

    final boolean breaksPipeline = this.inConn.isBreakingPipeline();

    // create all candidates
    for (PlanNode child : subPlans) {

      if (child.getGlobalProperties().isFullyReplicated()) {
        // fully replicated input is always locally forwarded if the parallelism is not changed
        if (parallelismChange) {
          // can not continue with this child
          childrenSkippedDueToReplicatedInput = true;
          continue;
        } else {
          this.inConn.setShipStrategy(ShipStrategyType.FORWARD);
        }
      }

      if (this.inConn.getShipStrategy() == null) {
        // pick the strategy ourselves
        for (RequestedGlobalProperties igps : intGlobal) {
          final Channel c = new Channel(child, this.inConn.getMaterializationMode());
          igps.parameterizeChannel(c, parallelismChange, executionMode, breaksPipeline);

          // if the parallelism changed, make sure that we cancel out properties, unless the
          // ship strategy preserves/establishes them even under changing parallelisms
          if (parallelismChange && !c.getShipStrategy().isNetworkStrategy()) {
            c.getGlobalProperties().reset();
          }

          // check whether we meet any of the accepted properties
          // we may remove this check, when we do a check to not inherit
          // requested global properties that are incompatible with all possible
          // requested properties
          for (RequestedGlobalProperties rgps : allValidGlobals) {
            if (rgps.isMetBy(c.getGlobalProperties())) {
              c.setRequiredGlobalProps(rgps);
              addLocalCandidates(c, broadcastPlanChannels, igps, outputPlans, estimator);
              break;
            }
          }
        }
      } else {
        // hint fixed the strategy
        final Channel c = new Channel(child, this.inConn.getMaterializationMode());
        final ShipStrategyType shipStrategy = this.inConn.getShipStrategy();
        final DataExchangeMode exMode =
            DataExchangeMode.select(executionMode, shipStrategy, breaksPipeline);

        if (this.keys != null) {
          c.setShipStrategy(shipStrategy, this.keys.toFieldList(), exMode);
        } else {
          c.setShipStrategy(shipStrategy, exMode);
        }

        if (parallelismChange) {
          c.adjustGlobalPropertiesForFullParallelismChange();
        }

        // check whether we meet any of the accepted properties
        for (RequestedGlobalProperties rgps : allValidGlobals) {
          if (rgps.isMetBy(c.getGlobalProperties())) {
            addLocalCandidates(c, broadcastPlanChannels, rgps, outputPlans, estimator);
            break;
          }
        }
      }
    }

    if (outputPlans.isEmpty()) {
      if (childrenSkippedDueToReplicatedInput) {
        throw new CompilerException(
            "No plan meeting the requirements could be created @ "
                + this
                + ". Most likely reason: Invalid use of replicated input.");
      } else {
        throw new CompilerException(
            "No plan meeting the requirements could be created @ "
                + this
                + ". Most likely reason: Too restrictive plan hints.");
      }
    }

    // cost and prune the plans
    for (PlanNode node : outputPlans) {
      estimator.costOperator(node);
    }
    prunePlanAlternatives(outputPlans);
    outputPlans.trimToSize();

    this.cachedPlans = outputPlans;
    return outputPlans;
  }