Ejemplo n.º 1
0
  protected void addLocalCandidates(
      Channel template,
      List<Set<? extends NamedChannel>> broadcastPlanChannels,
      RequestedGlobalProperties rgps,
      List<PlanNode> target,
      CostEstimator estimator) {
    for (RequestedLocalProperties ilp :
        this.inConn.getInterestingProperties().getLocalProperties()) {
      final Channel in = template.clone();
      ilp.parameterizeChannel(in);

      // instantiate a candidate, if the instantiated local properties meet one possible local
      // property set
      outer:
      for (OperatorDescriptorSingle dps : getPossibleProperties()) {
        for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
          if (ilps.isMetBy(in.getLocalProperties())) {
            in.setRequiredLocalProps(ilps);
            instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
            break outer;
          }
        }
      }
    }
  }
  @Override
  public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
    if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
      // adjust a sort (changes grouping, so it must be for this driver to combining sort
      if (in.getLocalStrategy() == LocalStrategy.SORT) {
        if (!in.getLocalStrategyKeys().isValidUnorderedPrefix(this.keys)) {
          throw new RuntimeException("Bug: Inconsistent sort for group strategy.");
        }
        in.setLocalStrategy(
            LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
      }
      return new SingleInputPlanNode(
          node,
          "Reduce(" + node.getOperator().getName() + ")",
          in,
          DriverStrategy.SORTED_GROUP_REDUCE,
          this.keyList);
    } else {
      // non forward case. all local properties are killed anyways, so we can safely plug in a
      // combiner
      Channel toCombiner = new Channel(in.getSource());
      toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);

      // create an input node for combine with same parallelism as input node
      GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
      combinerNode.setParallelism(in.getSource().getParallelism());

      SingleInputPlanNode combiner =
          new SingleInputPlanNode(
              combinerNode,
              "Combine(" + node.getOperator().getName() + ")",
              toCombiner,
              DriverStrategy.SORTED_GROUP_COMBINE);
      combiner.setCosts(new Costs(0, 0));
      combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
      // set sorting comparator key info
      combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
      // set grouping comparator key info
      combiner.setDriverKeyInfo(this.keyList, 1);

      Channel toReducer = new Channel(combiner);
      toReducer.setShipStrategy(
          in.getShipStrategy(),
          in.getShipStrategyKeys(),
          in.getShipStrategySortOrder(),
          in.getDataExchangeMode());
      if (in.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
        toReducer.setDataDistribution(in.getDataDistribution());
      }
      toReducer.setLocalStrategy(
          LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());

      return new SingleInputPlanNode(
          node,
          "Reduce (" + node.getOperator().getName() + ")",
          toReducer,
          DriverStrategy.SORTED_GROUP_REDUCE,
          this.keyList);
    }
  }
Ejemplo n.º 3
0
  protected void instantiateCandidate(
      OperatorDescriptorSingle dps,
      Channel in,
      List<Set<? extends NamedChannel>> broadcastPlanChannels,
      List<PlanNode> target,
      CostEstimator estimator,
      RequestedGlobalProperties globPropsReq,
      RequestedLocalProperties locPropsReq) {
    final PlanNode inputSource = in.getSource();

    for (List<NamedChannel> broadcastChannelsCombination :
        Sets.cartesianProduct(broadcastPlanChannels)) {

      boolean validCombination = true;
      boolean requiresPipelinebreaker = false;

      // check whether the broadcast inputs use the same plan candidate at the branching point
      for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
        NamedChannel nc = broadcastChannelsCombination.get(i);
        PlanNode bcSource = nc.getSource();

        // check branch compatibility against input
        if (!areBranchCompatible(bcSource, inputSource)) {
          validCombination = false;
          break;
        }

        // check branch compatibility against all other broadcast variables
        for (int k = 0; k < i; k++) {
          PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();

          if (!areBranchCompatible(bcSource, otherBcSource)) {
            validCombination = false;
            break;
          }
        }

        // check if there is a common predecessor and whether there is a dam on the way to all
        // common predecessors
        if (in.isOnDynamicPath() && this.hereJoinedBranches != null) {
          for (OptimizerNode brancher : this.hereJoinedBranches) {
            PlanNode candAtBrancher = in.getSource().getCandidateAtBranchPoint(brancher);

            if (candAtBrancher == null) {
              // closed branch between two broadcast variables
              continue;
            }

            SourceAndDamReport res = in.getSource().hasDamOnPathDownTo(candAtBrancher);
            if (res == NOT_FOUND) {
              throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
            } else if (res == FOUND_SOURCE) {
              requiresPipelinebreaker = true;
              break;
            } else if (res == FOUND_SOURCE_AND_DAM) {
              // good
            } else {
              throw new CompilerException();
            }
          }
        }
      }

      if (!validCombination) {
        continue;
      }

      if (requiresPipelinebreaker) {
        in.setTempMode(in.getTempMode().makePipelineBreaker());
      }

      final SingleInputPlanNode node = dps.instantiate(in, this);
      node.setBroadcastInputs(broadcastChannelsCombination);

      // compute how the strategy affects the properties
      GlobalProperties gProps = in.getGlobalProperties().clone();
      LocalProperties lProps = in.getLocalProperties().clone();
      gProps = dps.computeGlobalProperties(gProps);
      lProps = dps.computeLocalProperties(lProps);

      // filter by the user code field copies
      gProps =
          gProps.filterBySemanticProperties(getSemanticPropertiesForGlobalPropertyFiltering(), 0);
      lProps =
          lProps.filterBySemanticProperties(getSemanticPropertiesForLocalPropertyFiltering(), 0);

      // apply
      node.initProperties(gProps, lProps);
      node.updatePropertiesWithUniqueSets(getUniqueFields());
      target.add(node);
    }
  }