public static Map<Symbol, Symbol> exchangeInputToOutput(ExchangeNode node, int sourceIndex) {
   List<Symbol> inputSymbols = node.getInputs().get(sourceIndex);
   Map<Symbol, Symbol> inputToOutput = new HashMap<>();
   for (int i = 0; i < node.getOutputSymbols().size(); i++) {
     inputToOutput.put(inputSymbols.get(i), node.getOutputSymbols().get(i));
   }
   return inputToOutput;
 }
示例#2
0
    @Override
    public Void visitExchange(ExchangeNode node, Void context) {
      for (PlanFragmentId planFragmentId : node.getSourceFragmentIds()) {
        PlanFragment target = fragmentsById.get(planFragmentId);
        printEdge(node, target.getRoot());
      }

      return null;
    }
示例#3
0
 @Override
 public Void visitExchange(ExchangeNode node, Void context) {
   List<Symbol> symbols = node.getOutputSymbols();
   if (node.getType() == REPARTITION || node.getType() == REPARTITION_WITH_NULL_REPLICATION) {
     symbols = node.getPartitionKeys().orElseGet(() -> ImmutableList.of(new Symbol("(absent)")));
   }
   String columns = Joiner.on(", ").join(symbols);
   printNode(
       node,
       format("ExchangeNode[%s]", node.getType()),
       columns,
       NODE_COLORS.get(NodeType.EXCHANGE));
   for (PlanNode planNode : node.getSources()) {
     planNode.accept(this, context);
   }
   return null;
 }
    @Override
    public ActualProperties visitExchange(
        ExchangeNode node, List<ActualProperties> inputProperties) {
      checkArgument(
          node.getScope() != REMOTE
              || inputProperties.stream().noneMatch(ActualProperties::isNullsReplicated),
          "Null replicated inputs should not be remotely exchanged");

      Set<Map.Entry<Symbol, NullableValue>> entries = null;
      for (int sourceIndex = 0; sourceIndex < node.getSources().size(); sourceIndex++) {
        Map<Symbol, Symbol> inputToOutput = exchangeInputToOutput(node, sourceIndex);
        ActualProperties translated =
            inputProperties
                .get(sourceIndex)
                .translate(symbol -> Optional.ofNullable(inputToOutput.get(symbol)));

        entries =
            (entries == null)
                ? translated.getConstants().entrySet()
                : Sets.intersection(entries, translated.getConstants().entrySet());
      }
      checkState(entries != null);

      Map<Symbol, NullableValue> constants =
          entries.stream().collect(toMap(Map.Entry::getKey, Map.Entry::getValue));

      // Local exchanges are only created in AddLocalExchanges, at the end of optimization, and
      // local exchanges do not produce global properties as represented by ActualProperties.
      // This is acceptable because AddLocalExchanges does not use global properties and is only
      // interested in the local properties.
      // TODO: implement full properties for local exchanges
      if (node.getScope() == LOCAL) {
        return ActualProperties.builder().constants(constants).build();
      }

      switch (node.getType()) {
        case GATHER:
          boolean coordinatorOnly =
              node.getPartitioningScheme().getPartitioning().getHandle().isCoordinatorOnly();
          return ActualProperties.builder()
              .global(
                  coordinatorOnly ? coordinatorSingleStreamPartition() : singleStreamPartition())
              .constants(constants)
              .build();
        case REPARTITION:
          return ActualProperties.builder()
              .global(
                  partitionedOn(
                          node.getPartitioningScheme().getPartitioning(),
                          Optional.of(node.getPartitioningScheme().getPartitioning()))
                      .withReplicatedNulls(node.getPartitioningScheme().isReplicateNulls()))
              .constants(constants)
              .build();
        case REPLICATE:
          // TODO: this should have the same global properties as the stream taking the replicated
          // data
          return ActualProperties.builder()
              .global(arbitraryPartition())
              .constants(constants)
              .build();
      }

      throw new UnsupportedOperationException("not yet implemented");
    }
示例#5
0
    @Override
    public PlanNode visitExchange(
        ExchangeNode exchange, RewriteContext<FragmentProperties> context) {
      ImmutableList.Builder<SubPlan> builder = ImmutableList.builder();
      if (exchange.getType() == ExchangeNode.Type.GATHER) {
        context.get().setSingleNodeDistribution();

        for (int i = 0; i < exchange.getSources().size(); i++) {
          FragmentProperties childProperties = new FragmentProperties();
          childProperties.setUnpartitionedOutput();
          childProperties.setOutputLayout(exchange.getInputs().get(i));

          builder.add(buildSubPlan(exchange.getSources().get(i), childProperties, context));
        }
      } else if (exchange.getType() == ExchangeNode.Type.REPARTITION) {
        context.get().setFixedDistribution();

        FragmentProperties childProperties =
            new FragmentProperties()
                .setPartitionedOutput(exchange.getPartitionKeys(), exchange.getHashSymbol())
                .setOutputLayout(Iterables.getOnlyElement(exchange.getInputs()));

        builder.add(
            buildSubPlan(
                Iterables.getOnlyElement(exchange.getSources()), childProperties, context));
      } else if (exchange.getType() == ExchangeNode.Type.REPARTITION_WITH_NULL_REPLICATION) {
        context.get().setFixedDistribution();

        FragmentProperties childProperties =
            new FragmentProperties()
                .setPartitionedOutput(exchange.getPartitionKeys(), exchange.getHashSymbol())
                .replicateNulls()
                .setOutputLayout(Iterables.getOnlyElement(exchange.getInputs()));

        builder.add(
            buildSubPlan(
                Iterables.getOnlyElement(exchange.getSources()), childProperties, context));
      } else if (exchange.getType() == ExchangeNode.Type.REPLICATE) {
        FragmentProperties childProperties = new FragmentProperties();
        childProperties.setUnpartitionedOutput();
        childProperties.setOutputLayout(Iterables.getOnlyElement(exchange.getInputs()));

        builder.add(
            buildSubPlan(
                Iterables.getOnlyElement(exchange.getSources()), childProperties, context));
      }

      List<SubPlan> children = builder.build();
      context.get().addChildren(children);

      List<PlanFragmentId> childrenIds =
          children
              .stream()
              .map(SubPlan::getFragment)
              .map(PlanFragment::getId)
              .collect(toImmutableList());

      return new RemoteSourceNode(exchange.getId(), childrenIds, exchange.getOutputSymbols());
    }
    @Override
    public PlanNode visitExchange(ExchangeNode node, RewriteContext<Set<Symbol>> context) {
      Set<Symbol> expectedOutputSymbols = Sets.newHashSet(context.get());
      node.getPartitionFunction().getHashColumn().ifPresent(expectedOutputSymbols::add);
      node.getPartitionFunction()
          .getPartitionFunctionArguments()
          .stream()
          .filter(PartitionFunctionArgumentBinding::isVariable)
          .map(PartitionFunctionArgumentBinding::getColumn)
          .forEach(expectedOutputSymbols::add);

      List<List<Symbol>> inputsBySource = new ArrayList<>(node.getInputs().size());
      for (int i = 0; i < node.getInputs().size(); i++) {
        inputsBySource.add(new ArrayList<>());
      }

      List<Symbol> newOutputSymbols = new ArrayList<>(node.getOutputSymbols().size());
      for (int i = 0; i < node.getOutputSymbols().size(); i++) {
        Symbol outputSymbol = node.getOutputSymbols().get(i);
        if (expectedOutputSymbols.contains(outputSymbol)) {
          newOutputSymbols.add(outputSymbol);
          for (int source = 0; source < node.getInputs().size(); source++) {
            inputsBySource.get(source).add(node.getInputs().get(source).get(i));
          }
        }
      }

      // newOutputSymbols contains all partition and hash symbols so simply swap the output layout
      PartitionFunctionBinding partitionFunctionBinding =
          new PartitionFunctionBinding(
              node.getPartitionFunction().getPartitioningHandle(),
              newOutputSymbols,
              node.getPartitionFunction().getPartitionFunctionArguments(),
              node.getPartitionFunction().getHashColumn(),
              node.getPartitionFunction().isReplicateNulls(),
              node.getPartitionFunction().getBucketToPartition());

      ImmutableList.Builder<PlanNode> rewrittenSources = ImmutableList.<PlanNode>builder();
      for (int i = 0; i < node.getSources().size(); i++) {
        ImmutableSet.Builder<Symbol> expectedInputs =
            ImmutableSet.<Symbol>builder().addAll(inputsBySource.get(i));

        rewrittenSources.add(context.rewrite(node.getSources().get(i), expectedInputs.build()));
      }

      return new ExchangeNode(
          node.getId(),
          node.getType(),
          partitionFunctionBinding,
          rewrittenSources.build(),
          inputsBySource);
    }