コード例 #1
0
  /**
   * Helper method for parallelizing a given fragment. Dependent fragments are parallelized first
   * before parallelizing the given fragment.
   */
  private void parallelizeFragment(
      Wrapper fragmentWrapper,
      PlanningSet planningSet,
      Collection<DrillbitEndpoint> activeEndpoints)
      throws PhysicalOperatorSetupException {
    // If the fragment is already parallelized, return.
    if (fragmentWrapper.isEndpointsAssignmentDone()) {
      return;
    }

    // First parallelize fragments on which this fragment depends on.
    final List<Wrapper> fragmentDependencies = fragmentWrapper.getFragmentDependencies();
    if (fragmentDependencies != null && fragmentDependencies.size() > 0) {
      for (Wrapper dependency : fragmentDependencies) {
        parallelizeFragment(dependency, planningSet, activeEndpoints);
      }
    }

    // Find stats. Stats include various factors including cost of physical operators,
    // parallelizability of
    // work in physical operator and affinity of physical operator to certain nodes.
    fragmentWrapper.getNode().getRoot().accept(new StatsCollector(planningSet), fragmentWrapper);

    fragmentWrapper
        .getStats()
        .getDistributionAffinity()
        .getFragmentParallelizer()
        .parallelizeFragment(fragmentWrapper, this, activeEndpoints);
  }
コード例 #2
0
  /**
   * Based on the affinity of the Exchange that separates two fragments, setup fragment
   * dependencies.
   *
   * @param planningSet
   * @return Returns a list of leaf fragments in fragment dependency graph.
   */
  private static Set<Wrapper> constructFragmentDependencyGraph(PlanningSet planningSet) {

    // Set up dependency of fragments based on the affinity of exchange that separates the
    // fragments.
    for (Wrapper currentFragmentWrapper : planningSet) {
      ExchangeFragmentPair sendingExchange =
          currentFragmentWrapper.getNode().getSendingExchangePair();
      if (sendingExchange != null) {
        ParallelizationDependency dependency =
            sendingExchange.getExchange().getParallelizationDependency();
        Wrapper receivingFragmentWrapper = planningSet.get(sendingExchange.getNode());

        if (dependency == ParallelizationDependency.RECEIVER_DEPENDS_ON_SENDER) {
          receivingFragmentWrapper.addFragmentDependency(currentFragmentWrapper);
        } else if (dependency == ParallelizationDependency.SENDER_DEPENDS_ON_RECEIVER) {
          currentFragmentWrapper.addFragmentDependency(receivingFragmentWrapper);
        }
      }
    }

    // Identify leaf fragments. Leaf fragments are fragments that have no other fragments depending
    // on them for
    // parallelization info. First assume all fragments are leaf fragments. Go through the fragments
    // one by one and
    // remove the fragment on which the current fragment depends on.
    final Set<Wrapper> roots = Sets.newHashSet();
    for (Wrapper w : planningSet) {
      roots.add(w);
    }

    for (Wrapper wrapper : planningSet) {
      final List<Wrapper> fragmentDependencies = wrapper.getFragmentDependencies();
      if (fragmentDependencies != null && fragmentDependencies.size() > 0) {
        for (Wrapper dependency : fragmentDependencies) {
          if (roots.contains(dependency)) {
            roots.remove(dependency);
          }
        }
      }
    }

    return roots;
  }