public static int getFragmentCount(Fragment b) { int i = 1; for (ExchangeFragmentPair p : b) { i += getFragmentCount(p.getNode()); } return i; }
// For every fragment, create a Wrapper in PlanningSet. @VisibleForTesting public void initFragmentWrappers(Fragment rootFragment, PlanningSet planningSet) { planningSet.get(rootFragment); for (ExchangeFragmentPair fragmentPair : rootFragment) { initFragmentWrappers(fragmentPair.getNode(), planningSet); } }
private static void visit(PlanningSet planningSet, Fragment n) { Preconditions.checkNotNull(planningSet); Preconditions.checkNotNull(n); Wrapper wrapper = planningSet.get(n); n.getRoot().accept(opStatCollector, wrapper); // logger.debug("Set stats to {}", wrapper.getStats()); // receivers... for (ExchangeFragmentPair child : n) { // get the fragment node that feeds this node. Fragment childNode = child.getNode(); visit(planningSet, childNode); } }
/** * Based on the affinity of the Exchange that separates two fragments, setup fragment * dependencies. * * @param planningSet * @return Returns a list of leaf fragments in fragment dependency graph. */ private static Set<Wrapper> constructFragmentDependencyGraph(PlanningSet planningSet) { // Set up dependency of fragments based on the affinity of exchange that separates the // fragments. for (Wrapper currentFragmentWrapper : planningSet) { ExchangeFragmentPair sendingExchange = currentFragmentWrapper.getNode().getSendingExchangePair(); if (sendingExchange != null) { ParallelizationDependency dependency = sendingExchange.getExchange().getParallelizationDependency(); Wrapper receivingFragmentWrapper = planningSet.get(sendingExchange.getNode()); if (dependency == ParallelizationDependency.RECEIVER_DEPENDS_ON_SENDER) { receivingFragmentWrapper.addFragmentDependency(currentFragmentWrapper); } else if (dependency == ParallelizationDependency.SENDER_DEPENDS_ON_RECEIVER) { currentFragmentWrapper.addFragmentDependency(receivingFragmentWrapper); } } } // Identify leaf fragments. Leaf fragments are fragments that have no other fragments depending // on them for // parallelization info. First assume all fragments are leaf fragments. Go through the fragments // one by one and // remove the fragment on which the current fragment depends on. final Set<Wrapper> roots = Sets.newHashSet(); for (Wrapper w : planningSet) { roots.add(w); } for (Wrapper wrapper : planningSet) { final List<Wrapper> fragmentDependencies = wrapper.getFragmentDependencies(); if (fragmentDependencies != null && fragmentDependencies.size() > 0) { for (Wrapper dependency : fragmentDependencies) { if (roots.contains(dependency)) { roots.remove(dependency); } } } } return roots; }