public static int getFragmentCount(Fragment b) {
int i = 1;
for (ExchangeFragmentPair p : b) {
i += getFragmentCount(p.getNode());
}
return i;
}
// For every fragment, create a Wrapper in PlanningSet.
@VisibleForTesting
public void initFragmentWrappers(Fragment rootFragment, PlanningSet planningSet) {
planningSet.get(rootFragment);
for (ExchangeFragmentPair fragmentPair : rootFragment) {
initFragmentWrappers(fragmentPair.getNode(), planningSet);
}
}
private static void visit(PlanningSet planningSet, Fragment n) {
Preconditions.checkNotNull(planningSet);
Preconditions.checkNotNull(n);
Wrapper wrapper = planningSet.get(n);
n.getRoot().accept(opStatCollector, wrapper);
// logger.debug("Set stats to {}", wrapper.getStats());
// receivers...
for (ExchangeFragmentPair child : n) {
// get the fragment node that feeds this node.
Fragment childNode = child.getNode();
visit(planningSet, childNode);
}
}
/**
* Based on the affinity of the Exchange that separates two fragments, setup fragment
* dependencies.
*
* @param planningSet
* @return Returns a list of leaf fragments in fragment dependency graph.
*/
private static Set<Wrapper> constructFragmentDependencyGraph(PlanningSet planningSet) {
// Set up dependency of fragments based on the affinity of exchange that separates the
// fragments.
for (Wrapper currentFragmentWrapper : planningSet) {
ExchangeFragmentPair sendingExchange =
currentFragmentWrapper.getNode().getSendingExchangePair();
if (sendingExchange != null) {
ParallelizationDependency dependency =
sendingExchange.getExchange().getParallelizationDependency();
Wrapper receivingFragmentWrapper = planningSet.get(sendingExchange.getNode());
if (dependency == ParallelizationDependency.RECEIVER_DEPENDS_ON_SENDER) {
receivingFragmentWrapper.addFragmentDependency(currentFragmentWrapper);
} else if (dependency == ParallelizationDependency.SENDER_DEPENDS_ON_RECEIVER) {
currentFragmentWrapper.addFragmentDependency(receivingFragmentWrapper);
}
}
}
// Identify leaf fragments. Leaf fragments are fragments that have no other fragments depending
// on them for
// parallelization info. First assume all fragments are leaf fragments. Go through the fragments
// one by one and
// remove the fragment on which the current fragment depends on.
final Set<Wrapper> roots = Sets.newHashSet();
for (Wrapper w : planningSet) {
roots.add(w);
}
for (Wrapper wrapper : planningSet) {
final List<Wrapper> fragmentDependencies = wrapper.getFragmentDependencies();
if (fragmentDependencies != null && fragmentDependencies.size() > 0) {
for (Wrapper dependency : fragmentDependencies) {
if (roots.contains(dependency)) {
roots.remove(dependency);
}
}
}
}
return roots;
}