// dump topological order
static String getTopoOrder(@SuppressWarnings("rawtypes") DirectedGraph g) {
StringBuffer sb = new StringBuffer();
@SuppressWarnings("unchecked")
TopologicalOrderIterator<Integer, DefaultEdge> toi =
new TopologicalOrderIterator<Integer, DefaultEdge>(g);
while (toi.hasNext()) {
String canonicalName = toi.next() + ""; // Removing .java extension from topo order.
sb.append(canonicalName.substring(0, canonicalName.length() - 5));
sb.append("\n");
}
return sb.toString();
}
private void validateGraph(Map<String, Tap> traps) {
verifyTrapsAreUnique(traps);
traps = new HashMap<String, Tap>(traps); // make copy
TopologicalOrderIterator<FlowStep, Integer> iterator = getTopologicalIterator();
while (iterator.hasNext()) {
FlowStep step = iterator.next();
verifyTraps(traps, step.mapperTraps);
verifyTraps(traps, step.reducerTraps);
}
}
/**
* Orders the provided elements based on some partial ordering constraints. Ties are broken by the
* provided comparator (which may be arbitrary but must be consistent and specified).
*
* @param elements The elements to order. These elements must not be equal or they will be
* considered identical in the order.
* @param constraints The constraints on the ordering.
* @param tieBreaker The comparator used to break ties.
* @return The provided elements in an order respecting the provided constraints.
* @throws InconsistentPartialOrderException If the partial order is inconsistent based on the
* provided constraints.
*/
public static <T> List<T> orderByConstraints(
Collection<T> elements,
Collection<PartialOrderConstraint<T>> constraints,
Comparator<T> tieBreaker)
throws InconsistentPartialOrderException {
// Create a directed graph describing the partial ordering constraints.
DirectedGraph<T, DefaultEdge> partialOrderGraph = new DefaultDirectedGraph<>(DefaultEdge.class);
for (T element : elements) {
partialOrderGraph.addVertex(element);
}
for (PartialOrderConstraint<T> constraint : constraints) {
if (elements.contains(constraint.getLess()) && elements.contains(constraint.getGreater())) {
partialOrderGraph.addEdge(constraint.getLess(), constraint.getGreater());
}
}
// Determine if there are any strongly connected components. If so, pick one and scream about
// it.
StrongConnectivityInspector<T, DefaultEdge> strongConnectivityInspector =
new StrongConnectivityInspector<>(partialOrderGraph);
List<Set<T>> components = strongConnectivityInspector.stronglyConnectedSets();
for (Set<T> stronglyConnectedComponent : components) {
if (stronglyConnectedComponent.size() > 1) {
throw new InconsistentPartialOrderException(components.get(0));
}
}
// Otherwise, there is no cycle in the graph. That means we can just iterate a tie-breaking
// topological sort
// to get our ordering.
TopologicalOrderIterator<T, DefaultEdge> iterator =
new TopologicalOrderIterator<T, DefaultEdge>(
partialOrderGraph, new PriorityQueue<T>(elements.size(), tieBreaker));
List<T> result = new ArrayList<>(elements.size());
while (iterator.hasNext()) {
result.add(iterator.next());
}
return result;
}
/**
* Creates the map reduce step graph.
*
* @param flowName
* @param elementGraph
* @param traps
*/
private void makeStepGraph(String flowName, ElementGraph elementGraph, Map<String, Tap> traps) {
SimpleDirectedGraph<Tap, Integer> tapGraph = elementGraph.makeTapGraph();
int numJobs = countNumJobs(tapGraph);
Map<String, FlowStep> steps = new LinkedHashMap<String, FlowStep>();
TopologicalOrderIterator<Tap, Integer> topoIterator =
new TopologicalOrderIterator<Tap, Integer>(tapGraph);
int count = 0;
while (topoIterator.hasNext()) {
Tap source = topoIterator.next();
if (LOG.isDebugEnabled()) LOG.debug("handling source: " + source);
List<Tap> sinks = Graphs.successorListOf(tapGraph, source);
for (Tap sink : sinks) {
if (LOG.isDebugEnabled()) LOG.debug("handling path: " + source + " -> " + sink);
FlowStep step = getCreateFlowStep(flowName, steps, sink.toString(), numJobs);
addVertex(step);
if (steps.containsKey(source.toString()))
addEdge(steps.get(source.toString()), step, count++);
// support multiple paths from source to sink
// this allows for self joins on groups, even with different operation stacks between them
// note we must ignore paths with intermediate taps
List<GraphPath<FlowElement, Scope>> paths =
elementGraph.getAllShortestPathsBetween(source, sink);
for (GraphPath<FlowElement, Scope> path : paths) {
if (pathContainsTap(path)) continue;
List<Scope> scopes = path.getEdgeList();
String sourceName = scopes.get(0).getName(); // root node of the shortest path
step.sources.put((Tap) source, sourceName);
step.sink = sink;
if (step.sink.isWriteDirect())
step.tempSink = new TempHfs(sink.getPath().toUri().getPath());
FlowElement lhs = source;
step.graph.addVertex(lhs);
boolean onMapSide = true;
for (Scope scope : scopes) {
FlowElement rhs = elementGraph.getEdgeTarget(scope);
step.graph.addVertex(rhs);
step.graph.addEdge(lhs, rhs, scope);
if (rhs instanceof Group) {
step.group = (Group) rhs;
onMapSide = false;
} else if (rhs instanceof Pipe) // add relevant traps to step
{
String name = ((Pipe) rhs).getName();
if (traps.containsKey(name)) {
if (onMapSide) step.mapperTraps.put(name, traps.get(name));
else step.reducerTraps.put(name, traps.get(name));
}
}
lhs = rhs;
}
}
}
}
}