Example #1
0
 // dump topological order
 static String getTopoOrder(@SuppressWarnings("rawtypes") DirectedGraph g) {
   StringBuffer sb = new StringBuffer();
   @SuppressWarnings("unchecked")
   TopologicalOrderIterator<Integer, DefaultEdge> toi =
       new TopologicalOrderIterator<Integer, DefaultEdge>(g);
   while (toi.hasNext()) {
     String canonicalName = toi.next() + ""; // Removing .java extension from topo order.
     sb.append(canonicalName.substring(0, canonicalName.length() - 5));
     sb.append("\n");
   }
   return sb.toString();
 }
Example #2
0
  private void validateGraph(Map<String, Tap> traps) {
    verifyTrapsAreUnique(traps);

    traps = new HashMap<String, Tap>(traps); // make copy

    TopologicalOrderIterator<FlowStep, Integer> iterator = getTopologicalIterator();

    while (iterator.hasNext()) {
      FlowStep step = iterator.next();

      verifyTraps(traps, step.mapperTraps);
      verifyTraps(traps, step.reducerTraps);
    }
  }
Example #3
0
  /**
   * Orders the provided elements based on some partial ordering constraints. Ties are broken by the
   * provided comparator (which may be arbitrary but must be consistent and specified).
   *
   * @param elements The elements to order. These elements must not be equal or they will be
   *     considered identical in the order.
   * @param constraints The constraints on the ordering.
   * @param tieBreaker The comparator used to break ties.
   * @return The provided elements in an order respecting the provided constraints.
   * @throws InconsistentPartialOrderException If the partial order is inconsistent based on the
   *     provided constraints.
   */
  public static <T> List<T> orderByConstraints(
      Collection<T> elements,
      Collection<PartialOrderConstraint<T>> constraints,
      Comparator<T> tieBreaker)
      throws InconsistentPartialOrderException {
    // Create a directed graph describing the partial ordering constraints.
    DirectedGraph<T, DefaultEdge> partialOrderGraph = new DefaultDirectedGraph<>(DefaultEdge.class);
    for (T element : elements) {
      partialOrderGraph.addVertex(element);
    }
    for (PartialOrderConstraint<T> constraint : constraints) {
      if (elements.contains(constraint.getLess()) && elements.contains(constraint.getGreater())) {
        partialOrderGraph.addEdge(constraint.getLess(), constraint.getGreater());
      }
    }

    // Determine if there are any strongly connected components. If so, pick one and scream about
    // it.
    StrongConnectivityInspector<T, DefaultEdge> strongConnectivityInspector =
        new StrongConnectivityInspector<>(partialOrderGraph);
    List<Set<T>> components = strongConnectivityInspector.stronglyConnectedSets();
    for (Set<T> stronglyConnectedComponent : components) {
      if (stronglyConnectedComponent.size() > 1) {
        throw new InconsistentPartialOrderException(components.get(0));
      }
    }

    // Otherwise, there is no cycle in the graph. That means we can just iterate a tie-breaking
    // topological sort
    // to get our ordering.
    TopologicalOrderIterator<T, DefaultEdge> iterator =
        new TopologicalOrderIterator<T, DefaultEdge>(
            partialOrderGraph, new PriorityQueue<T>(elements.size(), tieBreaker));
    List<T> result = new ArrayList<>(elements.size());
    while (iterator.hasNext()) {
      result.add(iterator.next());
    }
    return result;
  }
Example #4
0
  /**
   * Creates the map reduce step graph.
   *
   * @param flowName
   * @param elementGraph
   * @param traps
   */
  private void makeStepGraph(String flowName, ElementGraph elementGraph, Map<String, Tap> traps) {
    SimpleDirectedGraph<Tap, Integer> tapGraph = elementGraph.makeTapGraph();

    int numJobs = countNumJobs(tapGraph);

    Map<String, FlowStep> steps = new LinkedHashMap<String, FlowStep>();
    TopologicalOrderIterator<Tap, Integer> topoIterator =
        new TopologicalOrderIterator<Tap, Integer>(tapGraph);
    int count = 0;

    while (topoIterator.hasNext()) {
      Tap source = topoIterator.next();

      if (LOG.isDebugEnabled()) LOG.debug("handling source: " + source);

      List<Tap> sinks = Graphs.successorListOf(tapGraph, source);

      for (Tap sink : sinks) {
        if (LOG.isDebugEnabled()) LOG.debug("handling path: " + source + " -> " + sink);

        FlowStep step = getCreateFlowStep(flowName, steps, sink.toString(), numJobs);

        addVertex(step);

        if (steps.containsKey(source.toString()))
          addEdge(steps.get(source.toString()), step, count++);

        // support multiple paths from source to sink
        // this allows for self joins on groups, even with different operation stacks between them
        // note we must ignore paths with intermediate taps
        List<GraphPath<FlowElement, Scope>> paths =
            elementGraph.getAllShortestPathsBetween(source, sink);

        for (GraphPath<FlowElement, Scope> path : paths) {
          if (pathContainsTap(path)) continue;

          List<Scope> scopes = path.getEdgeList();
          String sourceName = scopes.get(0).getName(); // root node of the shortest path

          step.sources.put((Tap) source, sourceName);
          step.sink = sink;

          if (step.sink.isWriteDirect())
            step.tempSink = new TempHfs(sink.getPath().toUri().getPath());

          FlowElement lhs = source;

          step.graph.addVertex(lhs);

          boolean onMapSide = true;

          for (Scope scope : scopes) {
            FlowElement rhs = elementGraph.getEdgeTarget(scope);

            step.graph.addVertex(rhs);
            step.graph.addEdge(lhs, rhs, scope);

            if (rhs instanceof Group) {
              step.group = (Group) rhs;
              onMapSide = false;
            } else if (rhs instanceof Pipe) // add relevant traps to step
            {
              String name = ((Pipe) rhs).getName();

              if (traps.containsKey(name)) {
                if (onMapSide) step.mapperTraps.put(name, traps.get(name));
                else step.reducerTraps.put(name, traps.get(name));
              }
            }

            lhs = rhs;
          }
        }
      }
    }
  }
Example #5
0
  protected static Element findRootLinkOfJointTree(
      ArrayList<Element> jointList, ArrayList<Element> linkList) {

    SimpleDirectedGraph<String, DefaultEdge> graph =
        new SimpleDirectedGraph<String, DefaultEdge>(DefaultEdge.class);

    for (Element joint : jointList) {

      Element parent = (Element) joint.getElementsByTagName("parent").item(0);
      Element child = (Element) joint.getElementsByTagName("child").item(0);

      String parentLinkName = parent.getAttribute("link");
      String childLinkName = child.getAttribute("link");

      if (!graph.containsVertex(parentLinkName)) {
        graph.addVertex(parentLinkName);
      }
      if (!graph.containsVertex(childLinkName)) {
        graph.addVertex(childLinkName);
      }

      if (graph.containsEdge(parentLinkName, childLinkName)) {
        throw new RuntimeException(
            "Multiple joints between parent link '"
                + parentLinkName
                + "' and child link '"
                + childLinkName
                + "'");
      } else if (graph.containsEdge(childLinkName, parentLinkName)) {
        throw new RuntimeException(
            "Inverse joint between parent link '"
                + parentLinkName
                + "' and child link '"
                + childLinkName
                + "' already exists");
      } else {
        graph.addEdge(parentLinkName, childLinkName);
      }
    }

    CycleDetector<String, DefaultEdge> cycDet;
    cycDet = new CycleDetector<String, DefaultEdge>(graph);
    Set<String> cycSet = cycDet.findCycles();
    if (cycSet.size() > 0) {
      Iterator<String> it = cycSet.iterator();
      StringBuffer sb = new StringBuffer(it.next());
      while (it.hasNext()) {
        sb.append(", " + it.next());
      }
      throw new RuntimeException("Cycle in joint graph, involving elements '" + sb + "'");
    }

    TopologicalOrderIterator<String, DefaultEdge> toit;
    toit = new TopologicalOrderIterator<String, DefaultEdge>(graph);
    String rootLinkName = toit.next();

    Element rootLink = null;
    for (Element link : linkList) {
      if (link.getAttribute("name").equals(rootLinkName)) {
        rootLink = link;
        break;
      }
    }

    if (rootLink == null) {
      throw new RuntimeException("Root joint's parent '" + rootLinkName + "' doesn't exist");
    }

    return rootLink;
  }