예제 #1
0
파일: Main.java 프로젝트: damorim/pbicc
 // dump topological order
 static String getTopoOrder(@SuppressWarnings("rawtypes") DirectedGraph g) {
   StringBuffer sb = new StringBuffer();
   @SuppressWarnings("unchecked")
   TopologicalOrderIterator<Integer, DefaultEdge> toi =
       new TopologicalOrderIterator<Integer, DefaultEdge>(g);
   while (toi.hasNext()) {
     String canonicalName = toi.next() + ""; // Removing .java extension from topo order.
     sb.append(canonicalName.substring(0, canonicalName.length() - 5));
     sb.append("\n");
   }
   return sb.toString();
 }
예제 #2
0
  private void validateGraph(Map<String, Tap> traps) {
    verifyTrapsAreUnique(traps);

    traps = new HashMap<String, Tap>(traps); // make copy

    TopologicalOrderIterator<FlowStep, Integer> iterator = getTopologicalIterator();

    while (iterator.hasNext()) {
      FlowStep step = iterator.next();

      verifyTraps(traps, step.mapperTraps);
      verifyTraps(traps, step.reducerTraps);
    }
  }
예제 #3
0
  /**
   * Orders the provided elements based on some partial ordering constraints. Ties are broken by the
   * provided comparator (which may be arbitrary but must be consistent and specified).
   *
   * @param elements The elements to order. These elements must not be equal or they will be
   *     considered identical in the order.
   * @param constraints The constraints on the ordering.
   * @param tieBreaker The comparator used to break ties.
   * @return The provided elements in an order respecting the provided constraints.
   * @throws InconsistentPartialOrderException If the partial order is inconsistent based on the
   *     provided constraints.
   */
  public static <T> List<T> orderByConstraints(
      Collection<T> elements,
      Collection<PartialOrderConstraint<T>> constraints,
      Comparator<T> tieBreaker)
      throws InconsistentPartialOrderException {
    // Create a directed graph describing the partial ordering constraints.
    DirectedGraph<T, DefaultEdge> partialOrderGraph = new DefaultDirectedGraph<>(DefaultEdge.class);
    for (T element : elements) {
      partialOrderGraph.addVertex(element);
    }
    for (PartialOrderConstraint<T> constraint : constraints) {
      if (elements.contains(constraint.getLess()) && elements.contains(constraint.getGreater())) {
        partialOrderGraph.addEdge(constraint.getLess(), constraint.getGreater());
      }
    }

    // Determine if there are any strongly connected components. If so, pick one and scream about
    // it.
    StrongConnectivityInspector<T, DefaultEdge> strongConnectivityInspector =
        new StrongConnectivityInspector<>(partialOrderGraph);
    List<Set<T>> components = strongConnectivityInspector.stronglyConnectedSets();
    for (Set<T> stronglyConnectedComponent : components) {
      if (stronglyConnectedComponent.size() > 1) {
        throw new InconsistentPartialOrderException(components.get(0));
      }
    }

    // Otherwise, there is no cycle in the graph. That means we can just iterate a tie-breaking
    // topological sort
    // to get our ordering.
    TopologicalOrderIterator<T, DefaultEdge> iterator =
        new TopologicalOrderIterator<T, DefaultEdge>(
            partialOrderGraph, new PriorityQueue<T>(elements.size(), tieBreaker));
    List<T> result = new ArrayList<>(elements.size());
    while (iterator.hasNext()) {
      result.add(iterator.next());
    }
    return result;
  }
예제 #4
0
  /**
   * Creates the map reduce step graph.
   *
   * @param flowName
   * @param elementGraph
   * @param traps
   */
  private void makeStepGraph(String flowName, ElementGraph elementGraph, Map<String, Tap> traps) {
    SimpleDirectedGraph<Tap, Integer> tapGraph = elementGraph.makeTapGraph();

    int numJobs = countNumJobs(tapGraph);

    Map<String, FlowStep> steps = new LinkedHashMap<String, FlowStep>();
    TopologicalOrderIterator<Tap, Integer> topoIterator =
        new TopologicalOrderIterator<Tap, Integer>(tapGraph);
    int count = 0;

    while (topoIterator.hasNext()) {
      Tap source = topoIterator.next();

      if (LOG.isDebugEnabled()) LOG.debug("handling source: " + source);

      List<Tap> sinks = Graphs.successorListOf(tapGraph, source);

      for (Tap sink : sinks) {
        if (LOG.isDebugEnabled()) LOG.debug("handling path: " + source + " -> " + sink);

        FlowStep step = getCreateFlowStep(flowName, steps, sink.toString(), numJobs);

        addVertex(step);

        if (steps.containsKey(source.toString()))
          addEdge(steps.get(source.toString()), step, count++);

        // support multiple paths from source to sink
        // this allows for self joins on groups, even with different operation stacks between them
        // note we must ignore paths with intermediate taps
        List<GraphPath<FlowElement, Scope>> paths =
            elementGraph.getAllShortestPathsBetween(source, sink);

        for (GraphPath<FlowElement, Scope> path : paths) {
          if (pathContainsTap(path)) continue;

          List<Scope> scopes = path.getEdgeList();
          String sourceName = scopes.get(0).getName(); // root node of the shortest path

          step.sources.put((Tap) source, sourceName);
          step.sink = sink;

          if (step.sink.isWriteDirect())
            step.tempSink = new TempHfs(sink.getPath().toUri().getPath());

          FlowElement lhs = source;

          step.graph.addVertex(lhs);

          boolean onMapSide = true;

          for (Scope scope : scopes) {
            FlowElement rhs = elementGraph.getEdgeTarget(scope);

            step.graph.addVertex(rhs);
            step.graph.addEdge(lhs, rhs, scope);

            if (rhs instanceof Group) {
              step.group = (Group) rhs;
              onMapSide = false;
            } else if (rhs instanceof Pipe) // add relevant traps to step
            {
              String name = ((Pipe) rhs).getName();

              if (traps.containsKey(name)) {
                if (onMapSide) step.mapperTraps.put(name, traps.get(name));
                else step.reducerTraps.put(name, traps.get(name));
              }
            }

            lhs = rhs;
          }
        }
      }
    }
  }