private void updateState() {
    synchronized (sync) {
      boolean allComplete = true;

      for (ExecutableFlow flow : flows) {
        switch (flow.getStatus()) {
          case FAILED:
            jobState = Status.FAILED;
            returnProps = new Props();
            return;
          case COMPLETED:
          case SUCCEEDED:
            continue;
          default:
            allComplete = false;
        }
      }

      if (allComplete) {
        jobState = Status.SUCCEEDED;

        returnProps = new Props();

        for (ExecutableFlow flow : flows) {
          returnProps = new Props(returnProps, flow.getReturnProps());
        }

        returnProps.logProperties("Output properties for " + getName());
      }
    }
  }
 @Override
 public boolean cancel() {
   boolean retVal = true;
   for (ExecutableFlow flow : flows) {
     retVal &= flow.cancel();
   }
   return retVal;
 }
  @Override
  public void execute(Props parentProps, final FlowCallback callback) {
    if (parentProps == null) {
      parentProps = new Props();
    }

    synchronized (sync) {
      if (this.parentProps == null) {
        this.parentProps = parentProps;
      } else if (jobState != Status.COMPLETED && !this.parentProps.equalsProps(parentProps)) {
        throw new IllegalArgumentException(
            String.format(
                "%s.execute() called with multiple differing parentProps objects.  "
                    + "Call reset() before executing again with a different Props object. this.parentProps[%s], parentProps[%s]",
                getClass().getSimpleName(), this.parentProps, parentProps));
      }

      switch (jobState) {
        case READY:
          jobState = Status.RUNNING;
          callbacksToCall.add(callback);
          break;
        case RUNNING:
          callbacksToCall.add(callback);
          return;
        case COMPLETED:
        case SUCCEEDED:
        case IGNORED:
          callback.completed(Status.SUCCEEDED);
          return;
        case FAILED:
          callback.completed(Status.FAILED);
          return;
      }
    }

    if (startTime == null) {
      startTime = new DateTime();
    }

    for (ExecutableFlow flow : flows) {
      if (jobState != Status.FAILED) {
        try {
          flow.execute(this.parentProps, theGroupCallback);
        } catch (RuntimeException e) {
          final List<FlowCallback> callbacks;
          synchronized (sync) {
            jobState = Status.FAILED;
            callbacks = callbacksToCall;
          }

          callCallbacks(callbacks, Status.FAILED);

          throw e;
        }
      }
    }
  }
  @Test
  public void testInitializationSecondSucceeded() throws Exception {
    DateTime expectedStartTime = new DateTime(0);

    EasyMock.expect(mockFlow1.getName()).andReturn("a").once();
    EasyMock.expect(mockFlow2.getName()).andReturn("b").once();

    EasyMock.expect(mockFlow1.getStatus()).andReturn(Status.READY).times(3);
    EasyMock.expect(mockFlow2.getStatus()).andReturn(Status.SUCCEEDED).times(3);

    EasyMock.expect(mockFlow1.getStartTime()).andReturn(null).once();
    EasyMock.expect(mockFlow2.getStartTime()).andReturn(expectedStartTime).once();

    EasyMock.expect(mockFlow2.getParentProps()).andReturn(props).once();

    EasyMock.expect(mockFlow1.getName()).andReturn("1").once();
    EasyMock.expect(mockFlow2.getName()).andReturn("2").once();

    EasyMock.replay(mockFlow1, mockFlow2, props);

    flow = new GroupedExecutableFlow("blah", mockFlow1, mockFlow2);

    Assert.assertEquals(Status.READY, flow.getStatus());
    Assert.assertEquals(expectedStartTime, flow.getStartTime());
    Assert.assertEquals(null, flow.getEndTime());
    // <<<<<<< HEAD:azkaban/src/unit/azkaban/flow/GroupedExecutableFlowTest.java
    Assert.assertEquals(emptyExceptions, flow.getExceptions());
    Assert.assertEquals(props, flow.getParentProps());
  }
Beispiel #5
0
  public static ExecutableFlow resetFailedFlows(final ExecutableFlow theFlow) {
    if (theFlow.getStatus() == Status.FAILED) {
      theFlow.reset();
    }

    if (theFlow.hasChildren()) {
      for (ExecutableFlow flow : theFlow.getChildren()) {
        resetFailedFlows(flow);
      }
    }

    return theFlow;
  }
  @Before
  public void setUp() throws Exception {
    props = EasyMock.createStrictMock(Props.class);
    mockFlow1 = EasyMock.createMock(ExecutableFlow.class);
    mockFlow2 = EasyMock.createMock(ExecutableFlow.class);

    EasyMock.expect(mockFlow1.getName()).andReturn("a").once();
    EasyMock.expect(mockFlow2.getName()).andReturn("b").once();

    EasyMock.expect(mockFlow1.getStatus()).andReturn(Status.READY).times(3);
    EasyMock.expect(mockFlow2.getStatus()).andReturn(Status.READY).times(3);

    EasyMock.expect(mockFlow1.getStartTime()).andReturn(null).once();
    EasyMock.expect(mockFlow2.getStartTime()).andReturn(null).once();

    EasyMock.expect(mockFlow1.getName()).andReturn("1").once();
    EasyMock.expect(mockFlow2.getName()).andReturn("2").once();
    EasyMock.replay(mockFlow1, mockFlow2, props);

    flow = new GroupedExecutableFlow("blah", mockFlow1, mockFlow2);
    Assert.assertEquals("1 + 2", flow.getName());

    EasyMock.verify(mockFlow1, mockFlow2, props);
    EasyMock.reset(mockFlow1, mockFlow2, props);
  }
  private void setAndVerifyParentProps() {
    for (ExecutableFlow flow : flows) {
      if (flow.getStatus() == Status.READY) {
        continue;
      }

      final Props childsParentProps = flow.getParentProps();

      if (parentProps == null) {
        parentProps = childsParentProps;
      } else {
        if (childsParentProps != null && !parentProps.equalsProps(childsParentProps)) {
          throw new IllegalStateException(
              String.format("Parent props differ for sub flows. Flow Id[%s]", id));
        }
      }
    }
  }
    @Override
    public void completed(final Status status) {
      final List<FlowCallback> callbackList;
      synchronized (sync) {
        updateState();
        callbackList = callbacksToCall; // Get the reference before leaving the synchronized
      }

      if (jobState == Status.SUCCEEDED && notifiedCallbackAlready.compareAndSet(false, true)) {
        callCallbacks(callbackList, Status.SUCCEEDED);
      } else if (jobState == Status.FAILED && notifiedCallbackAlready.compareAndSet(false, true)) {
        for (ExecutableFlow flow : flows) {
          exceptions.putAll(flow.getExceptions());
        }
        callCallbacks(callbackList, Status.FAILED);
      } else {
        for (FlowCallback flowCallback : callbackList) {
          flowCallback.progressMade();
        }
      }
    }
  public GroupedExecutableFlow(String id, ExecutableFlow... flows) {
    this.id = id;
    this.flows = flows;
    this.sortedFlows = Arrays.copyOf(this.flows, this.flows.length);
    Arrays.sort(
        this.sortedFlows,
        new Comparator<ExecutableFlow>() {
          @Override
          public int compare(ExecutableFlow o1, ExecutableFlow o2) {
            return o1.getName().compareTo(o2.getName());
          }
        });

    String[] names = new String[flows.length];
    for (int i = 0; i < flows.length; i++) {
      names[i] = flows[i].getName();
    }
    name = StringUtils.join(names, " + ");

    jobState = Status.READY;
    updateState();
    callbacksToCall = new ArrayList<FlowCallback>();

    theGroupCallback = new GroupedFlowCallback();

    switch (jobState) {
      case SUCCEEDED:
      case COMPLETED:
      case FAILED:
        DateTime theStartTime = new DateTime();
        DateTime theEndTime = new DateTime(0);
        for (ExecutableFlow flow : flows) {
          final DateTime subFlowStartTime = flow.getStartTime();
          if (theStartTime.isAfter(subFlowStartTime)) {
            theStartTime = subFlowStartTime;
          }

          final DateTime subFlowEndTime = flow.getEndTime();
          if (subFlowEndTime != null && subFlowEndTime.isAfter(theEndTime)) {
            theEndTime = subFlowEndTime;
          }
        }

        setAndVerifyParentProps();
        startTime = theStartTime;
        endTime = theEndTime;
        break;
      default:
        // Check for Flows that are "RUNNING"
        boolean allRunning = true;
        List<ExecutableFlow> runningFlows = new ArrayList<ExecutableFlow>();
        DateTime thisStartTime = null;

        for (ExecutableFlow flow : flows) {
          if (flow.getStatus() != Status.RUNNING) {
            allRunning = false;

            final DateTime subFlowStartTime = flow.getStartTime();
            if (subFlowStartTime != null && subFlowStartTime.isBefore(thisStartTime)) {
              thisStartTime = subFlowStartTime;
            }
          } else {
            runningFlows.add(flow);
          }
        }

        if (allRunning) {
          jobState = Status.RUNNING;
        }

        for (ExecutableFlow runningFlow : runningFlows) {
          final DateTime subFlowStartTime = runningFlow.getStartTime();
          if (subFlowStartTime != null && subFlowStartTime.isBefore(thisStartTime)) {
            thisStartTime = subFlowStartTime;
          }
        }
        setAndVerifyParentProps();

        startTime = thisStartTime;
        endTime = null;

        // Make sure everything is initialized before leaking the pointer to "this".
        // This is just installing the callback in an already running flow.
        for (ExecutableFlow runningFlow : runningFlows) {
          runningFlow.execute(parentProps, theGroupCallback);
        }
    }
  }
  @Test
  public void testInitializationBothRunning() throws Exception {
    DateTime expectedStartTime = new DateTime(0);
    DateTime falseStartTime = new DateTime(1);

    EasyMock.expect(mockFlow1.getName()).andReturn("a").once();
    EasyMock.expect(mockFlow2.getName()).andReturn("b").once();

    EasyMock.expect(mockFlow1.getStatus()).andReturn(Status.RUNNING).times(3);
    EasyMock.expect(mockFlow2.getStatus()).andReturn(Status.RUNNING).times(3);

    EasyMock.expect(mockFlow1.getStartTime()).andReturn(falseStartTime).once();
    EasyMock.expect(mockFlow2.getStartTime()).andReturn(expectedStartTime).once();

    Capture<FlowCallback> callbackCapture1 = new Capture<FlowCallback>();
    Capture<FlowCallback> callbackCapture2 = new Capture<FlowCallback>();
    mockFlow1.execute(EasyMock.eq(props), EasyMock.capture(callbackCapture1));
    mockFlow2.execute(EasyMock.eq(props), EasyMock.capture(callbackCapture2));

    EasyMock.expect(mockFlow1.getParentProps()).andReturn(props).once();
    EasyMock.expect(mockFlow2.getParentProps()).andReturn(props).once();
    EasyMock.expect(props.equalsProps(props)).andReturn(true).once();

    EasyMock.expect(mockFlow1.getName()).andReturn("1").once();
    EasyMock.expect(mockFlow2.getName()).andReturn("2").once();

    EasyMock.replay(mockFlow1, mockFlow2, props);

    flow = new GroupedExecutableFlow("blah", mockFlow1, mockFlow2);

    Assert.assertEquals(Status.RUNNING, flow.getStatus());
    Assert.assertEquals(expectedStartTime, flow.getStartTime());
    Assert.assertEquals(null, flow.getEndTime());

    EasyMock.verify(mockFlow1, mockFlow2, props);
    EasyMock.reset(mockFlow1, mockFlow2, props);

    EasyMock.expect(mockFlow1.getStatus()).andReturn(Status.RUNNING).once();
    EasyMock.expect(mockFlow2.getStatus()).andReturn(Status.SUCCEEDED).once();

    EasyMock.replay(mockFlow1, mockFlow2, props);

    Assert.assertSame(callbackCapture1.getValue(), callbackCapture2.getValue());

    callbackCapture1.getValue().completed(Status.SUCCEEDED);

    Assert.assertEquals(Status.RUNNING, flow.getStatus());
    Assert.assertEquals(expectedStartTime, flow.getStartTime());
    Assert.assertEquals(null, flow.getEndTime());
    Assert.assertEquals(emptyExceptions, flow.getExceptions());
    Assert.assertEquals(props, flow.getParentProps());

    EasyMock.verify(mockFlow1, mockFlow2, props);
    EasyMock.reset(mockFlow1, mockFlow2, props);

    EasyMock.expect(mockFlow1.getStatus()).andReturn(Status.SUCCEEDED).once();
    EasyMock.expect(mockFlow2.getStatus()).andReturn(Status.SUCCEEDED).once();

    EasyMock.expect(mockFlow1.getReturnProps()).andReturn(new Props()).once();
    EasyMock.expect(mockFlow2.getReturnProps()).andReturn(new Props()).once();

    EasyMock.replay(mockFlow1, mockFlow2, props);

    DateTime beforeTheEnd = new DateTime();
    callbackCapture2.getValue().completed(Status.SUCCEEDED);

    Assert.assertEquals(Status.SUCCEEDED, flow.getStatus());
    Assert.assertEquals(expectedStartTime, flow.getStartTime());
    Assert.assertFalse(
        String.format(
            "flow's end time[%s] should be after beforeTheEnd[%s]",
            flow.getEndTime(), beforeTheEnd),
        beforeTheEnd.isAfter(flow.getEndTime()));
    Assert.assertEquals(emptyExceptions, flow.getExceptions());
  }
  @Test
  public void testInitializationSecondRunning() throws Exception {
    DateTime expectedStartTime = new DateTime(0);

    EasyMock.expect(mockFlow1.getName()).andReturn("a").once();
    EasyMock.expect(mockFlow2.getName()).andReturn("b").once();

    EasyMock.expect(mockFlow1.getStatus()).andReturn(Status.READY).times(3);
    EasyMock.expect(mockFlow2.getStatus()).andReturn(Status.RUNNING).times(3);

    EasyMock.expect(mockFlow1.getStartTime()).andReturn(null).once();
    EasyMock.expect(mockFlow2.getStartTime()).andReturn(expectedStartTime).once();

    Capture<FlowCallback> callbackCapture = new Capture<FlowCallback>();
    mockFlow2.execute(EasyMock.eq(props), EasyMock.capture(callbackCapture));

    EasyMock.expect(mockFlow1.getName()).andReturn("1").once();
    EasyMock.expect(mockFlow2.getName()).andReturn("2").once();

    EasyMock.expect(mockFlow2.getParentProps()).andReturn(props).once();

    EasyMock.replay(mockFlow1, mockFlow2, props);

    flow = new GroupedExecutableFlow("blah", mockFlow1, mockFlow2);

    Assert.assertEquals(Status.READY, flow.getStatus());
    Assert.assertEquals(expectedStartTime, flow.getStartTime());
    Assert.assertEquals(null, flow.getEndTime());
    Assert.assertEquals(props, flow.getParentProps());

    EasyMock.verify(mockFlow1, mockFlow2, props);
    EasyMock.reset(mockFlow1, mockFlow2, props);

    EasyMock.expect(mockFlow1.getStatus()).andReturn(Status.READY).once();
    EasyMock.expect(mockFlow2.getStatus()).andReturn(Status.SUCCEEDED).once();

    EasyMock.replay(mockFlow1, mockFlow2, props);

    callbackCapture.getValue().completed(Status.SUCCEEDED);

    Assert.assertEquals(Status.READY, flow.getStatus());
    Assert.assertEquals(expectedStartTime, flow.getStartTime());
    Assert.assertEquals(null, flow.getEndTime());
    Assert.assertEquals(emptyExceptions, flow.getExceptions());
  }
  @Test
  public void testInitializationSecondFailed() throws Exception {
    DateTime expectedStartTime = new DateTime(0);
    DateTime falseStartTime = new DateTime(1);
    DateTime expectedEndTime = new DateTime(100);
    DateTime falseEndTime = new DateTime(99);

    EasyMock.expect(mockFlow1.getName()).andReturn("a").once();
    EasyMock.expect(mockFlow2.getName()).andReturn("b").once();

    EasyMock.expect(mockFlow1.getStatus()).andReturn(Status.READY).times(2);
    EasyMock.expect(mockFlow2.getStatus()).andReturn(Status.FAILED).times(2);

    EasyMock.expect(mockFlow1.getStartTime()).andReturn(expectedStartTime).once();
    EasyMock.expect(mockFlow1.getEndTime()).andReturn(falseEndTime).once();
    EasyMock.expect(mockFlow2.getStartTime()).andReturn(falseStartTime).once();
    EasyMock.expect(mockFlow2.getEndTime()).andReturn(expectedEndTime).once();

    EasyMock.expect(mockFlow2.getParentProps()).andReturn(props).once();

    EasyMock.expect(mockFlow1.getName()).andReturn("1").once();
    EasyMock.expect(mockFlow2.getName()).andReturn("2").once();

    EasyMock.replay(mockFlow1, mockFlow2, props);

    flow = new GroupedExecutableFlow("blah", mockFlow1, mockFlow2);

    Assert.assertEquals(Status.FAILED, flow.getStatus());
    Assert.assertEquals(expectedStartTime, flow.getStartTime());
    Assert.assertEquals(expectedEndTime, flow.getEndTime());
    Assert.assertEquals(emptyExceptions, flow.getExceptions());
    Assert.assertEquals(props, flow.getParentProps());
  }
  @Test
  public void testSanity() throws Exception {
    final AtomicLong numJobsComplete = new AtomicLong(0);

    /** ** Setup mockFlow1 *** */
    final Capture<FlowCallback> flow1Callback = new Capture<FlowCallback>();
    mockFlow1.execute(EasyMock.eq(props), EasyMock.capture(flow1Callback));
    EasyMock.expectLastCall()
        .andAnswer(
            new IAnswer<Void>() {
              @Override
              public Void answer() throws Throwable {
                Assert.assertEquals(Status.RUNNING, flow.getStatus());
                Assert.assertEquals(1, numJobsComplete.incrementAndGet());

                flow1Callback.getValue().completed(Status.SUCCEEDED);

                Assert.assertEquals(Status.RUNNING, flow.getStatus());

                return null;
              }
            })
        .once();

    Props mockFlow1Props = new Props();
    mockFlow1Props.put("1", "1");
    mockFlow1Props.put("2", "1");

    EasyMock.expect(mockFlow1.getStatus()).andReturn(Status.SUCCEEDED).times(2);
    EasyMock.expect(mockFlow1.getReturnProps()).andReturn(mockFlow1Props).once();

    /** ** Setup mockFlow2 *** */
    final Capture<FlowCallback> flow2Callback = new Capture<FlowCallback>();
    mockFlow2.execute(EasyMock.eq(props), EasyMock.capture(flow2Callback));
    EasyMock.expectLastCall()
        .andAnswer(
            new IAnswer<Void>() {
              @Override
              public Void answer() throws Throwable {
                Assert.assertEquals(Status.RUNNING, flow.getStatus());
                Assert.assertEquals(2, numJobsComplete.incrementAndGet());

                flow2Callback.getValue().completed(Status.SUCCEEDED);

                Assert.assertEquals(Status.SUCCEEDED, flow.getStatus());

                return null;
              }
            })
        .once();

    EasyMock.expect(mockFlow2.getStatus())
        .andAnswer(
            new IAnswer<Status>() {
              private volatile AtomicInteger count = new AtomicInteger(0);

              @Override
              public Status answer() throws Throwable {
                switch (count.getAndIncrement()) {
                  case 0:
                    return Status.READY;
                  case 1:
                    return Status.SUCCEEDED;
                  default:
                    Assert.fail("mockFlow2.getStatus() should only be called 2 times.");
                }
                return null;
              }
            })
        .times(2);

    Props mockFlow2Props = new Props();
    mockFlow2Props.put("2", "2");
    mockFlow2Props.put("3", "2");
    EasyMock.expect(mockFlow2.getReturnProps()).andReturn(mockFlow2Props).once();

    EasyMock.expect(props.equalsProps(props)).andReturn(true).once();

    EasyMock.replay(mockFlow1, mockFlow2, props);

    /** ** Start the test *** */
    AtomicBoolean callbackRan = new AtomicBoolean(false);
    flow.execute(
        props,
        new OneCallFlowCallback(callbackRan) {
          @Override
          public void theCallback(Status status) {
            Assert.assertEquals(Status.SUCCEEDED, status);
            Assert.assertEquals(2, numJobsComplete.get());
          }
        });

    Assert.assertTrue("Callback wasn't run.", callbackRan.get());
    Assert.assertEquals(Status.SUCCEEDED, flow.getStatus());
    Assert.assertEquals(emptyExceptions, flow.getExceptions());
    Assert.assertEquals(props, flow.getParentProps());

    callbackRan = new AtomicBoolean(false);
    flow.execute(
        props,
        new OneCallFlowCallback(callbackRan) {
          @Override
          protected void theCallback(Status status) {
            Assert.assertEquals(Status.SUCCEEDED, status);
            Assert.assertEquals(2, numJobsComplete.get());
          }
        });

    Assert.assertTrue("Callback wasn't run.", callbackRan.get());
    Assert.assertEquals(Status.SUCCEEDED, flow.getStatus());
    Assert.assertEquals(emptyExceptions, flow.getExceptions());

    Props retProps = flow.getReturnProps();
    Assert.assertEquals(3, retProps.size());
    Assert.assertEquals("1", retProps.get("1"));
    Assert.assertEquals("2", retProps.get("2"));
    Assert.assertEquals("2", retProps.get("3"));

    EasyMock.verify(props);
    EasyMock.reset(props);

    EasyMock.expect(props.equalsProps(props)).andReturn(false).once();

    EasyMock.replay(props);

    boolean exceptionThrown = false;
    try {
      flow.execute(
          props,
          new FlowCallback() {
            @Override
            public void progressMade() {}

            @Override
            public void completed(Status status) {}
          });
    } catch (IllegalArgumentException e) {
      exceptionThrown = true;
    }

    Assert.assertTrue(
        "Expected an IllegalArgumentException to be thrown because props weren't the same.",
        exceptionThrown);
  }
  @Test
  public void testAllCallbacksCalled() throws Exception {
    final AtomicLong numJobsComplete = new AtomicLong(0);
    final AtomicBoolean executeCallWhileStateWasRunningHadItsCallbackCalled =
        new AtomicBoolean(false);

    /** ** Setup mockFlow1 *** */
    final Capture<FlowCallback> flow1Callback = new Capture<FlowCallback>();
    mockFlow1.execute(EasyMock.eq(props), EasyMock.capture(flow1Callback));
    EasyMock.expectLastCall()
        .andAnswer(
            new IAnswer<Void>() {
              @Override
              public Void answer() throws Throwable {
                Assert.assertEquals(Status.RUNNING, flow.getStatus());
                Assert.assertEquals(1, numJobsComplete.incrementAndGet());

                flow.execute(
                    props,
                    new OneCallFlowCallback(executeCallWhileStateWasRunningHadItsCallbackCalled) {
                      @Override
                      protected void theCallback(Status status) {}
                    });

                flow1Callback.getValue().completed(Status.SUCCEEDED);

                return null;
              }
            })
        .once();

    EasyMock.expect(mockFlow1.getStatus()).andReturn(Status.SUCCEEDED).times(2);

    /** ** Setup mockFlow2 *** */
    final Capture<FlowCallback> flow2Callback = new Capture<FlowCallback>();
    mockFlow2.execute(EasyMock.eq(props), EasyMock.capture(flow2Callback));
    EasyMock.expectLastCall()
        .andAnswer(
            new IAnswer<Void>() {
              @Override
              public Void answer() throws Throwable {
                Assert.assertEquals(Status.RUNNING, flow.getStatus());
                Assert.assertEquals(2, numJobsComplete.incrementAndGet());

                flow2Callback.getValue().completed(Status.SUCCEEDED);

                return null;
              }
            })
        .once();

    EasyMock.expect(mockFlow2.getStatus())
        .andAnswer(
            new IAnswer<Status>() {
              private volatile AtomicInteger count = new AtomicInteger(0);

              @Override
              public Status answer() throws Throwable {
                switch (count.getAndIncrement()) {
                  case 0:
                    return Status.READY;
                  case 1:
                    return Status.SUCCEEDED;
                  default:
                    Assert.fail("mockFlow2.getStatus() should only be called 2 times.");
                }
                return null;
              }
            })
        .times(2);

    EasyMock.expect(mockFlow1.getReturnProps()).andReturn(new Props()).once();
    EasyMock.expect(mockFlow2.getReturnProps()).andReturn(new Props()).once();

    EasyMock.expect(props.equalsProps(props)).andReturn(true).times(2);

    EasyMock.replay(mockFlow1, mockFlow2, props);

    /** ** Start the test *** */
    AtomicBoolean callbackRan = new AtomicBoolean(false);
    flow.execute(
        props,
        new OneCallFlowCallback(callbackRan) {
          @Override
          public void theCallback(Status status) {
            Assert.assertEquals(Status.SUCCEEDED, status);
            Assert.assertEquals(2, numJobsComplete.get());
          }
        });

    Assert.assertTrue("Callback wasn't run.", callbackRan.get());
    Assert.assertEquals(Status.SUCCEEDED, flow.getStatus());
    Assert.assertTrue(
        "mockFlow1, upon completion, sends another execute() call to the flow.  "
            + "The callback from that execute call was apparently not called.",
        executeCallWhileStateWasRunningHadItsCallbackCalled.get());
    Assert.assertEquals(emptyExceptions, flow.getExceptions());

    callbackRan = new AtomicBoolean(false);
    flow.execute(
        props,
        new OneCallFlowCallback(callbackRan) {
          @Override
          protected void theCallback(Status status) {
            Assert.assertEquals(Status.SUCCEEDED, status);
            Assert.assertEquals(2, numJobsComplete.get());
          }
        });

    Assert.assertTrue("Callback wasn't run.", callbackRan.get());
    Assert.assertEquals(Status.SUCCEEDED, flow.getStatus());
    Assert.assertEquals(emptyExceptions, flow.getExceptions());
  }
  @Test
  public void testFailureJob2() throws Exception {

    final AtomicLong numJobsComplete = new AtomicLong(0);

    /** ** Setup mockFlow1 *** */
    final Capture<FlowCallback> flow1Callback = new Capture<FlowCallback>();
    mockFlow1.execute(EasyMock.eq(props), EasyMock.capture(flow1Callback));
    EasyMock.expectLastCall()
        .andAnswer(
            new IAnswer<Void>() {
              @Override
              public Void answer() throws Throwable {
                Assert.assertEquals(Status.RUNNING, flow.getStatus());
                Assert.assertEquals(1, numJobsComplete.incrementAndGet());

                flow1Callback.getValue().completed(Status.SUCCEEDED);

                return null;
              }
            })
        .once();

    EasyMock.expect(mockFlow1.getStatus()).andReturn(Status.SUCCEEDED).times(2);
    EasyMock.expect(mockFlow1.getExceptions()).andReturn(emptyExceptions).times(1);

    /** ** Setup mockFlow2 *** */
    final Capture<FlowCallback> flow2Callback = new Capture<FlowCallback>();
    mockFlow2.execute(EasyMock.eq(props), EasyMock.capture(flow2Callback));
    EasyMock.expectLastCall()
        .andAnswer(
            new IAnswer<Void>() {
              @Override
              public Void answer() throws Throwable {
                Assert.assertEquals(Status.RUNNING, flow.getStatus());
                Assert.assertEquals(2, numJobsComplete.incrementAndGet());

                flow2Callback.getValue().completed(Status.FAILED);

                return null;
              }
            })
        .once();

    EasyMock.expect(mockFlow2.getStatus())
        .andAnswer(
            new IAnswer<Status>() {
              private volatile AtomicInteger count = new AtomicInteger(0);

              @Override
              public Status answer() throws Throwable {
                switch (count.getAndIncrement()) {
                  case 0:
                    return Status.READY;
                  case 1:
                    return Status.FAILED;
                  default:
                    Assert.fail("mockFlow2.getStatus() should only be called 2 times.");
                }
                return null;
              }
            })
        .times(2);

    final RuntimeException e1 = new RuntimeException();
    final RuntimeException e2 = new RuntimeException();

    final Map<String, Throwable> e1s = new HashMap<String, Throwable>();
    e1s.put("e1", e1);
    e1s.put("e2", e2);

    EasyMock.expect(mockFlow2.getExceptions()).andReturn(e1s).times(1);
    EasyMock.expect(props.equalsProps(props)).andReturn(true).once();

    EasyMock.replay(mockFlow1, mockFlow2, props);

    /** ** Start the test *** */
    AtomicBoolean callbackRan = new AtomicBoolean(false);
    flow.execute(
        props,
        new OneCallFlowCallback(callbackRan) {
          @Override
          public void theCallback(Status status) {
            Assert.assertEquals(Status.FAILED, status);
          }
        });

    Assert.assertTrue("Callback wasn't run.", callbackRan.get());
    Assert.assertEquals(Status.FAILED, flow.getStatus());
    Assert.assertEquals(e1s, flow.getExceptions());

    callbackRan = new AtomicBoolean(false);
    flow.execute(
        props,
        new OneCallFlowCallback(callbackRan) {
          @Override
          protected void theCallback(Status status) {
            Assert.assertEquals(Status.FAILED, status);
            Assert.assertEquals(2, numJobsComplete.get());
          }
        });

    Assert.assertTrue("Callback wasn't run.", callbackRan.get());
    Assert.assertEquals(Status.FAILED, flow.getStatus());
    Assert.assertEquals(e1s, flow.getExceptions());

    Assert.assertTrue("Expected to be able to reset the flow", flow.reset());
    Assert.assertEquals(Status.READY, flow.getStatus());
    Assert.assertEquals(emptyExceptions, flow.getExceptions());
  }
  @Test
  public void testFailureJob1() throws Exception {

    final AtomicLong numJobsComplete = new AtomicLong(0);

    /** ** Setup mockFlow1 *** */
    final Capture<FlowCallback> flow1Callback = new Capture<FlowCallback>();
    mockFlow1.execute(EasyMock.eq(props), EasyMock.capture(flow1Callback));
    EasyMock.expectLastCall()
        .andAnswer(
            new IAnswer<Void>() {
              @Override
              public Void answer() throws Throwable {
                Assert.assertEquals(Status.RUNNING, flow.getStatus());
                Assert.assertEquals(1, numJobsComplete.incrementAndGet());

                flow1Callback.getValue().completed(Status.FAILED);

                return null;
              }
            })
        .once();

    EasyMock.expect(mockFlow1.getStatus()).andReturn(Status.FAILED).times(1);
    EasyMock.expect(mockFlow1.getExceptions()).andReturn(theExceptions).times(1);

    EasyMock.expect(props.equalsProps(props)).andReturn(true).once();

    /** ** Setup mockFlow2 *** */
    EasyMock.expect(mockFlow2.getExceptions()).andReturn(emptyExceptions).times(1);
    EasyMock.replay(mockFlow1, mockFlow2, props);

    /** ** Start the test *** */
    AtomicBoolean callbackRan = new AtomicBoolean(false);
    flow.execute(
        props,
        new OneCallFlowCallback(callbackRan) {
          @Override
          public void theCallback(Status status) {
            Assert.assertEquals(Status.FAILED, status);
          }
        });

    Assert.assertTrue("Callback wasn't run.", callbackRan.get());
    Assert.assertEquals(Status.FAILED, flow.getStatus());
    Assert.assertEquals(theExceptions, flow.getExceptions());

    callbackRan = new AtomicBoolean(false);
    flow.execute(
        props,
        new OneCallFlowCallback(callbackRan) {
          @Override
          protected void theCallback(Status status) {
            Assert.assertEquals(Status.FAILED, status);
          }
        });

    Assert.assertTrue("Callback wasn't run.", callbackRan.get());
    Assert.assertEquals(Status.FAILED, flow.getStatus());
    Assert.assertEquals(theExceptions, flow.getExceptions());

    Assert.assertTrue("Expected to be able to reset the flow", flow.reset());
    Assert.assertEquals(Status.READY, flow.getStatus());
    Assert.assertEquals(emptyExceptions, flow.getExceptions());
  }