/**
   * Unit test for write pipeline add/remove.
   *
   * @throws InterruptedException
   */
  public void test_pipelineAddRemove() throws InterruptedException {

    final Quorum<?, ?> quorum = quorums[0];
    final MockQuorumMember<?> client = clients[0];
    final QuorumActor<?, ?> actor = actors[0];
    final UUID serviceId = client.getServiceId();

    assertFalse(client.isMember());
    assertNull(client.downStreamId);
    assertFalse(client.isPipelineMember());
    assertEquals(new UUID[] {}, quorum.getPipeline());

    actor.memberAdd();
    fixture.awaitDeque();

    /*
     * add to the pipeline. since this is a singleton quorum, the downstream
     * service will remain null.
     */
    assertNull(client.downStreamId);
    actor.pipelineAdd();
    fixture.awaitDeque();

    assertNull(client.downStreamId);
    assertTrue(client.isPipelineMember());
    assertEquals(new UUID[] {serviceId}, quorum.getPipeline());

    /*
     * remove from the pipeline. since this is a singleton quorum, the
     * downstream service will remain null.
     */
    assertNull(client.downStreamId);
    actor.pipelineRemove();
    fixture.awaitDeque();

    assertNull(client.downStreamId);
    assertFalse(client.isPipelineMember());
    assertEquals(new UUID[] {}, quorum.getPipeline());

    actor.memberRemove();
    fixture.awaitDeque();

    assertFalse(client.isMember());
  }
  /**
   * Unit test of timeout in {@link Quorum#awaitQuorum(long, TimeUnit)}. and {@link
   * Quorum#awaitBreak(long, TimeUnit)}.
   *
   * @throws AsynchronousQuorumCloseException
   * @throws InterruptedException
   */
  public void test_awaitQuorum() throws AsynchronousQuorumCloseException, InterruptedException {

    final AbstractQuorum<?, ?> quorum = quorums[0];
    final MockQuorumMember<?> client = clients[0];
    final QuorumActor<?, ?> actor = actors[0];
    final UUID serviceId = client.getServiceId();

    final long lastCommitTime = 0L;
    final long lastCommitTime2 = 2L;

    // declare the service as a quorum member.
    actor.memberAdd();
    fixture.awaitDeque();

    assertTrue(client.isMember());
    assertEquals(new UUID[] {serviceId}, quorum.getMembers());

    // add to the pipeline.
    actor.pipelineAdd();
    fixture.awaitDeque();

    assertTrue(client.isPipelineMember());
    assertEquals(new UUID[] {serviceId}, quorum.getPipeline());

    final long timeout = 1500; // ms
    final long slop = 100; // margin of error.
    {
      /*
       * Verify that a we timeout when awaiting a quorum meet that does
       * not occur.
       */
      final AtomicLong didTimeout = new AtomicLong(-1L);

      final Thread t =
          new Thread() {
            public void run() {
              final long begin = System.currentTimeMillis();
              try {
                // wait for a quorum (but will not meet).
                log.info("Waiting for quorum meet.");
                quorum.awaitQuorum(timeout, TimeUnit.MILLISECONDS);
              } catch (TimeoutException e) {
                // This is what we are looking for.
                final long elapsed = System.currentTimeMillis() - begin;
                didTimeout.set(elapsed);
                if (log.isInfoEnabled()) log.info("Timeout after " + elapsed + "ms");
              } catch (Exception e) {
                log.error(e, e);
              }
            }
          };
      t.run();
      Thread.sleep(timeout + 250 /* ms */);
      t.interrupt();
      final long elapsed = didTimeout.get();
      assertTrue("did not timeout", elapsed != -1);
      assertTrue(
          "Timeout occurred too soon: elapsed=" + elapsed + ",timeout=" + timeout,
          elapsed >= timeout);
      assertTrue(
          "Timeout took too long: elapsed=" + elapsed + ",timeout=" + timeout,
          elapsed < (timeout + slop));
    }

    // cast a vote for a lastCommitTime.
    actor.castVote(lastCommitTime);
    fixture.awaitDeque();

    assertEquals(1, quorum.getVotes().size());
    assertEquals(new UUID[] {serviceId}, quorum.getVotes().get(lastCommitTime));

    // verify the consensus was updated.
    assertEquals(lastCommitTime, client.lastConsensusValue);

    // wait for quorum meet.
    final long token1 = quorum.awaitQuorum();

    // verify service was joined.
    assertTrue(client.isJoinedMember(quorum.token()));
    assertEquals(new UUID[] {serviceId}, quorum.getJoined());

    // validate the token was assigned.
    fixture.awaitDeque();
    assertEquals(Quorum.NO_QUORUM + 1, quorum.lastValidToken());
    assertEquals(Quorum.NO_QUORUM + 1, quorum.token());
    assertTrue(quorum.isQuorumMet());

    {
      /*
       * Verify that we timeout when awaiting a quorum break that does not
       * occur.
       */
      final AtomicLong didTimeout = new AtomicLong(-1L);

      final Thread t =
          new Thread() {
            public void run() {
              final long begin = System.currentTimeMillis();
              try {
                // wait for a quorum break (but will not break).
                log.info("Waiting for quorum break.");
                quorum.awaitBreak(timeout, TimeUnit.MILLISECONDS);
              } catch (TimeoutException e) {
                // This is what we are looking for.
                final long elapsed = System.currentTimeMillis() - begin;
                didTimeout.set(elapsed);
                if (log.isInfoEnabled()) log.error("Timeout after " + elapsed + "ms");
              } catch (Exception e) {
                log.error(e, e);
              }
            }
          };
      t.run();
      Thread.sleep(timeout + 250 /* ms */);
      t.interrupt();
      final long elapsed = didTimeout.get();
      assertTrue("did not timeout", elapsed != -1);
      assertTrue(
          "Timeout occurred too soon: elapsed=" + elapsed + ",timeout=" + timeout,
          elapsed >= timeout);
      assertTrue(
          "Timeout took too long: elapsed=" + elapsed + ",timeout=" + timeout,
          elapsed < (timeout + slop));
    }

    try {
      // Verify awaitBreak() does not return normally.
      quorum.awaitBreak(1, TimeUnit.MILLISECONDS);
      fail("Not expecting quorum break");
    } catch (TimeoutException e) {
      if (log.isInfoEnabled()) log.info("Ignoring expected excption: " + e);
    }

    /*
     * Do service leave, quorum should break.
     */

    actor.serviceLeave();
    fixture.awaitDeque();

    quorum.awaitBreak();

    try {
      // Verify awaitBreak() returns normally.
      quorum.awaitBreak(1, TimeUnit.MILLISECONDS);
    } catch (TimeoutException e) {
      fail("Not expecting " + e, e);
    }
  }
  /**
   * Unit test verifying that we clear down the quorum's reflection of the distributed quorum state
   * where we first have a quorum meet and then terminate the quorum client.
   *
   * @throws InterruptedException
   */
  public void test_serviceJoin_terminateClient() throws InterruptedException {

    final AbstractQuorum<?, ?> quorum = quorums[0];
    final MockQuorumMember<?> client = clients[0];
    final QuorumActor<?, ?> actor = actors[0];
    final UUID serviceId = client.getServiceId();

    final long lastCommitTime = 0L;
    final long lastCommitTime2 = 2L;

    // declare the service as a quorum member.
    actor.memberAdd();
    fixture.awaitDeque();

    assertTrue(client.isMember());
    assertEquals(new UUID[] {serviceId}, quorum.getMembers());

    // add to the pipeline.
    actor.pipelineAdd();
    fixture.awaitDeque();

    assertTrue(client.isPipelineMember());
    assertEquals(new UUID[] {serviceId}, quorum.getPipeline());

    // cast a vote for a lastCommitTime.
    actor.castVote(lastCommitTime);
    fixture.awaitDeque();

    assertEquals(1, quorum.getVotes().size());
    assertEquals(new UUID[] {serviceId}, quorum.getVotes().get(lastCommitTime));

    // verify the consensus was updated.
    assertEquals(lastCommitTime, client.lastConsensusValue);

    // wait for quorum meet.
    final long token1 = quorum.awaitQuorum();

    // verify service was joined.
    assertTrue(client.isJoinedMember(quorum.token()));
    assertEquals(new UUID[] {serviceId}, quorum.getJoined());

    // validate the token was assigned.
    fixture.awaitDeque();
    assertEquals(Quorum.NO_QUORUM + 1, quorum.lastValidToken());
    assertEquals(Quorum.NO_QUORUM + 1, quorum.token());
    assertTrue(quorum.isQuorumMet());

    /*
     * Terminate the quorum.  The state should be cleared down.
     */

    // Verify termination of the quorum for that client.
    assertEquals(client, quorum.getClient());

    // terminate the quorum.
    quorum.terminate();

    try {
      quorum.getClient();
    } catch (IllegalStateException ex) {
      log.info("Ignoring expected exception: " + ex);
    }

    // State was cleared.
    assertEquals(Quorum.NO_QUORUM, quorum.token());
    assertEquals(Quorum.NO_QUORUM, quorum.lastValidToken());
    assertEquals(new UUID[] {}, quorum.getMembers());
    assertEquals(new UUID[] {}, quorum.getJoined());
    assertEquals(new UUID[] {}, quorum.getPipeline());
    assertEquals(Collections.emptyMap(), quorum.getVotes());

    try {
      // Note: Quorum reference was cleared. Client throws exception.
      assertFalse(client.isMember());
    } catch (IllegalStateException ex) {
      log.info("Ignoring expected exception: " + ex);
    }

    // re-terminate() is safe.
    quorum.terminate();
  }
  /**
   * Unit test for the protocol up to a service join, which triggers a leader election. Since the
   * singleton quorum has only one member our client will be elected the leader.
   *
   * @throws InterruptedException
   */
  public void test_serviceJoin() throws InterruptedException {

    final AbstractQuorum<?, ?> quorum = quorums[0];
    final MockQuorumMember<?> client = clients[0];
    final QuorumActor<?, ?> actor = actors[0];
    final UUID serviceId = client.getServiceId();

    final long lastCommitTime = 0L;
    final long lastCommitTime2 = 2L;

    // declare the service as a quorum member.
    actor.memberAdd();
    fixture.awaitDeque();

    assertTrue(client.isMember());
    assertEquals(new UUID[] {serviceId}, quorum.getMembers());

    // add to the pipeline.
    actor.pipelineAdd();
    fixture.awaitDeque();

    assertTrue(client.isPipelineMember());
    assertEquals(new UUID[] {serviceId}, quorum.getPipeline());

    // cast a vote for a lastCommitTime.
    actor.castVote(lastCommitTime);
    fixture.awaitDeque();

    assertEquals(1, quorum.getVotes().size());
    assertEquals(new UUID[] {serviceId}, quorum.getVotes().get(lastCommitTime));

    // verify the consensus was updated.
    assertEquals(lastCommitTime, client.lastConsensusValue);

    // wait for quorum meet.
    final long token1 = quorum.awaitQuorum();

    // verify service was joined.
    assertTrue(client.isJoinedMember(quorum.token()));
    assertEquals(new UUID[] {serviceId}, quorum.getJoined());

    // validate the token was assigned.
    fixture.awaitDeque();
    assertEquals(Quorum.NO_QUORUM + 1, quorum.lastValidToken());
    assertEquals(Quorum.NO_QUORUM + 1, quorum.token());
    assertTrue(quorum.isQuorumMet());

    /*
     * Do service leave, quorum should break.
     */

    actor.serviceLeave();
    fixture.awaitDeque();

    quorum.awaitBreak();

    // vote was withdrawn.
    assertEquals(0, quorum.getVotes().size());
    assertEquals(null, quorum.getVotes().get(lastCommitTime));

    // verify the consensus was updated.
    assertEquals(-1L, client.lastConsensusValue);

    assertFalse(quorum.isQuorumMet());
    assertEquals(Quorum.NO_QUORUM, quorum.token());
    assertEquals(token1, quorum.lastValidToken());
    assertFalse(client.isJoinedMember(quorum.token()));
    assertEquals(new UUID[] {}, quorum.getJoined());
    assertFalse(client.isPipelineMember());
    assertEquals(new UUID[] {}, quorum.getPipeline());

    /*
     * Cast another vote, the quorum should meet again.
     */

    actor.pipelineAdd();
    fixture.awaitDeque();

    actor.castVote(lastCommitTime2);
    fixture.awaitDeque();

    assertEquals(1, quorum.getVotes().size());
    assertEquals(null, quorum.getVotes().get(lastCommitTime));
    assertEquals(new UUID[] {serviceId}, quorum.getVotes().get(lastCommitTime2));

    // verify the consensus was updated.
    assertEquals(lastCommitTime2, client.lastConsensusValue);

    // await meet.
    final long token2 = quorum.awaitQuorum();

    // verify the joined services.
    assertEquals(new UUID[] {serviceId}, quorum.getJoined());

    // validate the token was assigned by the leader.
    assertTrue(quorum.isQuorumMet());
    assertEquals(token1 + 1, token2);
    assertEquals(token1 + 1, quorum.lastValidToken());
    assertTrue(client.isJoinedMember(token2));
    assertTrue(client.isLeader(token2));
    assertFalse(client.isFollower(token2));

    /*
     * Do service leave, quorum should break again.
     */

    actor.serviceLeave();
    fixture.awaitDeque();

    quorum.awaitBreak();

    // vote was withdrawn.
    assertEquals(0, quorum.getVotes().size());
    assertEquals(null, quorum.getVotes().get(lastCommitTime));
    assertEquals(null, quorum.getVotes().get(lastCommitTime2));

    // verify the consensus was updated.
    assertEquals(-1L, client.lastConsensusValue);

    assertFalse(quorum.isQuorumMet());
    assertEquals(Quorum.NO_QUORUM, quorum.token());
    assertEquals(token2, quorum.lastValidToken());
    assertFalse(client.isJoinedMember(quorum.token()));
    assertEquals(new UUID[] {}, quorum.getJoined());
    assertFalse(client.isPipelineMember());
    assertEquals(new UUID[] {}, quorum.getPipeline());
  }
  /**
   * Unit test for the voting protocol for a singleton quorum.
   *
   * <p>FIXME For some reason this unit test occasionally takes much longer to run than would
   * otherwise be expected (up to a few seconds versus a small fraction of a second). You can see
   * this in the timestamps of the logger.
   *
   * <p>The test terminates when the fixture tears down the AbstractQuorum's internal watcher action
   * service, which has a hung action. If the WatcherActionService is also single-threaded, then
   * this could clearly lead to a deadlock since there would be no thread available to handle new
   * events.
   *
   * <p>It is awaiting the quorumBreak condition in AbstractQuorumActor.clearToken(). This issue may
   * be that we have two distinct signals for quorumBreak versus quorumMeet which need to be
   * combined and then the various methods modified to also test the condition variable. [I've made
   * that change.]
   *
   * <p>It seems likely that either a concurrent watcherActionService -or- a finite timeout would
   * get the unit tests to pass. However, only the former would work around a deadlock due to a
   * stuck Condition.
   *
   * <p>Look again at what Condition is getting stuck and at the stress test in {@link
   * #main(String[])} for causes. [nhang=17, nerr=0, nrun=1000]. There are several different causes,
   * each of which clearly reflects a different ordering of the events.
   *
   * <p>It maybe that we see the problem with a singleton quorum because there are no other sources
   * of events to kick the quorum into motion again once it fails to join under the initial impetus.
   *
   * <p>The problem appears to stem from withdrawing the cast vote before the quorum meets.
   */
  public void test_voting()
      throws InterruptedException, AsynchronousQuorumCloseException, TimeoutException {

    /*
     * When true, this test also exercises the awaitQuorum() and
     * awaitBreak() methods that accept a timeout, but only for the case in
     * which the condition should be true on entry. There is another unit
     * test in this class that verifies that the TimeoutException is
     * correctly thrown if the condition does not become true within the
     * timeout.
     */
    final boolean awaitMeetsAndBreaks = true;

    final Quorum<?, ?> quorum = quorums[0];
    final MockQuorumMember<?> client = clients[0];
    final QuorumActor<?, ?> actor = actors[0];
    final UUID serviceId = client.getServiceId();

    final long lastCommitTime1 = 0L;

    final long lastCommitTime2 = 2L;

    // Verify that no consensus has been achieved yet.
    assertEquals(-1L, clients[0].lastConsensusValue);

    // add as member service.
    actor.memberAdd();
    fixture.awaitDeque();

    assertTrue(clients[0].isMember());

    // join the pipeline.
    actor.pipelineAdd();
    fixture.awaitDeque();

    // Verify that timestamps must be non-negative.
    try {
      actor.castVote(-1L);
      fail("Expected " + IllegalArgumentException.class);
    } catch (IllegalArgumentException ex) {
      if (log.isInfoEnabled()) log.info("Ignoring expected exception: " + ex);
    }

    // Should not be any votes.
    assertEquals(0, quorum.getVotes().size());

    // Cast a vote.
    actor.castVote(lastCommitTime1);
    fixture.awaitDeque();

    // Should be just one vote.
    assertEquals(1, quorum.getVotes().size());

    // Verify the consensus was updated
    assertEquals(lastCommitTime1, client.lastConsensusValue);

    if (awaitMeetsAndBreaks)
      assertEquals(Quorum.NO_QUORUM + 1, quorum.awaitQuorum(100, TimeUnit.MILLISECONDS));

    if (awaitMeetsAndBreaks) {
      actor.withdrawVote();
      quorum.awaitBreak();
      /*
       * FIXME I have modified castVote() to automatically add the service
       * to the pipeline. Otherwise we must do this explicitly after a
       * quorum break. Maybe the better approach to take is to have each
       * service self-report its lastCommitTime and have Quorum#start(M)
       * automatically strive towards a service join while
       * Quorum#terminate() causes leaves and then halts processing for
       * the service.  Alternatively, each client could set its target
       * state in MEMBER, PIPELINE, JOIN.  That would allow clients to
       * manage resynchronization, which they need to do.
       */
      //            actor.pipelineAdd();
    }

    // Cast another vote.
    actor.castVote(lastCommitTime2);
    fixture.awaitDeque();

    // Should be just one vote since a service can only vote for one
    // lastCommitTime at a time.
    if (quorum.getVotes().size() != 1) {
      assertEquals(quorum.getVotes().toString(), 1, quorum.getVotes().size());
    }

    // Verify the consensus was updated again.
    assertEquals(lastCommitTime2, client.lastConsensusValue);

    if (awaitMeetsAndBreaks)
      assertEquals(Quorum.NO_QUORUM + 2, quorum.awaitQuorum(100, TimeUnit.MILLISECONDS));

    // Remove as a member.
    actor.memberRemove();
    fixture.awaitDeque();

    assertFalse(clients[0].isMember());

    // The service vote was also removed.
    assertEquals(0, quorum.getVotes().size());

    if (awaitMeetsAndBreaks) quorum.awaitBreak(100, TimeUnit.MILLISECONDS);
  }