@Test
  public void requireConnectionThroughGetState() throws Exception {
    SocketAddress addr = mock(SocketAddress.class);
    createTransportSet(addr);

    assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(true));
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(true));

    transportSet
        .obtainActiveTransport()
        .newStream(method, new Metadata(), waitForReadyCallOptions, statsTraceCtx);
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(true));

    // Fail it
    transports.peek().listener.transportShutdown(Status.UNAVAILABLE);
    // requireConnection == true doesn't skip the back-off
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(true));
    transports.poll().listener.transportTerminated();
    fakeClock.forwardMillis(9);
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(true));
    fakeClock.forwardMillis(1);
    // Only when back-off is over, do we try to connect again
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));

    // Let it through and fail, thus a go-away
    transports.peek().listener.transportReady();
    transports.peek().listener.transportShutdown(Status.UNAVAILABLE);
    assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
    transports.poll().listener.transportTerminated();

    // Request for connecting again
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(true));
    // And fail it again
    transports.peek().listener.transportShutdown(Status.UNAVAILABLE);
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(true));

    // Shut it down
    transportSet.shutdown();
    assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(true));
    assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
    assertFalse(transportSet.isTerminated());

    // Terminate it
    transports.poll().listener.transportTerminated();
    assertTrue(transportSet.isTerminated());
    assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(true));
    assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));

    fakeExecutor.runDueTasks(); // What tasks are scheduled is not important to this test.
  }
Ejemplo n.º 2
0
  @Test
  public void expiredDeadlineCancelsStream_CallOptions() {
    fakeClock.forwardTime(System.nanoTime(), TimeUnit.NANOSECONDS);
    ClientCallImpl<Void, Void> call =
        new ClientCallImpl<Void, Void>(
            DESCRIPTOR,
            MoreExecutors.directExecutor(),
            CallOptions.DEFAULT.withDeadline(Deadline.after(1000, TimeUnit.MILLISECONDS)),
            provider,
            deadlineCancellationExecutor);

    call.start(callListener, new Metadata());

    fakeClock.forwardMillis(1001);

    verify(stream, times(1)).cancel(statusCaptor.capture());
    assertEquals(Status.Code.DEADLINE_EXCEEDED, statusCaptor.getValue().getCode());
  }
  @Test
  public void connectIsLazy() {
    SocketAddress addr = mock(SocketAddress.class);
    createTransportSet(addr);

    // Invocation counters
    int transportsCreated = 0;

    // Won't connect until requested
    verify(mockTransportFactory, times(transportsCreated))
        .newClientTransport(addr, AUTHORITY, USER_AGENT);

    // First attempt
    transportSet.obtainActiveTransport().newStream(method, new Metadata());
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    verify(mockTransportFactory, times(++transportsCreated))
        .newClientTransport(addr, AUTHORITY, USER_AGENT);

    // Fail this one
    transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));

    // Will always reconnect after back-off
    fakeClock.forwardMillis(10);
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    verify(mockTransportFactory, times(++transportsCreated))
        .newClientTransport(addr, AUTHORITY, USER_AGENT);

    // Make this one proceed
    transports.peek().listener.transportReady();
    assertEquals(ConnectivityState.READY, transportSet.getState(false));
    // Then go-away
    transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
    assertEquals(ConnectivityState.IDLE, transportSet.getState(false));

    // Request immediately
    transportSet
        .obtainActiveTransport()
        .newStream(method, new Metadata(), waitForReadyCallOptions, statsTraceCtx);
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    verify(mockTransportFactory, times(++transportsCreated))
        .newClientTransport(addr, AUTHORITY, USER_AGENT);
    fakeExecutor.runDueTasks(); // Drain new 'real' stream creation; not important to this test.
  }
Ejemplo n.º 4
0
  @Test
  public void streamCancelAbortsDeadlineTimer() {
    fakeClock.forwardTime(System.nanoTime(), TimeUnit.NANOSECONDS);

    ClientCallImpl<Void, Void> call =
        new ClientCallImpl<Void, Void>(
            DESCRIPTOR,
            MoreExecutors.directExecutor(),
            CallOptions.DEFAULT.withDeadline(Deadline.after(1000, TimeUnit.MILLISECONDS)),
            provider,
            deadlineCancellationExecutor);
    call.start(callListener, new Metadata());
    call.cancel("canceled", null);

    // Run the deadline timer, which should have been cancelled by the previous call to cancel()
    fakeClock.forwardMillis(1001);

    verify(stream, times(1)).cancel(statusCaptor.capture());

    assertEquals(Status.CANCELLED.getCode(), statusCaptor.getValue().getCode());
  }
  @Test
  public void shutdownBeforeTransportCreatedWithoutPendingStream() throws Exception {
    SocketAddress addr = mock(SocketAddress.class);
    createTransportSet(addr);

    // First transport is created immediately
    ClientTransport pick = transportSet.obtainActiveTransport();
    verify(mockTransportFactory).newClientTransport(addr, AUTHORITY, USER_AGENT);
    assertNotNull(pick);
    // Fail this one
    MockClientTransportInfo transportInfo = transports.poll();
    transportInfo.listener.transportShutdown(Status.UNAVAILABLE);
    transportInfo.listener.transportTerminated();

    // Second transport will wait for back-off
    pick = transportSet.obtainActiveTransport();
    assertTrue(pick instanceof DelayedClientTransport);

    // Shut down TransportSet before the transport is created. Futher call to
    // obtainActiveTransport() gets failing transports
    transportSet.shutdown();
    assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
    pick = transportSet.obtainActiveTransport();
    assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
    assertNotNull(pick);
    assertTrue(pick instanceof FailingClientTransport);

    // TransportSet terminated promptly.
    verify(mockTransportSetCallback).onTerminated(transportSet);
    assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));

    // No more transports will be created.
    fakeClock.forwardMillis(10000);
    assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
    verifyNoMoreInteractions(mockTransportFactory);
    assertEquals(0, transports.size());
  }
  @Test
  public void shutdownBeforeTransportCreatedWithPendingStream() throws Exception {
    SocketAddress addr = mock(SocketAddress.class);
    createTransportSet(addr);

    // First transport is created immediately
    ClientTransport pick = transportSet.obtainActiveTransport();
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    verify(mockTransportFactory).newClientTransport(addr, AUTHORITY, USER_AGENT);
    assertNotNull(pick);
    // Fail this one
    MockClientTransportInfo transportInfo = transports.poll();
    transportInfo.listener.transportShutdown(Status.UNAVAILABLE);
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
    transportInfo.listener.transportTerminated();

    // Second transport will wait for back-off
    pick = transportSet.obtainActiveTransport();
    assertTrue(pick instanceof DelayedClientTransport);
    // Start a stream, which will be pending in the delayed transport
    ClientStream pendingStream =
        pick.newStream(method, headers, waitForReadyCallOptions, statsTraceCtx);
    pendingStream.start(mockStreamListener);

    // Shut down TransportSet before the transport is created. Further call to
    // obtainActiveTransport() gets failing transports
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
    transportSet.shutdown();
    assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
    pick = transportSet.obtainActiveTransport();
    assertNotNull(pick);
    assertTrue(pick instanceof FailingClientTransport);
    verify(mockTransportFactory).newClientTransport(addr, AUTHORITY, USER_AGENT);

    // Reconnect will eventually happen, even though TransportSet has been shut down
    fakeClock.forwardMillis(10);
    assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
    verify(mockTransportFactory, times(2)).newClientTransport(addr, AUTHORITY, USER_AGENT);
    // The pending stream will be started on this newly started transport after it's ready.
    // The transport is shut down by TransportSet right after the stream is created.
    transportInfo = transports.poll();
    verify(transportInfo.transport, times(0)).shutdown();
    assertEquals(0, fakeExecutor.numPendingTasks());
    transportInfo.listener.transportReady();
    assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
    verify(transportInfo.transport, times(0))
        .newStream(any(MethodDescriptor.class), any(Metadata.class));
    assertEquals(1, fakeExecutor.runDueTasks());
    verify(transportInfo.transport)
        .newStream(
            same(method),
            same(headers),
            same(waitForReadyCallOptions),
            any(StatsTraceContext.class));
    verify(transportInfo.transport).shutdown();
    transportInfo.listener.transportShutdown(Status.UNAVAILABLE);
    assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
    verify(mockTransportSetCallback, never()).onTerminated(any(TransportSet.class));
    // Terminating the transport will let TransportSet to be terminated.
    transportInfo.listener.transportTerminated();
    assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
    verify(mockTransportSetCallback).onTerminated(transportSet);

    // No more transports will be created.
    fakeClock.forwardMillis(10000);
    assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
    verifyNoMoreInteractions(mockTransportFactory);
    assertEquals(0, transports.size());
  }
  @Test
  public void verifyFailFastAndNonFailFastBehaviors() {
    int pendingStreamsCount = 0;
    int failFastPendingStreamsCount = 0;

    final SocketAddress addr1 = mock(SocketAddress.class);
    final SocketAddress addr2 = mock(SocketAddress.class);
    createTransportSet(addr1, addr2);

    final DelayedClientTransport delayedTransport =
        (DelayedClientTransport) transportSet.obtainActiveTransport();
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    assertFalse(delayedTransport.isInBackoffPeriod());

    // Create a new fail fast stream.
    ClientStream ffStream =
        delayedTransport.newStream(method, headers, failFastCallOptions, statsTraceCtx);
    ffStream.start(mockStreamListener);
    // Verify it is queued.
    assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());
    failFastPendingStreamsCount++;
    // Create a new non fail fast stream.
    delayedTransport.newStream(method, headers, waitForReadyCallOptions, statsTraceCtx);
    // Verify it is queued.
    assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());

    // Let this 1st address fail without success.
    transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    assertFalse(delayedTransport.isInBackoffPeriod());
    // Verify pending streams still in queue.
    assertEquals(pendingStreamsCount, delayedTransport.getPendingStreamsCount());

    // Create a new fail fast stream.
    delayedTransport.newStream(method, headers, failFastCallOptions, statsTraceCtx);
    // Verify it is queued.
    assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());
    failFastPendingStreamsCount++;
    // Create a new non fail fast stream
    delayedTransport.newStream(method, headers, waitForReadyCallOptions, statsTraceCtx);
    // Verify it is queued.
    assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());

    // Let this 2nd address fail without success.
    Status failureStatus = Status.UNAVAILABLE.withDescription("some unique failure");
    transports.poll().listener.transportShutdown(failureStatus);
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
    assertTrue(delayedTransport.isInBackoffPeriod());
    // Fail fast pending streams should be cleared
    assertEquals(
        pendingStreamsCount - failFastPendingStreamsCount,
        delayedTransport.getPendingStreamsCount());
    pendingStreamsCount -= failFastPendingStreamsCount;
    failFastPendingStreamsCount = 0;
    fakeExecutor.runDueTasks();
    verify(mockStreamListener).closed(same(failureStatus), any(Metadata.class));

    // Create a new fail fast stream.
    delayedTransport.newStream(method, headers, failFastCallOptions, statsTraceCtx);
    // Verify it is not queued.
    assertEquals(pendingStreamsCount, delayedTransport.getPendingStreamsCount());
    // Create a new non fail fast stream
    delayedTransport.newStream(method, headers, waitForReadyCallOptions, statsTraceCtx);
    // Verify it is queued.
    assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());

    fakeClock.forwardMillis(10);
    // Now back-off is over
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    assertFalse(delayedTransport.isInBackoffPeriod());

    // Create a new fail fast stream.
    delayedTransport.newStream(method, headers, failFastCallOptions, statsTraceCtx);
    // Verify it is queued.
    assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());
    failFastPendingStreamsCount++;
    assertEquals(1, failFastPendingStreamsCount);

    fakeExecutor.runDueTasks(); // Drain new 'real' stream creation; not important to this test.
  }
  @Test
  public void twoAddressesReconnect() {
    SocketAddress addr1 = mock(SocketAddress.class);
    SocketAddress addr2 = mock(SocketAddress.class);
    createTransportSet(addr1, addr2);
    assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
    // Invocation counters
    int transportsAddr1 = 0;
    int transportsAddr2 = 0;
    int backoff1Consulted = 0;
    int backoff2Consulted = 0;
    int backoff3Consulted = 0;
    int backoffReset = 0;
    int onAllAddressesFailed = 0;

    // First attempt
    DelayedClientTransport delayedTransport1 =
        (DelayedClientTransport) transportSet.obtainActiveTransport();
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    verify(mockTransportFactory, times(++transportsAddr1))
        .newClientTransport(addr1, AUTHORITY, USER_AGENT);
    delayedTransport1.newStream(method, new Metadata(), waitForReadyCallOptions, statsTraceCtx);
    // Let this one fail without success
    transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    assertNull(delayedTransport1.getTransportSupplier());
    verify(mockTransportSetCallback, times(onAllAddressesFailed)).onAllAddressesFailed();

    // Second attempt will start immediately. Still no back-off policy.
    DelayedClientTransport delayedTransport2 =
        (DelayedClientTransport) transportSet.obtainActiveTransport();
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    assertSame(delayedTransport1, delayedTransport2);
    verify(mockBackoffPolicyProvider, times(backoffReset)).get();
    verify(mockTransportFactory, times(++transportsAddr2))
        .newClientTransport(addr2, AUTHORITY, USER_AGENT);
    // Fail this one too
    transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
    // All addresses have failed. Delayed transport will be in back-off interval.
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
    assertNull(delayedTransport2.getTransportSupplier());
    assertTrue(delayedTransport2.isInBackoffPeriod());
    verify(mockTransportSetCallback, times(++onAllAddressesFailed)).onAllAddressesFailed();
    // Backoff reset and first back-off interval begins
    verify(mockBackoffPolicy1, times(++backoff1Consulted)).nextBackoffMillis();
    verify(mockBackoffPolicyProvider, times(++backoffReset)).get();

    // Third attempt is the first address, thus controlled by the first back-off interval.
    DelayedClientTransport delayedTransport3 =
        (DelayedClientTransport) transportSet.obtainActiveTransport();
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
    assertSame(delayedTransport2, delayedTransport3);
    fakeClock.forwardMillis(9);
    verify(mockTransportFactory, times(transportsAddr1))
        .newClientTransport(addr1, AUTHORITY, USER_AGENT);
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
    fakeClock.forwardMillis(1);
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    verify(mockTransportFactory, times(++transportsAddr1))
        .newClientTransport(addr1, AUTHORITY, USER_AGENT);
    // Fail this one too
    transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    assertNull(delayedTransport3.getTransportSupplier());
    verify(mockTransportSetCallback, times(onAllAddressesFailed)).onAllAddressesFailed();

    // Forth attempt will start immediately. Keep back-off policy.
    DelayedClientTransport delayedTransport4 =
        (DelayedClientTransport) transportSet.obtainActiveTransport();
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    assertSame(delayedTransport3, delayedTransport4);
    verify(mockBackoffPolicyProvider, times(backoffReset)).get();
    verify(mockTransportFactory, times(++transportsAddr2))
        .newClientTransport(addr2, AUTHORITY, USER_AGENT);
    // Fail this one too
    transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
    // All addresses have failed again. Delayed transport will be in back-off interval.
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
    assertNull(delayedTransport4.getTransportSupplier());
    assertTrue(delayedTransport4.isInBackoffPeriod());
    verify(mockTransportSetCallback, times(++onAllAddressesFailed)).onAllAddressesFailed();
    // Second back-off interval begins
    verify(mockBackoffPolicy1, times(++backoff1Consulted)).nextBackoffMillis();
    verify(mockBackoffPolicyProvider, times(backoffReset)).get();

    // Fifth attempt for the first address, thus controlled by the second back-off interval.
    DelayedClientTransport delayedTransport5 =
        (DelayedClientTransport) transportSet.obtainActiveTransport();
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
    assertSame(delayedTransport4, delayedTransport5);
    fakeClock.forwardMillis(99);
    verify(mockTransportFactory, times(transportsAddr1))
        .newClientTransport(addr1, AUTHORITY, USER_AGENT);
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
    fakeClock.forwardMillis(1);
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    verify(mockTransportFactory, times(++transportsAddr1))
        .newClientTransport(addr1, AUTHORITY, USER_AGENT);
    // Let it through
    transports.peek().listener.transportReady();
    assertEquals(ConnectivityState.READY, transportSet.getState(false));
    // Delayed transport will see the connected transport.
    assertSame(transports.peek().transport, delayedTransport5.getTransportSupplier().get());
    verify(mockTransportSetCallback, never()).onConnectionClosedByServer(any(Status.class));
    // Then close it.
    transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
    assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
    verify(mockTransportSetCallback).onConnectionClosedByServer(same(Status.UNAVAILABLE));
    verify(mockTransportSetCallback, times(onAllAddressesFailed)).onAllAddressesFailed();

    // First attempt after a successful connection. Old back-off policy should be ignored, but there
    // is not yet a need for a new one. Start from the first address.
    DelayedClientTransport delayedTransport6 =
        (DelayedClientTransport) transportSet.obtainActiveTransport();
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    assertNotSame(delayedTransport5, delayedTransport6);
    delayedTransport6.newStream(method, headers, waitForReadyCallOptions, statsTraceCtx);
    verify(mockBackoffPolicyProvider, times(backoffReset)).get();
    verify(mockTransportFactory, times(++transportsAddr1))
        .newClientTransport(addr1, AUTHORITY, USER_AGENT);
    // Fail the transport
    transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    assertNull(delayedTransport6.getTransportSupplier());
    verify(mockTransportSetCallback, times(onAllAddressesFailed)).onAllAddressesFailed();

    // Second attempt will start immediately. Still no new back-off policy.
    DelayedClientTransport delayedTransport7 =
        (DelayedClientTransport) transportSet.obtainActiveTransport();
    assertSame(delayedTransport6, delayedTransport7);
    verify(mockBackoffPolicyProvider, times(backoffReset)).get();
    verify(mockTransportFactory, times(++transportsAddr2))
        .newClientTransport(addr2, AUTHORITY, USER_AGENT);
    // Fail this one too
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
    // All addresses have failed. Delayed transport will be in back-off interval.
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
    assertNull(delayedTransport7.getTransportSupplier());
    assertTrue(delayedTransport7.isInBackoffPeriod());
    verify(mockTransportSetCallback, times(++onAllAddressesFailed)).onAllAddressesFailed();
    // Back-off reset and first back-off interval begins
    verify(mockBackoffPolicy2, times(++backoff2Consulted)).nextBackoffMillis();
    verify(mockBackoffPolicyProvider, times(++backoffReset)).get();

    // Third attempt is the first address, thus controlled by the first back-off interval.
    DelayedClientTransport delayedTransport8 =
        (DelayedClientTransport) transportSet.obtainActiveTransport();
    assertSame(delayedTransport7, delayedTransport8);
    fakeClock.forwardMillis(9);
    verify(mockTransportFactory, times(transportsAddr1))
        .newClientTransport(addr1, AUTHORITY, USER_AGENT);
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
    fakeClock.forwardMillis(1);
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    verify(mockTransportFactory, times(++transportsAddr1))
        .newClientTransport(addr1, AUTHORITY, USER_AGENT);

    // Final checks on invocations on back-off policies
    verify(mockBackoffPolicy1, times(backoff1Consulted)).nextBackoffMillis();
    verify(mockBackoffPolicy2, times(backoff2Consulted)).nextBackoffMillis();
    verify(mockBackoffPolicy3, times(backoff3Consulted)).nextBackoffMillis();
    verify(mockTransportSetCallback, atLeast(0)).onInUse(transportSet);
    verify(mockTransportSetCallback, atLeast(0)).onNotInUse(transportSet);
    verifyNoMoreInteractions(mockTransportSetCallback);
    fakeExecutor.runDueTasks(); // Drain new 'real' stream creation; not important to this test.
  }
  @Test
  public void singleAddressReconnect() {
    SocketAddress addr = mock(SocketAddress.class);
    createTransportSet(addr);
    assertEquals(ConnectivityState.IDLE, transportSet.getState(false));

    // Invocation counters
    int transportsCreated = 0;
    int backoff1Consulted = 0;
    int backoff2Consulted = 0;
    int backoffReset = 0;
    int onAllAddressesFailed = 0;

    // First attempt
    transportSet
        .obtainActiveTransport()
        .newStream(method, new Metadata(), waitForReadyCallOptions, statsTraceCtx);
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    verify(mockTransportFactory, times(++transportsCreated))
        .newClientTransport(addr, AUTHORITY, USER_AGENT);

    // Fail this one
    transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
    verify(mockTransportSetCallback, times(++onAllAddressesFailed)).onAllAddressesFailed();
    // Backoff reset and using first back-off value interval
    verify(mockBackoffPolicy1, times(++backoff1Consulted)).nextBackoffMillis();
    verify(mockBackoffPolicyProvider, times(++backoffReset)).get();

    // Second attempt
    // Transport creation doesn't happen until time is due
    fakeClock.forwardMillis(9);
    verify(mockTransportFactory, times(transportsCreated))
        .newClientTransport(addr, AUTHORITY, USER_AGENT);
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
    fakeClock.forwardMillis(1);
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    verify(mockTransportFactory, times(++transportsCreated))
        .newClientTransport(addr, AUTHORITY, USER_AGENT);
    // Fail this one too
    transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
    verify(mockTransportSetCallback, times(++onAllAddressesFailed)).onAllAddressesFailed();
    // Second back-off interval
    verify(mockBackoffPolicy1, times(++backoff1Consulted)).nextBackoffMillis();
    verify(mockBackoffPolicyProvider, times(backoffReset)).get();

    // Third attempt
    // Transport creation doesn't happen until time is due
    fakeClock.forwardMillis(99);
    verify(mockTransportFactory, times(transportsCreated))
        .newClientTransport(addr, AUTHORITY, USER_AGENT);
    assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
    fakeClock.forwardMillis(1);
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    verify(mockTransportFactory, times(++transportsCreated))
        .newClientTransport(addr, AUTHORITY, USER_AGENT);
    // Let this one succeed
    transports.peek().listener.transportReady();
    assertEquals(ConnectivityState.READY, transportSet.getState(false));
    fakeClock.runDueTasks();
    verify(mockTransportSetCallback, never()).onConnectionClosedByServer(any(Status.class));
    // And close it
    transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
    assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
    verify(mockTransportSetCallback).onConnectionClosedByServer(same(Status.UNAVAILABLE));
    verify(mockTransportSetCallback, times(onAllAddressesFailed)).onAllAddressesFailed();

    // Back-off is reset, and the next attempt will happen immediately
    transportSet.obtainActiveTransport();
    assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
    verify(mockBackoffPolicyProvider, times(backoffReset)).get();
    verify(mockTransportFactory, times(++transportsCreated))
        .newClientTransport(addr, AUTHORITY, USER_AGENT);

    // Final checks for consultations on back-off policies
    verify(mockBackoffPolicy1, times(backoff1Consulted)).nextBackoffMillis();
    verify(mockBackoffPolicy2, times(backoff2Consulted)).nextBackoffMillis();
    verify(mockTransportSetCallback, atLeast(0)).onInUse(transportSet);
    verify(mockTransportSetCallback, atLeast(0)).onNotInUse(transportSet);
    verifyNoMoreInteractions(mockTransportSetCallback);
    fakeExecutor.runDueTasks(); // Drain new 'real' stream creation; not important to this test.
  }