@Test
public void requireConnectionThroughGetState() throws Exception {
SocketAddress addr = mock(SocketAddress.class);
createTransportSet(addr);
assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(true));
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(true));
transportSet
.obtainActiveTransport()
.newStream(method, new Metadata(), waitForReadyCallOptions, statsTraceCtx);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(true));
// Fail it
transports.peek().listener.transportShutdown(Status.UNAVAILABLE);
// requireConnection == true doesn't skip the back-off
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(true));
transports.poll().listener.transportTerminated();
fakeClock.forwardMillis(9);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(true));
fakeClock.forwardMillis(1);
// Only when back-off is over, do we try to connect again
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
// Let it through and fail, thus a go-away
transports.peek().listener.transportReady();
transports.peek().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
transports.poll().listener.transportTerminated();
// Request for connecting again
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(true));
// And fail it again
transports.peek().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(true));
// Shut it down
transportSet.shutdown();
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(true));
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
assertFalse(transportSet.isTerminated());
// Terminate it
transports.poll().listener.transportTerminated();
assertTrue(transportSet.isTerminated());
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(true));
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
fakeExecutor.runDueTasks(); // What tasks are scheduled is not important to this test.
}
@Test
public void expiredDeadlineCancelsStream_CallOptions() {
fakeClock.forwardTime(System.nanoTime(), TimeUnit.NANOSECONDS);
ClientCallImpl<Void, Void> call =
new ClientCallImpl<Void, Void>(
DESCRIPTOR,
MoreExecutors.directExecutor(),
CallOptions.DEFAULT.withDeadline(Deadline.after(1000, TimeUnit.MILLISECONDS)),
provider,
deadlineCancellationExecutor);
call.start(callListener, new Metadata());
fakeClock.forwardMillis(1001);
verify(stream, times(1)).cancel(statusCaptor.capture());
assertEquals(Status.Code.DEADLINE_EXCEEDED, statusCaptor.getValue().getCode());
}
@Test
public void connectIsLazy() {
SocketAddress addr = mock(SocketAddress.class);
createTransportSet(addr);
// Invocation counters
int transportsCreated = 0;
// Won't connect until requested
verify(mockTransportFactory, times(transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
// First attempt
transportSet.obtainActiveTransport().newStream(method, new Metadata());
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
// Fail this one
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
// Will always reconnect after back-off
fakeClock.forwardMillis(10);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
// Make this one proceed
transports.peek().listener.transportReady();
assertEquals(ConnectivityState.READY, transportSet.getState(false));
// Then go-away
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
// Request immediately
transportSet
.obtainActiveTransport()
.newStream(method, new Metadata(), waitForReadyCallOptions, statsTraceCtx);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
fakeExecutor.runDueTasks(); // Drain new 'real' stream creation; not important to this test.
}
@Test
public void streamCancelAbortsDeadlineTimer() {
fakeClock.forwardTime(System.nanoTime(), TimeUnit.NANOSECONDS);
ClientCallImpl<Void, Void> call =
new ClientCallImpl<Void, Void>(
DESCRIPTOR,
MoreExecutors.directExecutor(),
CallOptions.DEFAULT.withDeadline(Deadline.after(1000, TimeUnit.MILLISECONDS)),
provider,
deadlineCancellationExecutor);
call.start(callListener, new Metadata());
call.cancel("canceled", null);
// Run the deadline timer, which should have been cancelled by the previous call to cancel()
fakeClock.forwardMillis(1001);
verify(stream, times(1)).cancel(statusCaptor.capture());
assertEquals(Status.CANCELLED.getCode(), statusCaptor.getValue().getCode());
}
@Test
public void shutdownBeforeTransportCreatedWithoutPendingStream() throws Exception {
SocketAddress addr = mock(SocketAddress.class);
createTransportSet(addr);
// First transport is created immediately
ClientTransport pick = transportSet.obtainActiveTransport();
verify(mockTransportFactory).newClientTransport(addr, AUTHORITY, USER_AGENT);
assertNotNull(pick);
// Fail this one
MockClientTransportInfo transportInfo = transports.poll();
transportInfo.listener.transportShutdown(Status.UNAVAILABLE);
transportInfo.listener.transportTerminated();
// Second transport will wait for back-off
pick = transportSet.obtainActiveTransport();
assertTrue(pick instanceof DelayedClientTransport);
// Shut down TransportSet before the transport is created. Futher call to
// obtainActiveTransport() gets failing transports
transportSet.shutdown();
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
pick = transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
assertNotNull(pick);
assertTrue(pick instanceof FailingClientTransport);
// TransportSet terminated promptly.
verify(mockTransportSetCallback).onTerminated(transportSet);
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
// No more transports will be created.
fakeClock.forwardMillis(10000);
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
verifyNoMoreInteractions(mockTransportFactory);
assertEquals(0, transports.size());
}
@Test
public void shutdownBeforeTransportCreatedWithPendingStream() throws Exception {
SocketAddress addr = mock(SocketAddress.class);
createTransportSet(addr);
// First transport is created immediately
ClientTransport pick = transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory).newClientTransport(addr, AUTHORITY, USER_AGENT);
assertNotNull(pick);
// Fail this one
MockClientTransportInfo transportInfo = transports.poll();
transportInfo.listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
transportInfo.listener.transportTerminated();
// Second transport will wait for back-off
pick = transportSet.obtainActiveTransport();
assertTrue(pick instanceof DelayedClientTransport);
// Start a stream, which will be pending in the delayed transport
ClientStream pendingStream =
pick.newStream(method, headers, waitForReadyCallOptions, statsTraceCtx);
pendingStream.start(mockStreamListener);
// Shut down TransportSet before the transport is created. Further call to
// obtainActiveTransport() gets failing transports
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
transportSet.shutdown();
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
pick = transportSet.obtainActiveTransport();
assertNotNull(pick);
assertTrue(pick instanceof FailingClientTransport);
verify(mockTransportFactory).newClientTransport(addr, AUTHORITY, USER_AGENT);
// Reconnect will eventually happen, even though TransportSet has been shut down
fakeClock.forwardMillis(10);
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
verify(mockTransportFactory, times(2)).newClientTransport(addr, AUTHORITY, USER_AGENT);
// The pending stream will be started on this newly started transport after it's ready.
// The transport is shut down by TransportSet right after the stream is created.
transportInfo = transports.poll();
verify(transportInfo.transport, times(0)).shutdown();
assertEquals(0, fakeExecutor.numPendingTasks());
transportInfo.listener.transportReady();
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
verify(transportInfo.transport, times(0))
.newStream(any(MethodDescriptor.class), any(Metadata.class));
assertEquals(1, fakeExecutor.runDueTasks());
verify(transportInfo.transport)
.newStream(
same(method),
same(headers),
same(waitForReadyCallOptions),
any(StatsTraceContext.class));
verify(transportInfo.transport).shutdown();
transportInfo.listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
verify(mockTransportSetCallback, never()).onTerminated(any(TransportSet.class));
// Terminating the transport will let TransportSet to be terminated.
transportInfo.listener.transportTerminated();
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
verify(mockTransportSetCallback).onTerminated(transportSet);
// No more transports will be created.
fakeClock.forwardMillis(10000);
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
verifyNoMoreInteractions(mockTransportFactory);
assertEquals(0, transports.size());
}
@Test
public void verifyFailFastAndNonFailFastBehaviors() {
int pendingStreamsCount = 0;
int failFastPendingStreamsCount = 0;
final SocketAddress addr1 = mock(SocketAddress.class);
final SocketAddress addr2 = mock(SocketAddress.class);
createTransportSet(addr1, addr2);
final DelayedClientTransport delayedTransport =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertFalse(delayedTransport.isInBackoffPeriod());
// Create a new fail fast stream.
ClientStream ffStream =
delayedTransport.newStream(method, headers, failFastCallOptions, statsTraceCtx);
ffStream.start(mockStreamListener);
// Verify it is queued.
assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());
failFastPendingStreamsCount++;
// Create a new non fail fast stream.
delayedTransport.newStream(method, headers, waitForReadyCallOptions, statsTraceCtx);
// Verify it is queued.
assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());
// Let this 1st address fail without success.
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertFalse(delayedTransport.isInBackoffPeriod());
// Verify pending streams still in queue.
assertEquals(pendingStreamsCount, delayedTransport.getPendingStreamsCount());
// Create a new fail fast stream.
delayedTransport.newStream(method, headers, failFastCallOptions, statsTraceCtx);
// Verify it is queued.
assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());
failFastPendingStreamsCount++;
// Create a new non fail fast stream
delayedTransport.newStream(method, headers, waitForReadyCallOptions, statsTraceCtx);
// Verify it is queued.
assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());
// Let this 2nd address fail without success.
Status failureStatus = Status.UNAVAILABLE.withDescription("some unique failure");
transports.poll().listener.transportShutdown(failureStatus);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
assertTrue(delayedTransport.isInBackoffPeriod());
// Fail fast pending streams should be cleared
assertEquals(
pendingStreamsCount - failFastPendingStreamsCount,
delayedTransport.getPendingStreamsCount());
pendingStreamsCount -= failFastPendingStreamsCount;
failFastPendingStreamsCount = 0;
fakeExecutor.runDueTasks();
verify(mockStreamListener).closed(same(failureStatus), any(Metadata.class));
// Create a new fail fast stream.
delayedTransport.newStream(method, headers, failFastCallOptions, statsTraceCtx);
// Verify it is not queued.
assertEquals(pendingStreamsCount, delayedTransport.getPendingStreamsCount());
// Create a new non fail fast stream
delayedTransport.newStream(method, headers, waitForReadyCallOptions, statsTraceCtx);
// Verify it is queued.
assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());
fakeClock.forwardMillis(10);
// Now back-off is over
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertFalse(delayedTransport.isInBackoffPeriod());
// Create a new fail fast stream.
delayedTransport.newStream(method, headers, failFastCallOptions, statsTraceCtx);
// Verify it is queued.
assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());
failFastPendingStreamsCount++;
assertEquals(1, failFastPendingStreamsCount);
fakeExecutor.runDueTasks(); // Drain new 'real' stream creation; not important to this test.
}
@Test
public void twoAddressesReconnect() {
SocketAddress addr1 = mock(SocketAddress.class);
SocketAddress addr2 = mock(SocketAddress.class);
createTransportSet(addr1, addr2);
assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
// Invocation counters
int transportsAddr1 = 0;
int transportsAddr2 = 0;
int backoff1Consulted = 0;
int backoff2Consulted = 0;
int backoff3Consulted = 0;
int backoffReset = 0;
int onAllAddressesFailed = 0;
// First attempt
DelayedClientTransport delayedTransport1 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
delayedTransport1.newStream(method, new Metadata(), waitForReadyCallOptions, statsTraceCtx);
// Let this one fail without success
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertNull(delayedTransport1.getTransportSupplier());
verify(mockTransportSetCallback, times(onAllAddressesFailed)).onAllAddressesFailed();
// Second attempt will start immediately. Still no back-off policy.
DelayedClientTransport delayedTransport2 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertSame(delayedTransport1, delayedTransport2);
verify(mockBackoffPolicyProvider, times(backoffReset)).get();
verify(mockTransportFactory, times(++transportsAddr2))
.newClientTransport(addr2, AUTHORITY, USER_AGENT);
// Fail this one too
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
// All addresses have failed. Delayed transport will be in back-off interval.
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
assertNull(delayedTransport2.getTransportSupplier());
assertTrue(delayedTransport2.isInBackoffPeriod());
verify(mockTransportSetCallback, times(++onAllAddressesFailed)).onAllAddressesFailed();
// Backoff reset and first back-off interval begins
verify(mockBackoffPolicy1, times(++backoff1Consulted)).nextBackoffMillis();
verify(mockBackoffPolicyProvider, times(++backoffReset)).get();
// Third attempt is the first address, thus controlled by the first back-off interval.
DelayedClientTransport delayedTransport3 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
assertSame(delayedTransport2, delayedTransport3);
fakeClock.forwardMillis(9);
verify(mockTransportFactory, times(transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
fakeClock.forwardMillis(1);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
// Fail this one too
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertNull(delayedTransport3.getTransportSupplier());
verify(mockTransportSetCallback, times(onAllAddressesFailed)).onAllAddressesFailed();
// Forth attempt will start immediately. Keep back-off policy.
DelayedClientTransport delayedTransport4 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertSame(delayedTransport3, delayedTransport4);
verify(mockBackoffPolicyProvider, times(backoffReset)).get();
verify(mockTransportFactory, times(++transportsAddr2))
.newClientTransport(addr2, AUTHORITY, USER_AGENT);
// Fail this one too
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
// All addresses have failed again. Delayed transport will be in back-off interval.
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
assertNull(delayedTransport4.getTransportSupplier());
assertTrue(delayedTransport4.isInBackoffPeriod());
verify(mockTransportSetCallback, times(++onAllAddressesFailed)).onAllAddressesFailed();
// Second back-off interval begins
verify(mockBackoffPolicy1, times(++backoff1Consulted)).nextBackoffMillis();
verify(mockBackoffPolicyProvider, times(backoffReset)).get();
// Fifth attempt for the first address, thus controlled by the second back-off interval.
DelayedClientTransport delayedTransport5 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
assertSame(delayedTransport4, delayedTransport5);
fakeClock.forwardMillis(99);
verify(mockTransportFactory, times(transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
fakeClock.forwardMillis(1);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
// Let it through
transports.peek().listener.transportReady();
assertEquals(ConnectivityState.READY, transportSet.getState(false));
// Delayed transport will see the connected transport.
assertSame(transports.peek().transport, delayedTransport5.getTransportSupplier().get());
verify(mockTransportSetCallback, never()).onConnectionClosedByServer(any(Status.class));
// Then close it.
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
verify(mockTransportSetCallback).onConnectionClosedByServer(same(Status.UNAVAILABLE));
verify(mockTransportSetCallback, times(onAllAddressesFailed)).onAllAddressesFailed();
// First attempt after a successful connection. Old back-off policy should be ignored, but there
// is not yet a need for a new one. Start from the first address.
DelayedClientTransport delayedTransport6 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertNotSame(delayedTransport5, delayedTransport6);
delayedTransport6.newStream(method, headers, waitForReadyCallOptions, statsTraceCtx);
verify(mockBackoffPolicyProvider, times(backoffReset)).get();
verify(mockTransportFactory, times(++transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
// Fail the transport
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertNull(delayedTransport6.getTransportSupplier());
verify(mockTransportSetCallback, times(onAllAddressesFailed)).onAllAddressesFailed();
// Second attempt will start immediately. Still no new back-off policy.
DelayedClientTransport delayedTransport7 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertSame(delayedTransport6, delayedTransport7);
verify(mockBackoffPolicyProvider, times(backoffReset)).get();
verify(mockTransportFactory, times(++transportsAddr2))
.newClientTransport(addr2, AUTHORITY, USER_AGENT);
// Fail this one too
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
// All addresses have failed. Delayed transport will be in back-off interval.
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
assertNull(delayedTransport7.getTransportSupplier());
assertTrue(delayedTransport7.isInBackoffPeriod());
verify(mockTransportSetCallback, times(++onAllAddressesFailed)).onAllAddressesFailed();
// Back-off reset and first back-off interval begins
verify(mockBackoffPolicy2, times(++backoff2Consulted)).nextBackoffMillis();
verify(mockBackoffPolicyProvider, times(++backoffReset)).get();
// Third attempt is the first address, thus controlled by the first back-off interval.
DelayedClientTransport delayedTransport8 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertSame(delayedTransport7, delayedTransport8);
fakeClock.forwardMillis(9);
verify(mockTransportFactory, times(transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
fakeClock.forwardMillis(1);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
// Final checks on invocations on back-off policies
verify(mockBackoffPolicy1, times(backoff1Consulted)).nextBackoffMillis();
verify(mockBackoffPolicy2, times(backoff2Consulted)).nextBackoffMillis();
verify(mockBackoffPolicy3, times(backoff3Consulted)).nextBackoffMillis();
verify(mockTransportSetCallback, atLeast(0)).onInUse(transportSet);
verify(mockTransportSetCallback, atLeast(0)).onNotInUse(transportSet);
verifyNoMoreInteractions(mockTransportSetCallback);
fakeExecutor.runDueTasks(); // Drain new 'real' stream creation; not important to this test.
}
@Test
public void singleAddressReconnect() {
SocketAddress addr = mock(SocketAddress.class);
createTransportSet(addr);
assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
// Invocation counters
int transportsCreated = 0;
int backoff1Consulted = 0;
int backoff2Consulted = 0;
int backoffReset = 0;
int onAllAddressesFailed = 0;
// First attempt
transportSet
.obtainActiveTransport()
.newStream(method, new Metadata(), waitForReadyCallOptions, statsTraceCtx);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
// Fail this one
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
verify(mockTransportSetCallback, times(++onAllAddressesFailed)).onAllAddressesFailed();
// Backoff reset and using first back-off value interval
verify(mockBackoffPolicy1, times(++backoff1Consulted)).nextBackoffMillis();
verify(mockBackoffPolicyProvider, times(++backoffReset)).get();
// Second attempt
// Transport creation doesn't happen until time is due
fakeClock.forwardMillis(9);
verify(mockTransportFactory, times(transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
fakeClock.forwardMillis(1);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
// Fail this one too
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
verify(mockTransportSetCallback, times(++onAllAddressesFailed)).onAllAddressesFailed();
// Second back-off interval
verify(mockBackoffPolicy1, times(++backoff1Consulted)).nextBackoffMillis();
verify(mockBackoffPolicyProvider, times(backoffReset)).get();
// Third attempt
// Transport creation doesn't happen until time is due
fakeClock.forwardMillis(99);
verify(mockTransportFactory, times(transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
fakeClock.forwardMillis(1);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
// Let this one succeed
transports.peek().listener.transportReady();
assertEquals(ConnectivityState.READY, transportSet.getState(false));
fakeClock.runDueTasks();
verify(mockTransportSetCallback, never()).onConnectionClosedByServer(any(Status.class));
// And close it
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
verify(mockTransportSetCallback).onConnectionClosedByServer(same(Status.UNAVAILABLE));
verify(mockTransportSetCallback, times(onAllAddressesFailed)).onAllAddressesFailed();
// Back-off is reset, and the next attempt will happen immediately
transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockBackoffPolicyProvider, times(backoffReset)).get();
verify(mockTransportFactory, times(++transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
// Final checks for consultations on back-off policies
verify(mockBackoffPolicy1, times(backoff1Consulted)).nextBackoffMillis();
verify(mockBackoffPolicy2, times(backoff2Consulted)).nextBackoffMillis();
verify(mockTransportSetCallback, atLeast(0)).onInUse(transportSet);
verify(mockTransportSetCallback, atLeast(0)).onNotInUse(transportSet);
verifyNoMoreInteractions(mockTransportSetCallback);
fakeExecutor.runDueTasks(); // Drain new 'real' stream creation; not important to this test.
}