/**
* Initiates an orderly shutdown in which preexisting calls continue but new calls are immediately
* cancelled.
*/
@Override
public ManagedChannelImpl shutdown() {
ArrayList<TransportSet> transportsCopy = new ArrayList<TransportSet>();
synchronized (lock) {
if (shutdown) {
return this;
}
shutdown = true;
// After shutdown there are no new calls, so no new cancellation tasks are needed
scheduledExecutor = SharedResourceHolder.release(TIMER_SERVICE, scheduledExecutor);
if (transports.isEmpty()) {
terminated = true;
lock.notifyAll();
onChannelTerminated();
} else {
transportsCopy.addAll(transports.values());
}
}
loadBalancer.shutdown();
nameResolver.shutdown();
for (TransportSet ts : transportsCopy) {
ts.shutdown();
}
return this;
}
@Test
public void obtainTransportAfterShutdown() throws Exception {
SocketAddress addr = mock(SocketAddress.class);
createTransportSet(addr);
transportSet.shutdown();
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
ClientTransport pick = transportSet.obtainActiveTransport();
assertNotNull(pick);
verify(mockTransportFactory, times(0)).newClientTransport(addr, AUTHORITY, USER_AGENT);
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
}
@Test
public void shutdownBeforeTransportReady() throws Exception {
SocketAddress addr = mock(SocketAddress.class);
createTransportSet(addr);
ClientTransport pick = transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
MockClientTransportInfo transportInfo = transports.poll();
assertNotSame(transportInfo.transport, pick);
// Shutdown the TransportSet before the pending transport is ready
transportSet.shutdown();
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
// The transport should've been shut down even though it's not the active transport yet.
verify(transportInfo.transport).shutdown();
transportInfo.listener.transportShutdown(Status.UNAVAILABLE);
transportInfo.listener.transportTerminated();
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
}
@Override
public ListenableFuture<ClientTransport> getTransport(
final EquivalentAddressGroup addressGroup) {
Preconditions.checkNotNull(addressGroup, "addressGroup");
TransportSet ts;
synchronized (lock) {
if (shutdown) {
return NULL_VALUE_TRANSPORT_FUTURE;
}
ts = transports.get(addressGroup);
if (ts == null) {
ts =
new TransportSet(
addressGroup,
authority(),
loadBalancer,
backoffPolicyProvider,
transportFactory,
scheduledExecutor,
new TransportSet.Callback() {
@Override
public void onTerminated() {
synchronized (lock) {
transports.remove(addressGroup);
if (shutdown && transports.isEmpty()) {
if (terminated) {
log.warning("transportTerminated called after already terminated");
}
terminated = true;
lock.notifyAll();
onChannelTerminated();
}
}
}
});
transports.put(addressGroup, ts);
}
}
return ts.obtainActiveTransport();
}
@Test
public void connectIsLazy() {
SocketAddress addr = mock(SocketAddress.class);
createTransportSet(addr);
// Invocation counters
int transportsCreated = 0;
// Won't connect until requested
verify(mockTransportFactory, times(transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
// First attempt
transportSet.obtainActiveTransport().newStream(method, new Metadata());
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
// Fail this one
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
// Will always reconnect after back-off
fakeClock.forwardMillis(10);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
// Make this one proceed
transports.peek().listener.transportReady();
assertEquals(ConnectivityState.READY, transportSet.getState(false));
// Then go-away
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
// Request immediately
transportSet
.obtainActiveTransport()
.newStream(method, new Metadata(), waitForReadyCallOptions, statsTraceCtx);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
fakeExecutor.runDueTasks(); // Drain new 'real' stream creation; not important to this test.
}
@Test
public void scheduleBackoff_DoNotScheduleEndOfBackoffIfAlreadyShutdown() {
// Setup
final boolean[] startBackoffAndShutdownAreCalled = {false};
Executor executor =
new Executor() {
@Override
public void execute(Runnable command) {
if (command.getClass().getName().contains("FailTheFailFastPendingStreams")) {
// shutdown during startBackoff
transportSet.shutdown();
startBackoffAndShutdownAreCalled[0] = true;
}
fakeExecutor.getScheduledExecutorService().execute(command);
}
};
SocketAddress addr = mock(SocketAddress.class);
addressGroup = new EquivalentAddressGroup(Arrays.asList(addr));
transportSet =
new TransportSet(
addressGroup,
AUTHORITY,
USER_AGENT,
mockLoadBalancer,
mockBackoffPolicyProvider,
mockTransportFactory,
fakeClock.getScheduledExecutorService(),
fakeClock.getStopwatchSupplier(),
executor,
mockTransportSetCallback);
// Attempt and fail, scheduleBackoff should be triggered,
// and transportSet.shutdown should be triggered by setup
transportSet
.obtainActiveTransport()
.newStream(method, new Metadata(), waitForReadyCallOptions, statsTraceCtx);
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
verify(mockTransportSetCallback, times(1)).onAllAddressesFailed();
assertTrue(startBackoffAndShutdownAreCalled[0]);
fakeExecutor.runDueTasks();
// verify endOfBackoff not scheduled
verify(mockBackoffPolicy1, never()).nextBackoffMillis();
}
@Test
public void shutdownBeforeTransportCreatedWithoutPendingStream() throws Exception {
SocketAddress addr = mock(SocketAddress.class);
createTransportSet(addr);
// First transport is created immediately
ClientTransport pick = transportSet.obtainActiveTransport();
verify(mockTransportFactory).newClientTransport(addr, AUTHORITY, USER_AGENT);
assertNotNull(pick);
// Fail this one
MockClientTransportInfo transportInfo = transports.poll();
transportInfo.listener.transportShutdown(Status.UNAVAILABLE);
transportInfo.listener.transportTerminated();
// Second transport will wait for back-off
pick = transportSet.obtainActiveTransport();
assertTrue(pick instanceof DelayedClientTransport);
// Shut down TransportSet before the transport is created. Futher call to
// obtainActiveTransport() gets failing transports
transportSet.shutdown();
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
pick = transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
assertNotNull(pick);
assertTrue(pick instanceof FailingClientTransport);
// TransportSet terminated promptly.
verify(mockTransportSetCallback).onTerminated(transportSet);
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
// No more transports will be created.
fakeClock.forwardMillis(10000);
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
verifyNoMoreInteractions(mockTransportFactory);
assertEquals(0, transports.size());
}
@Test
public void inUseState() {
SocketAddress addr = mock(SocketAddress.class);
createTransportSet(addr);
// Invocation counters
int inUse = 0;
int notInUse = 0;
verify(mockTransportSetCallback, never()).onInUse(any(TransportSet.class));
transportSet
.obtainActiveTransport()
.newStream(method, new Metadata(), waitForReadyCallOptions, statsTraceCtx);
verify(mockTransportSetCallback, times(++inUse)).onInUse(transportSet);
MockClientTransportInfo t0 = transports.poll();
verify(mockTransportSetCallback, never()).onNotInUse(any(TransportSet.class));
t0.listener.transportReady();
// The race between delayed transport being terminated (thus not in-use) and
// the real transport become in-use, caused a brief period of TransportSet not in-use.
verify(mockTransportSetCallback, times(++notInUse)).onNotInUse(transportSet);
// Delayed transport calls newStream() on the real transport in the executor
fakeExecutor.runDueTasks();
verify(t0.transport)
.newStream(
same(method),
any(Metadata.class),
same(waitForReadyCallOptions),
any(StatsTraceContext.class));
verify(mockTransportSetCallback, times(inUse)).onInUse(transportSet);
t0.listener.transportInUse(true);
verify(mockTransportSetCallback, times(++inUse)).onInUse(transportSet);
t0.listener.transportInUse(false);
verify(mockTransportSetCallback, times(++notInUse)).onNotInUse(transportSet);
t0.listener.transportInUse(true);
verify(mockTransportSetCallback, times(++inUse)).onInUse(transportSet);
// Simulate that the server sends a go-away
t0.listener.transportShutdown(Status.UNAVAILABLE);
// Creates a new transport
transportSet
.obtainActiveTransport()
.newStream(method, new Metadata(), waitForReadyCallOptions, statsTraceCtx);
MockClientTransportInfo t1 = transports.poll();
t1.listener.transportReady();
// Delayed transport calls newStream() on the real transport in the executor
fakeExecutor.runDueTasks();
verify(t1.transport)
.newStream(
same(method),
any(Metadata.class),
same(waitForReadyCallOptions),
any(StatsTraceContext.class));
t1.listener.transportInUse(true);
// No turbulance from the race mentioned eariler, because t0 has been in-use
verify(mockTransportSetCallback, times(inUse)).onInUse(transportSet);
verify(mockTransportSetCallback, times(notInUse)).onNotInUse(transportSet);
// TransportSet is not in-use when both transports are not in-use.
t1.listener.transportInUse(false);
verify(mockTransportSetCallback, times(notInUse)).onNotInUse(transportSet);
t0.listener.transportInUse(false);
verify(mockTransportSetCallback, times(++notInUse)).onNotInUse(transportSet);
verify(mockTransportSetCallback, times(inUse)).onInUse(transportSet);
}
@Test
public void logId() {
createTransportSet(mock(SocketAddress.class));
assertNotNull(transportSet.getLogId());
}
@Test
public void requireConnectionThroughGetState() throws Exception {
SocketAddress addr = mock(SocketAddress.class);
createTransportSet(addr);
assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(true));
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(true));
transportSet
.obtainActiveTransport()
.newStream(method, new Metadata(), waitForReadyCallOptions, statsTraceCtx);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(true));
// Fail it
transports.peek().listener.transportShutdown(Status.UNAVAILABLE);
// requireConnection == true doesn't skip the back-off
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(true));
transports.poll().listener.transportTerminated();
fakeClock.forwardMillis(9);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(true));
fakeClock.forwardMillis(1);
// Only when back-off is over, do we try to connect again
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
// Let it through and fail, thus a go-away
transports.peek().listener.transportReady();
transports.peek().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
transports.poll().listener.transportTerminated();
// Request for connecting again
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(true));
// And fail it again
transports.peek().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(true));
// Shut it down
transportSet.shutdown();
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(true));
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
assertFalse(transportSet.isTerminated());
// Terminate it
transports.poll().listener.transportTerminated();
assertTrue(transportSet.isTerminated());
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(true));
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
fakeExecutor.runDueTasks(); // What tasks are scheduled is not important to this test.
}
@Test
public void shutdownBeforeTransportCreatedWithPendingStream() throws Exception {
SocketAddress addr = mock(SocketAddress.class);
createTransportSet(addr);
// First transport is created immediately
ClientTransport pick = transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory).newClientTransport(addr, AUTHORITY, USER_AGENT);
assertNotNull(pick);
// Fail this one
MockClientTransportInfo transportInfo = transports.poll();
transportInfo.listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
transportInfo.listener.transportTerminated();
// Second transport will wait for back-off
pick = transportSet.obtainActiveTransport();
assertTrue(pick instanceof DelayedClientTransport);
// Start a stream, which will be pending in the delayed transport
ClientStream pendingStream =
pick.newStream(method, headers, waitForReadyCallOptions, statsTraceCtx);
pendingStream.start(mockStreamListener);
// Shut down TransportSet before the transport is created. Further call to
// obtainActiveTransport() gets failing transports
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
transportSet.shutdown();
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
pick = transportSet.obtainActiveTransport();
assertNotNull(pick);
assertTrue(pick instanceof FailingClientTransport);
verify(mockTransportFactory).newClientTransport(addr, AUTHORITY, USER_AGENT);
// Reconnect will eventually happen, even though TransportSet has been shut down
fakeClock.forwardMillis(10);
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
verify(mockTransportFactory, times(2)).newClientTransport(addr, AUTHORITY, USER_AGENT);
// The pending stream will be started on this newly started transport after it's ready.
// The transport is shut down by TransportSet right after the stream is created.
transportInfo = transports.poll();
verify(transportInfo.transport, times(0)).shutdown();
assertEquals(0, fakeExecutor.numPendingTasks());
transportInfo.listener.transportReady();
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
verify(transportInfo.transport, times(0))
.newStream(any(MethodDescriptor.class), any(Metadata.class));
assertEquals(1, fakeExecutor.runDueTasks());
verify(transportInfo.transport)
.newStream(
same(method),
same(headers),
same(waitForReadyCallOptions),
any(StatsTraceContext.class));
verify(transportInfo.transport).shutdown();
transportInfo.listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
verify(mockTransportSetCallback, never()).onTerminated(any(TransportSet.class));
// Terminating the transport will let TransportSet to be terminated.
transportInfo.listener.transportTerminated();
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
verify(mockTransportSetCallback).onTerminated(transportSet);
// No more transports will be created.
fakeClock.forwardMillis(10000);
assertEquals(ConnectivityState.SHUTDOWN, transportSet.getState(false));
verifyNoMoreInteractions(mockTransportFactory);
assertEquals(0, transports.size());
}
@Test
public void verifyFailFastAndNonFailFastBehaviors() {
int pendingStreamsCount = 0;
int failFastPendingStreamsCount = 0;
final SocketAddress addr1 = mock(SocketAddress.class);
final SocketAddress addr2 = mock(SocketAddress.class);
createTransportSet(addr1, addr2);
final DelayedClientTransport delayedTransport =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertFalse(delayedTransport.isInBackoffPeriod());
// Create a new fail fast stream.
ClientStream ffStream =
delayedTransport.newStream(method, headers, failFastCallOptions, statsTraceCtx);
ffStream.start(mockStreamListener);
// Verify it is queued.
assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());
failFastPendingStreamsCount++;
// Create a new non fail fast stream.
delayedTransport.newStream(method, headers, waitForReadyCallOptions, statsTraceCtx);
// Verify it is queued.
assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());
// Let this 1st address fail without success.
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertFalse(delayedTransport.isInBackoffPeriod());
// Verify pending streams still in queue.
assertEquals(pendingStreamsCount, delayedTransport.getPendingStreamsCount());
// Create a new fail fast stream.
delayedTransport.newStream(method, headers, failFastCallOptions, statsTraceCtx);
// Verify it is queued.
assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());
failFastPendingStreamsCount++;
// Create a new non fail fast stream
delayedTransport.newStream(method, headers, waitForReadyCallOptions, statsTraceCtx);
// Verify it is queued.
assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());
// Let this 2nd address fail without success.
Status failureStatus = Status.UNAVAILABLE.withDescription("some unique failure");
transports.poll().listener.transportShutdown(failureStatus);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
assertTrue(delayedTransport.isInBackoffPeriod());
// Fail fast pending streams should be cleared
assertEquals(
pendingStreamsCount - failFastPendingStreamsCount,
delayedTransport.getPendingStreamsCount());
pendingStreamsCount -= failFastPendingStreamsCount;
failFastPendingStreamsCount = 0;
fakeExecutor.runDueTasks();
verify(mockStreamListener).closed(same(failureStatus), any(Metadata.class));
// Create a new fail fast stream.
delayedTransport.newStream(method, headers, failFastCallOptions, statsTraceCtx);
// Verify it is not queued.
assertEquals(pendingStreamsCount, delayedTransport.getPendingStreamsCount());
// Create a new non fail fast stream
delayedTransport.newStream(method, headers, waitForReadyCallOptions, statsTraceCtx);
// Verify it is queued.
assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());
fakeClock.forwardMillis(10);
// Now back-off is over
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertFalse(delayedTransport.isInBackoffPeriod());
// Create a new fail fast stream.
delayedTransport.newStream(method, headers, failFastCallOptions, statsTraceCtx);
// Verify it is queued.
assertEquals(++pendingStreamsCount, delayedTransport.getPendingStreamsCount());
failFastPendingStreamsCount++;
assertEquals(1, failFastPendingStreamsCount);
fakeExecutor.runDueTasks(); // Drain new 'real' stream creation; not important to this test.
}
@Test
public void twoAddressesReconnect() {
SocketAddress addr1 = mock(SocketAddress.class);
SocketAddress addr2 = mock(SocketAddress.class);
createTransportSet(addr1, addr2);
assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
// Invocation counters
int transportsAddr1 = 0;
int transportsAddr2 = 0;
int backoff1Consulted = 0;
int backoff2Consulted = 0;
int backoff3Consulted = 0;
int backoffReset = 0;
int onAllAddressesFailed = 0;
// First attempt
DelayedClientTransport delayedTransport1 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
delayedTransport1.newStream(method, new Metadata(), waitForReadyCallOptions, statsTraceCtx);
// Let this one fail without success
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertNull(delayedTransport1.getTransportSupplier());
verify(mockTransportSetCallback, times(onAllAddressesFailed)).onAllAddressesFailed();
// Second attempt will start immediately. Still no back-off policy.
DelayedClientTransport delayedTransport2 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertSame(delayedTransport1, delayedTransport2);
verify(mockBackoffPolicyProvider, times(backoffReset)).get();
verify(mockTransportFactory, times(++transportsAddr2))
.newClientTransport(addr2, AUTHORITY, USER_AGENT);
// Fail this one too
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
// All addresses have failed. Delayed transport will be in back-off interval.
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
assertNull(delayedTransport2.getTransportSupplier());
assertTrue(delayedTransport2.isInBackoffPeriod());
verify(mockTransportSetCallback, times(++onAllAddressesFailed)).onAllAddressesFailed();
// Backoff reset and first back-off interval begins
verify(mockBackoffPolicy1, times(++backoff1Consulted)).nextBackoffMillis();
verify(mockBackoffPolicyProvider, times(++backoffReset)).get();
// Third attempt is the first address, thus controlled by the first back-off interval.
DelayedClientTransport delayedTransport3 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
assertSame(delayedTransport2, delayedTransport3);
fakeClock.forwardMillis(9);
verify(mockTransportFactory, times(transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
fakeClock.forwardMillis(1);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
// Fail this one too
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertNull(delayedTransport3.getTransportSupplier());
verify(mockTransportSetCallback, times(onAllAddressesFailed)).onAllAddressesFailed();
// Forth attempt will start immediately. Keep back-off policy.
DelayedClientTransport delayedTransport4 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertSame(delayedTransport3, delayedTransport4);
verify(mockBackoffPolicyProvider, times(backoffReset)).get();
verify(mockTransportFactory, times(++transportsAddr2))
.newClientTransport(addr2, AUTHORITY, USER_AGENT);
// Fail this one too
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
// All addresses have failed again. Delayed transport will be in back-off interval.
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
assertNull(delayedTransport4.getTransportSupplier());
assertTrue(delayedTransport4.isInBackoffPeriod());
verify(mockTransportSetCallback, times(++onAllAddressesFailed)).onAllAddressesFailed();
// Second back-off interval begins
verify(mockBackoffPolicy1, times(++backoff1Consulted)).nextBackoffMillis();
verify(mockBackoffPolicyProvider, times(backoffReset)).get();
// Fifth attempt for the first address, thus controlled by the second back-off interval.
DelayedClientTransport delayedTransport5 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
assertSame(delayedTransport4, delayedTransport5);
fakeClock.forwardMillis(99);
verify(mockTransportFactory, times(transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
fakeClock.forwardMillis(1);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
// Let it through
transports.peek().listener.transportReady();
assertEquals(ConnectivityState.READY, transportSet.getState(false));
// Delayed transport will see the connected transport.
assertSame(transports.peek().transport, delayedTransport5.getTransportSupplier().get());
verify(mockTransportSetCallback, never()).onConnectionClosedByServer(any(Status.class));
// Then close it.
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
verify(mockTransportSetCallback).onConnectionClosedByServer(same(Status.UNAVAILABLE));
verify(mockTransportSetCallback, times(onAllAddressesFailed)).onAllAddressesFailed();
// First attempt after a successful connection. Old back-off policy should be ignored, but there
// is not yet a need for a new one. Start from the first address.
DelayedClientTransport delayedTransport6 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertNotSame(delayedTransport5, delayedTransport6);
delayedTransport6.newStream(method, headers, waitForReadyCallOptions, statsTraceCtx);
verify(mockBackoffPolicyProvider, times(backoffReset)).get();
verify(mockTransportFactory, times(++transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
// Fail the transport
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
assertNull(delayedTransport6.getTransportSupplier());
verify(mockTransportSetCallback, times(onAllAddressesFailed)).onAllAddressesFailed();
// Second attempt will start immediately. Still no new back-off policy.
DelayedClientTransport delayedTransport7 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertSame(delayedTransport6, delayedTransport7);
verify(mockBackoffPolicyProvider, times(backoffReset)).get();
verify(mockTransportFactory, times(++transportsAddr2))
.newClientTransport(addr2, AUTHORITY, USER_AGENT);
// Fail this one too
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
// All addresses have failed. Delayed transport will be in back-off interval.
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
assertNull(delayedTransport7.getTransportSupplier());
assertTrue(delayedTransport7.isInBackoffPeriod());
verify(mockTransportSetCallback, times(++onAllAddressesFailed)).onAllAddressesFailed();
// Back-off reset and first back-off interval begins
verify(mockBackoffPolicy2, times(++backoff2Consulted)).nextBackoffMillis();
verify(mockBackoffPolicyProvider, times(++backoffReset)).get();
// Third attempt is the first address, thus controlled by the first back-off interval.
DelayedClientTransport delayedTransport8 =
(DelayedClientTransport) transportSet.obtainActiveTransport();
assertSame(delayedTransport7, delayedTransport8);
fakeClock.forwardMillis(9);
verify(mockTransportFactory, times(transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
fakeClock.forwardMillis(1);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsAddr1))
.newClientTransport(addr1, AUTHORITY, USER_AGENT);
// Final checks on invocations on back-off policies
verify(mockBackoffPolicy1, times(backoff1Consulted)).nextBackoffMillis();
verify(mockBackoffPolicy2, times(backoff2Consulted)).nextBackoffMillis();
verify(mockBackoffPolicy3, times(backoff3Consulted)).nextBackoffMillis();
verify(mockTransportSetCallback, atLeast(0)).onInUse(transportSet);
verify(mockTransportSetCallback, atLeast(0)).onNotInUse(transportSet);
verifyNoMoreInteractions(mockTransportSetCallback);
fakeExecutor.runDueTasks(); // Drain new 'real' stream creation; not important to this test.
}
@Test
public void singleAddressReconnect() {
SocketAddress addr = mock(SocketAddress.class);
createTransportSet(addr);
assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
// Invocation counters
int transportsCreated = 0;
int backoff1Consulted = 0;
int backoff2Consulted = 0;
int backoffReset = 0;
int onAllAddressesFailed = 0;
// First attempt
transportSet
.obtainActiveTransport()
.newStream(method, new Metadata(), waitForReadyCallOptions, statsTraceCtx);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
// Fail this one
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
verify(mockTransportSetCallback, times(++onAllAddressesFailed)).onAllAddressesFailed();
// Backoff reset and using first back-off value interval
verify(mockBackoffPolicy1, times(++backoff1Consulted)).nextBackoffMillis();
verify(mockBackoffPolicyProvider, times(++backoffReset)).get();
// Second attempt
// Transport creation doesn't happen until time is due
fakeClock.forwardMillis(9);
verify(mockTransportFactory, times(transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
fakeClock.forwardMillis(1);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
// Fail this one too
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
verify(mockTransportSetCallback, times(++onAllAddressesFailed)).onAllAddressesFailed();
// Second back-off interval
verify(mockBackoffPolicy1, times(++backoff1Consulted)).nextBackoffMillis();
verify(mockBackoffPolicyProvider, times(backoffReset)).get();
// Third attempt
// Transport creation doesn't happen until time is due
fakeClock.forwardMillis(99);
verify(mockTransportFactory, times(transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
assertEquals(ConnectivityState.TRANSIENT_FAILURE, transportSet.getState(false));
fakeClock.forwardMillis(1);
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockTransportFactory, times(++transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
// Let this one succeed
transports.peek().listener.transportReady();
assertEquals(ConnectivityState.READY, transportSet.getState(false));
fakeClock.runDueTasks();
verify(mockTransportSetCallback, never()).onConnectionClosedByServer(any(Status.class));
// And close it
transports.poll().listener.transportShutdown(Status.UNAVAILABLE);
assertEquals(ConnectivityState.IDLE, transportSet.getState(false));
verify(mockTransportSetCallback).onConnectionClosedByServer(same(Status.UNAVAILABLE));
verify(mockTransportSetCallback, times(onAllAddressesFailed)).onAllAddressesFailed();
// Back-off is reset, and the next attempt will happen immediately
transportSet.obtainActiveTransport();
assertEquals(ConnectivityState.CONNECTING, transportSet.getState(false));
verify(mockBackoffPolicyProvider, times(backoffReset)).get();
verify(mockTransportFactory, times(++transportsCreated))
.newClientTransport(addr, AUTHORITY, USER_AGENT);
// Final checks for consultations on back-off policies
verify(mockBackoffPolicy1, times(backoff1Consulted)).nextBackoffMillis();
verify(mockBackoffPolicy2, times(backoff2Consulted)).nextBackoffMillis();
verify(mockTransportSetCallback, atLeast(0)).onInUse(transportSet);
verify(mockTransportSetCallback, atLeast(0)).onNotInUse(transportSet);
verifyNoMoreInteractions(mockTransportSetCallback);
fakeExecutor.runDueTasks(); // Drain new 'real' stream creation; not important to this test.
}
