/** This is the main point of entry into the DSO client. */ public class DistributedObjectClient implements TCClient { protected static final TCLogger DSO_LOGGER = CustomerLogging.getDSOGenericLogger(); private static final TCLogger CONSOLE_LOGGER = CustomerLogging.getConsoleLogger(); private static final int MAX_CONNECT_TRIES = -1; private static final String L1VMShutdownHookName = "L1 VM Shutdown Hook"; private final ClientBuilder clientBuilder; private final ClientConfig config; private final ClusterInternal cluster; private final TCThreadGroup threadGroup; private final ThreadIDMap threadIDMap; protected final PreparedComponentsFromL2Connection connectionComponents; private final ProductID productId; private ClientMessageChannel channel; private ClientLockManager lockManager; private CommunicationsManager communicationsManager; private ClientHandshakeManager clientHandshakeManager; private CounterManager counterManager; private ThreadIDManager threadIDManager; private final CallbackDumpHandler dumpHandler = new CallbackDumpHandler(); private TCMemoryManagerImpl tcMemManager; private Stage<ClusterInternalEventsContext> clusterEventsStage; private final TCSecurityManager securityManager; private final String uuid; private final String name; private final TaskRunner taskRunner; private ClientShutdownManager shutdownManager; private final Thread shutdownAction; private final SetOnceFlag clientStopped = new SetOnceFlag(); private final SetOnceFlag connectionMade = new SetOnceFlag(); private ClientEntityManager clientEntityManager; private final StageManager communicationStageManager; public DistributedObjectClient( ClientConfig config, TCThreadGroup threadGroup, PreparedComponentsFromL2Connection connectionComponents, ClusterInternal cluster) { this( config, threadGroup, connectionComponents, cluster, null, UUID.NULL_ID.toString(), "", null); } public DistributedObjectClient( ClientConfig config, TCThreadGroup threadGroup, PreparedComponentsFromL2Connection connectionComponents, ClusterInternal cluster, TCSecurityManager securityManager, String uuid, String name, ProductID productId) { this.productId = productId; Assert.assertNotNull(config); this.config = config; this.securityManager = securityManager; this.connectionComponents = connectionComponents; this.cluster = cluster; this.threadGroup = threadGroup; this.threadIDMap = new ThreadIDMapImpl(); this.clientBuilder = createClientBuilder(); this.uuid = uuid; this.name = name; this.taskRunner = Runners.newDefaultCachedScheduledTaskRunner(threadGroup); this.shutdownAction = new Thread(new ShutdownAction(), L1VMShutdownHookName); Runtime.getRuntime().addShutdownHook(this.shutdownAction); // We need a StageManager to create the SEDA stages used for handling the messages. final SEDA<Void> seda = new SEDA<Void>(threadGroup); communicationStageManager = seda.getStageManager(); this.tcMemManager = new TCMemoryManagerImpl(threadGroup); } protected ClientBuilder createClientBuilder() { return new StandardClientBuilder(); } @Override public ThreadIDMap getThreadIDMap() { return this.threadIDMap; } @Override public void addAllLocksTo(LockInfoByThreadID lockInfo) { if (this.lockManager != null) { for (final ClientServerExchangeLockContext c : this.lockManager.getAllLockContexts()) { switch (c.getState().getType()) { case GREEDY_HOLDER: case HOLDER: lockInfo.addLock(LockState.HOLDING, c.getThreadID(), c.getLockID().toString()); break; case WAITER: lockInfo.addLock(LockState.WAITING_ON, c.getThreadID(), c.getLockID().toString()); break; case TRY_PENDING: case PENDING: lockInfo.addLock(LockState.WAITING_TO, c.getThreadID(), c.getLockID().toString()); break; default: throw new AssertionError(c.getState().getType()); } } } else { DSO_LOGGER.error("LockManager not initialised still. LockInfo for threads cannot be updated"); } } private void validateSecurityConfig() { if (config.getSecurityInfo().isSecure() && securityManager == null) { throw new TCRuntimeException( "client configured as secure but was constructed without securityManager"); } if (!config.getSecurityInfo().isSecure() && securityManager != null) { throw new TCRuntimeException( "client not configured as secure but was constructed with securityManager"); } } private ReconnectConfig getReconnectPropertiesFromServer() { ReconnectConfig reconnectConfig = new ReconnectConfig() { @Override public boolean getReconnectEnabled() { return true; } @Override public int getReconnectTimeout() { return 5000; } @Override public int getSendQueueCapacity() { return 5000; } @Override public int getMaxDelayAcks() { return 16; } @Override public int getSendWindow() { return 32; } }; return reconnectConfig; } private NetworkStackHarnessFactory getNetworkStackHarnessFactory( boolean useOOOLayer, ReconnectConfig l1ReconnectConfig) { if (useOOOLayer) { return new OOONetworkStackHarnessFactory( new OnceAndOnlyOnceProtocolNetworkLayerFactoryImpl(), l1ReconnectConfig); } else { return new PlainNetworkStackHarnessFactory(); } } public Stage<ClusterInternalEventsContext> getClusterEventsStage() { return clusterEventsStage; } public synchronized void start() { validateSecurityConfig(); final TCProperties tcProperties = TCPropertiesImpl.getProperties(); final int maxSize = tcProperties.getInt(TCPropertiesConsts.L1_SEDA_STAGE_SINK_CAPACITY); final SessionManager sessionManager = new SessionManagerImpl( new SessionManagerImpl.SequenceFactory() { @Override public Sequence newSequence() { return new SimpleSequence(); } }); this.threadGroup.addCallbackOnExitDefaultHandler( new CallbackOnExitHandler() { @Override public void callbackOnExit(CallbackOnExitState state) { cluster.fireNodeError(); } }); this.dumpHandler.registerForDump(new CallbackDumpAdapter(this.communicationStageManager)); final ReconnectConfig l1ReconnectConfig = getReconnectPropertiesFromServer(); final boolean useOOOLayer = l1ReconnectConfig.getReconnectEnabled(); final NetworkStackHarnessFactory networkStackHarnessFactory = getNetworkStackHarnessFactory(useOOOLayer, l1ReconnectConfig); this.counterManager = new CounterManagerImpl(); final MessageMonitor mm = MessageMonitorImpl.createMonitor(tcProperties, DSO_LOGGER); final TCMessageRouter messageRouter = new TCMessageRouterImpl(); this.communicationsManager = this.clientBuilder.createCommunicationsManager( mm, messageRouter, networkStackHarnessFactory, new NullConnectionPolicy(), this.connectionComponents.createConnectionInfoConfigItemByGroup().length, new HealthCheckerConfigClientImpl( tcProperties.getPropertiesFor(TCPropertiesConsts.L1_L2_HEALTH_CHECK_CATEGORY), "DSO Client"), getMessageTypeClassMapping(), ReconnectionRejectedHandlerL1.SINGLETON, securityManager, productId); DSO_LOGGER.debug("Created CommunicationsManager."); final ConnectionInfoConfig[] connectionInfoItems = this.connectionComponents.createConnectionInfoConfigItemByGroup(); final ConnectionInfo[] connectionInfo = connectionInfoItems[0].getConnectionInfos(); final String serverHost = connectionInfo[0].getHostname(); final int serverPort = connectionInfo[0].getPort(); clusterEventsStage = this.communicationStageManager.createStage( ClientConfigurationContext.CLUSTER_EVENTS_STAGE, ClusterInternalEventsContext.class, new ClusterInternalEventsHandler<ClusterInternalEventsContext>(cluster), 1, maxSize); final int socketConnectTimeout = tcProperties.getInt(TCPropertiesConsts.L1_SOCKET_CONNECT_TIMEOUT); if (socketConnectTimeout < 0) { throw new IllegalArgumentException("invalid socket time value: " + socketConnectTimeout); } this.channel = this.clientBuilder.createClientMessageChannel( this.communicationsManager, this.connectionComponents, sessionManager, MAX_CONNECT_TRIES, socketConnectTimeout, this); final ClientIDLoggerProvider cidLoggerProvider = new ClientIDLoggerProvider(this.channel); this.communicationStageManager.setLoggerProvider(cidLoggerProvider); DSO_LOGGER.debug("Created channel."); this.clientEntityManager = this.clientBuilder.createClientEntityManager(this.channel, this.communicationStageManager); RequestReceiveHandler receivingHandler = new RequestReceiveHandler(this.clientEntityManager); Stage<VoltronEntityResponse> entityResponseStage = this.communicationStageManager.createStage( ClientConfigurationContext.VOLTRON_ENTITY_RESPONSE_STAGE, VoltronEntityResponse.class, receivingHandler, 1, maxSize); Stage<Void> serverMessageStage = this.communicationStageManager.createStage( ClientConfigurationContext.SERVER_ENTITY_MESSAGE_STAGE, Void.class, new ServerMessageReceiveHandler<Void>(channel), 1, maxSize); TerracottaOperatorEventLogging.setNodeNameProvider(new ClientNameProvider(this.cluster)); final SampledRateCounterConfig sampledRateCounterConfig = new SampledRateCounterConfig(1, 300, true); this.counterManager.createCounter(sampledRateCounterConfig); this.counterManager.createCounter(sampledRateCounterConfig); // for SRA L1 Tx count final SampledCounterConfig sampledCounterConfig = new SampledCounterConfig(1, 300, true, 0L); this.counterManager.createCounter(sampledCounterConfig); this.threadGroup.addCallbackOnExitDefaultHandler( new CallbackDumpAdapter(this.clientEntityManager)); this.dumpHandler.registerForDump(new CallbackDumpAdapter(this.clientEntityManager)); final long timeOut = TCPropertiesImpl.getProperties().getLong(TCPropertiesConsts.LOGGING_LONG_GC_THRESHOLD); final LongGCLogger gcLogger = this.clientBuilder.createLongGCLogger(timeOut); this.tcMemManager.registerForMemoryEvents(gcLogger); // CDV-1181 warn if using CMS this.tcMemManager.checkGarbageCollectors(); this.threadIDManager = new ThreadIDManagerImpl(this.threadIDMap); // Setup the lock manager this.lockManager = this.clientBuilder.createLockManager( this.channel, new ClientIDLogger(this.channel, TCLogging.getLogger(ClientLockManager.class)), sessionManager, this.channel.getLockRequestMessageFactory(), this.threadIDManager, new ClientLockManagerConfigImpl( tcProperties.getPropertiesFor(TCPropertiesConsts.L1_LOCK_MANAGER_CATEGORY)), this.taskRunner); final CallbackDumpAdapter lockDumpAdapter = new CallbackDumpAdapter(this.lockManager); this.threadGroup.addCallbackOnExitDefaultHandler(lockDumpAdapter); this.dumpHandler.registerForDump(lockDumpAdapter); // Create the SEDA stages final Stage<Void> lockResponse = this.communicationStageManager.createStage( ClientConfigurationContext.LOCK_RESPONSE_STAGE, Void.class, new LockResponseHandler<Void>(sessionManager), 1, maxSize); final Stage<HydrateContext> hydrateStage = this.communicationStageManager.createStage( ClientConfigurationContext.HYDRATE_MESSAGE_STAGE, HydrateContext.class, new HydrateHandler(), 1, maxSize); // By design this stage needs to be single threaded. If it wasn't then cluster membership // messages could get // processed before the client handshake ack, and this client would get a faulty view of the // cluster at best, or // more likely an AssertionError final Stage<PauseContext> pauseStage = this.communicationStageManager.createStage( ClientConfigurationContext.CLIENT_COORDINATION_STAGE, PauseContext.class, new ClientCoordinationHandler<PauseContext>(), 1, maxSize); final Sink<PauseContext> pauseSink = pauseStage.getSink(); final Stage<Void> clusterMembershipEventStage = this.communicationStageManager.createStage( ClientConfigurationContext.CLUSTER_MEMBERSHIP_EVENT_STAGE, Void.class, new ClusterMembershipEventsHandler<Void>(cluster), 1, maxSize); final List<ClientHandshakeCallback> clientHandshakeCallbacks = new ArrayList<ClientHandshakeCallback>(); clientHandshakeCallbacks.add(this.lockManager); clientHandshakeCallbacks.add(this.clientEntityManager); final ProductInfo pInfo = ProductInfo.getInstance(); this.clientHandshakeManager = this.clientBuilder.createClientHandshakeManager( new ClientIDLogger(this.channel, TCLogging.getLogger(ClientHandshakeManagerImpl.class)), this.channel.getClientHandshakeMessageFactory(), pauseSink, sessionManager, cluster, this.uuid, this.name, pInfo.version(), Collections.unmodifiableCollection(clientHandshakeCallbacks)); ClientChannelEventController.connectChannelEventListener( channel, pauseSink, clientHandshakeManager); this.shutdownManager = new ClientShutdownManager(this, connectionComponents); final ClientConfigurationContext cc = new ClientConfigurationContext( this.communicationStageManager, this.lockManager, this.clientEntityManager, this.clientHandshakeManager); // DO NOT create any stages after this call this.communicationStageManager.startAll(cc, Collections.<PostInit>emptyList()); initChannelMessageRouter( messageRouter, hydrateStage.getSink(), lockResponse.getSink(), pauseSink, clusterMembershipEventStage.getSink(), entityResponseStage.getSink(), serverMessageStage.getSink()); new Thread( threadGroup, new Runnable() { public void run() { while (!clientStopped.isSet()) { try { openChannel(serverHost, serverPort); waitForHandshake(); connectionMade(); break; } catch (InterruptedException ie) { // We are in the process of letting the thread terminate so we don't handle this // in a special way. } } // don't reset interrupted, thread is done } }, "Connection Establisher - " + uuid) .start(); } private synchronized void connectionMade() { connectionMade.attemptSet(); notifyAll(); } public synchronized boolean waitForConnection(long timeout, TimeUnit units) throws InterruptedException { long left = timeout > 0 ? units.toMillis(timeout) : Long.MAX_VALUE; while (!connectionMade.isSet() && left > 0) { long start = System.currentTimeMillis(); this.wait(units.toMillis(timeout)); left -= (System.currentTimeMillis() - start); } return connectionMade.isSet(); } private synchronized void openChannel(String serverHost, int serverPort) throws InterruptedException { while (!clientStopped.isSet()) { try { DSO_LOGGER.debug("Trying to open channel...."); final char[] pw; if (config.getSecurityInfo().hasCredentials()) { Assert.assertNotNull(securityManager); pw = securityManager.getPasswordForTC( config.getSecurityInfo().getUsername(), serverHost, serverPort); } else { pw = null; } this.channel.open(pw); DSO_LOGGER.debug("Channel open"); break; } catch (final TCTimeoutException tcte) { CONSOLE_LOGGER.warn("Timeout connecting to server: " + tcte.getMessage()); this.wait(5000); } catch (final ConnectException e) { CONSOLE_LOGGER.warn("Connection refused from server: " + e); this.wait(5000); } catch (final MaxConnectionsExceededException e) { DSO_LOGGER.fatal(e.getMessage()); CONSOLE_LOGGER.fatal(e.getMessage()); throw new IllegalStateException(e.getMessage(), e); } catch (final CommStackMismatchException e) { DSO_LOGGER.fatal(e.getMessage()); CONSOLE_LOGGER.fatal(e.getMessage()); throw new IllegalStateException(e.getMessage(), e); } catch (final IOException ioe) { CONSOLE_LOGGER.warn( "IOException connecting to server: " + serverHost + ":" + serverPort + ". " + ioe.getMessage()); this.wait(5000); } } } private synchronized void waitForHandshake() { this.clientHandshakeManager.waitForHandshake(); if (this.channel != null) { final TCSocketAddress remoteAddress = this.channel.getRemoteAddress(); final String infoMsg = "Connection successfully established to server at " + remoteAddress; CONSOLE_LOGGER.info(infoMsg); DSO_LOGGER.info(infoMsg); } } private Map<TCMessageType, Class<? extends TCMessage>> getMessageTypeClassMapping() { final Map<TCMessageType, Class<? extends TCMessage>> messageTypeClassMapping = new HashMap<TCMessageType, Class<? extends TCMessage>>(); messageTypeClassMapping.put(TCMessageType.LOCK_REQUEST_MESSAGE, LockRequestMessage.class); messageTypeClassMapping.put(TCMessageType.LOCK_RESPONSE_MESSAGE, LockResponseMessage.class); messageTypeClassMapping.put(TCMessageType.LOCK_RECALL_MESSAGE, LockResponseMessage.class); messageTypeClassMapping.put( TCMessageType.LOCK_QUERY_RESPONSE_MESSAGE, LockResponseMessage.class); messageTypeClassMapping.put( TCMessageType.CLIENT_HANDSHAKE_MESSAGE, ClientHandshakeMessageImpl.class); messageTypeClassMapping.put( TCMessageType.CLIENT_HANDSHAKE_ACK_MESSAGE, ClientHandshakeAckMessageImpl.class); messageTypeClassMapping.put( TCMessageType.CLIENT_HANDSHAKE_REFUSED_MESSAGE, ClientHandshakeRefusedMessageImpl.class); messageTypeClassMapping.put( TCMessageType.CLUSTER_MEMBERSHIP_EVENT_MESSAGE, ClusterMembershipMessage.class); messageTypeClassMapping.put( TCMessageType.LIST_REGISTERED_SERVICES_MESSAGE, ListRegisteredServicesMessage.class); messageTypeClassMapping.put( TCMessageType.LIST_REGISTERED_SERVICES_RESPONSE_MESSAGE, ListRegisteredServicesResponseMessage.class); messageTypeClassMapping.put( TCMessageType.INVOKE_REGISTERED_SERVICE_MESSAGE, InvokeRegisteredServiceMessage.class); messageTypeClassMapping.put( TCMessageType.INVOKE_REGISTERED_SERVICE_RESPONSE_MESSAGE, InvokeRegisteredServiceResponseMessage.class); messageTypeClassMapping.put( TCMessageType.VOLTRON_ENTITY_MESSAGE, NetworkVoltronEntityMessageImpl.class); messageTypeClassMapping.put( TCMessageType.VOLTRON_ENTITY_RECEIVED_RESPONSE, VoltronEntityReceivedResponseImpl.class); messageTypeClassMapping.put( TCMessageType.VOLTRON_ENTITY_APPLIED_RESPONSE, VoltronEntityAppliedResponseImpl.class); messageTypeClassMapping.put( TCMessageType.VOLTRON_ENTITY_RETIRED_RESPONSE, VoltronEntityRetiredResponseImpl.class); messageTypeClassMapping.put(TCMessageType.SERVER_ENTITY_MESSAGE, ServerEntityMessageImpl.class); messageTypeClassMapping.put( TCMessageType.SERVER_ENTITY_RESPONSE_MESSAGE, ServerEntityResponseMessageImpl.class); return messageTypeClassMapping; } private void initChannelMessageRouter( TCMessageRouter messageRouter, Sink<HydrateContext> hydrateSink, Sink<Void> lockResponseSink, Sink<PauseContext> pauseSink, Sink<Void> clusterMembershipEventSink, Sink<VoltronEntityResponse> responseSink, Sink<Void> serverEntityMessageSink) { messageRouter.routeMessageType( TCMessageType.LOCK_RESPONSE_MESSAGE, lockResponseSink, hydrateSink); messageRouter.routeMessageType( TCMessageType.LOCK_QUERY_RESPONSE_MESSAGE, lockResponseSink, hydrateSink); messageRouter.routeMessageType( TCMessageType.LOCK_RECALL_MESSAGE, lockResponseSink, hydrateSink); messageRouter.routeMessageType( TCMessageType.CLIENT_HANDSHAKE_ACK_MESSAGE, pauseSink, hydrateSink); messageRouter.routeMessageType( TCMessageType.CLIENT_HANDSHAKE_REFUSED_MESSAGE, pauseSink, hydrateSink); messageRouter.routeMessageType( TCMessageType.CLUSTER_MEMBERSHIP_EVENT_MESSAGE, clusterMembershipEventSink, hydrateSink); messageRouter.routeMessageType( TCMessageType.VOLTRON_ENTITY_RECEIVED_RESPONSE, responseSink, hydrateSink); messageRouter.routeMessageType( TCMessageType.VOLTRON_ENTITY_APPLIED_RESPONSE, responseSink, hydrateSink); messageRouter.routeMessageType( TCMessageType.VOLTRON_ENTITY_RETIRED_RESPONSE, responseSink, hydrateSink); messageRouter.routeMessageType( TCMessageType.SERVER_ENTITY_MESSAGE, serverEntityMessageSink, hydrateSink); DSO_LOGGER.debug("Added message routing types."); } public ClientLockManager getLockManager() { return this.lockManager; } public ClientEntityManager getEntityManager() { return this.clientEntityManager; } public CommunicationsManager getCommunicationsManager() { return this.communicationsManager; } public ClientMessageChannel getChannel() { return this.channel; } public ClientHandshakeManager getClientHandshakeManager() { return this.clientHandshakeManager; } @Override public void dump() { this.dumpHandler.dump(); } protected ClientConfig getClientConfigHelper() { return this.config; } public void shutdown() { shutdown(false, false); } void shutdownResources() { final TCLogger logger = DSO_LOGGER; if (this.counterManager != null) { try { this.counterManager.shutdown(); } catch (final Throwable t) { logger.error("error shutting down counter manager", t); } finally { this.counterManager = null; } } if (this.tcMemManager != null) { try { this.tcMemManager.shutdown(); } catch (final Throwable t) { logger.error("Error stopping memory manager", t); } finally { this.tcMemManager = null; } } if (this.lockManager != null) { try { this.lockManager.shutdown(false); } catch (final Throwable t) { logger.error("Error stopping lock manager", t); } finally { this.lockManager = null; } } try { this.communicationStageManager.stopAll(); } catch (final Throwable t) { logger.error("Error stopping stage manager", t); } if (this.channel != null) { try { this.channel.close(); } catch (final Throwable t) { logger.error("Error closing channel", t); } finally { this.channel = null; } } if (this.communicationsManager != null) { try { this.communicationsManager.shutdown(); } catch (final Throwable t) { logger.error("Error shutting down communications manager", t); } finally { this.communicationsManager = null; } } if (taskRunner != null) { logger.info("Shutting down TaskRunner"); taskRunner.shutdown(); } CommonShutDownHook.shutdown(); this.cluster.shutdown(); if (this.threadGroup != null) { boolean interrupted = false; try { final long end = System.currentTimeMillis() + TCPropertiesImpl.getProperties() .getLong(TCPropertiesConsts.L1_SHUTDOWN_THREADGROUP_GRACETIME); int threadCount = this.threadGroup.activeCount(); Thread[] t = new Thread[threadCount]; threadCount = this.threadGroup.enumerate(t); final long time = System.currentTimeMillis(); for (int x = 0; x < threadCount; x++) { long start = System.currentTimeMillis(); while (System.currentTimeMillis() < end && t[x].isAlive()) { t[x].join(1000); } logger.info( "Destroyed thread " + t[x].getName() + " time to destroy:" + (System.currentTimeMillis() - start) + " millis"); } logger.info( "time to destroy thread group:" + TimeUnit.SECONDS.convert(System.currentTimeMillis() - time, TimeUnit.MILLISECONDS) + " seconds"); if (this.threadGroup.activeCount() > 0) { logger.warn( "Timed out waiting for TC thread group threads to die - probable shutdown memory leak\n" + "Live threads: " + getLiveThreads(this.threadGroup)); Thread threadGroupCleanerThread = new Thread( this.threadGroup.getParent(), new TCThreadGroupCleanerRunnable(threadGroup), "TCThreadGroup last chance cleaner thread"); threadGroupCleanerThread.setDaemon(true); threadGroupCleanerThread.start(); logger.warn("Spawning TCThreadGroup last chance cleaner thread"); } else { logger.info("Destroying TC thread group"); this.threadGroup.destroy(); } } catch (final Throwable t) { logger.error("Error destroying TC thread group", t); } finally { if (interrupted) { Thread.currentThread().interrupt(); } } } if (TCPropertiesImpl.getProperties() .getBoolean(TCPropertiesConsts.L1_SHUTDOWN_FORCE_FINALIZATION)) System.runFinalization(); } private static List<Thread> getLiveThreads(ThreadGroup group) { final int estimate = group.activeCount(); Thread[] threads = new Thread[estimate + 1]; while (true) { final int count = group.enumerate(threads); if (count < threads.length) { final List<Thread> l = new ArrayList<Thread>(count); for (final Thread t : threads) { if (t != null) { l.add(t); } } return l; } else { threads = new Thread[threads.length * 2]; } } } @Override public String[] processArguments() { return null; } @Override public String getUUID() { return uuid; } private static class TCThreadGroupCleanerRunnable implements Runnable { private final TCThreadGroup threadGroup; public TCThreadGroupCleanerRunnable(TCThreadGroup threadGroup) { this.threadGroup = threadGroup; } @Override public void run() { while (threadGroup.activeCount() > 0) { for (Thread liveThread : getLiveThreads(threadGroup)) { liveThread.interrupt(); } try { Thread.sleep(1000); } catch (final InterruptedException e) { // ignore } } try { threadGroup.destroy(); } catch (Exception e) { // the logger is closed by now so we can't even log that } } } public Cluster getCluster() { return this.cluster; } private void shutdownClient(boolean fromShutdownHook, boolean forceImmediate) { if (this.shutdownManager != null) { try { this.shutdownManager.execute(fromShutdownHook, forceImmediate); } finally { // If we're not being called as a result of the shutdown hook, de-register the hook if (Thread.currentThread() != this.shutdownAction) { try { Runtime.getRuntime().removeShutdownHook(this.shutdownAction); } catch (final Exception e) { // ignore } } } } } private void shutdown(boolean fromShutdownHook, boolean forceImmediate) { if (clientStopped.attemptSet()) { DSO_LOGGER.info( "shuting down Terracotta Client hook=" + fromShutdownHook + " force=" + forceImmediate); shutdownClient(fromShutdownHook, forceImmediate); } else { DSO_LOGGER.info("Client already shutdown."); } synchronized (this) { // notify in case the connection establisher is waiting for something notifyAll(); } } private class ShutdownAction implements Runnable { @Override public void run() { DSO_LOGGER.info("Running L1 VM shutdown hook"); shutdown(true, false); } } }