/**
* Stops the block worker. This method should only be called to terminate the worker.
*
* @throws IOException if the data server fails to close
*/
public void stop() throws IOException {
mDataServer.close();
mThriftServer.stop();
mThriftServerSocket.close();
mSessionCleanerThread.stop();
mBlockMasterClient.close();
if (mSpaceReserver != null) {
mSpaceReserver.stop();
}
mFileSystemMasterClient.close();
// Use shutdownNow because HeartbeatThreads never finish until they are interrupted
getExecutorService().shutdownNow();
mWorkerMetricsSystem.stop();
try {
mWebServer.shutdownWebServer();
} catch (Exception e) {
LOG.error("Failed to stop web server", e);
}
mBlockDataManager.stop();
while (!mDataServer.isClosed() || mThriftServer.isServing()) {
// The reason to stop and close again is due to some issues in Thrift.
mDataServer.close();
mThriftServer.stop();
mThriftServerSocket.close();
CommonUtils.sleepMs(100);
}
}
/**
* Stop this TachyonWorker. Stop all the threads belong to this TachyonWorker.
*
* @throws IOException
* @throws InterruptedException
*/
public void stop() throws IOException, InterruptedException {
mStop = true;
mWorkerStorage.stop();
mDataServer.close();
mServer.stop();
mServerTNonblockingServerSocket.close();
mExecutorService.shutdown();
while (!mDataServer.isClosed() || mServer.isServing() || mHeartbeatThread.isAlive()) {
// TODO The reason to stop and close again is due to some issues in Thrift.
mServer.stop();
mServerTNonblockingServerSocket.close();
CommonUtils.sleepMs(null, 100);
}
mHeartbeatThread.join();
}
/**
* @param masterAddress The TachyonMaster's address.
* @param workerAddress This TachyonWorker's address.
* @param dataPort This TachyonWorker's data server's port
* @param selectorThreads The number of selector threads of the worker's thrift server
* @param acceptQueueSizePerThreads The accept queue size per thread of the worker's thrift server
* @param workerThreads The number of threads of the worker's thrift server
* @param dataFolder This TachyonWorker's local folder's path
* @param memoryCapacityBytes The maximum memory space this TachyonWorker can use, in bytes
*/
private TachyonWorker(
InetSocketAddress masterAddress,
InetSocketAddress workerAddress,
int dataPort,
int selectorThreads,
int acceptQueueSizePerThreads,
int workerThreads,
String dataFolder,
long memoryCapacityBytes) {
CommonConf.assertValidPort(masterAddress);
CommonConf.assertValidPort(workerAddress);
CommonConf.assertValidPort(dataPort);
mMasterAddress = masterAddress;
mWorkerStorage =
new WorkerStorage(mMasterAddress, dataFolder, memoryCapacityBytes, mExecutorService);
mWorkerServiceHandler = new WorkerServiceHandler(mWorkerStorage);
// Extract the port from the generated socket.
// When running tests, its great to use port '0' so the system will figure out what port to use
// (any random free port).
// In a production or any real deployment setup, port '0' should not be used as it will make
// deployment more complicated.
InetSocketAddress dataAddress = new InetSocketAddress(workerAddress.getHostName(), dataPort);
BlocksLocker blockLocker = new BlocksLocker(mWorkerStorage, Users.DATASERVER_USER_ID);
mDataServer = createDataServer(dataAddress, blockLocker);
mDataPort = mDataServer.getPort();
mHeartbeatThread = new Thread(this);
try {
LOG.info("Tachyon Worker version " + Version.VERSION + " tries to start @ " + workerAddress);
WorkerService.Processor<WorkerServiceHandler> processor =
new WorkerService.Processor<WorkerServiceHandler>(mWorkerServiceHandler);
mServerTNonblockingServerSocket = new TNonblockingServerSocket(workerAddress);
mPort = NetworkUtils.getPort(mServerTNonblockingServerSocket);
mServer =
new TThreadedSelectorServer(
new TThreadedSelectorServer.Args(mServerTNonblockingServerSocket)
.processor(processor)
.selectorThreads(selectorThreads)
.acceptQueueSizePerThread(acceptQueueSizePerThreads)
.workerThreads(workerThreads));
} catch (TTransportException e) {
LOG.error(e.getMessage(), e);
throw Throwables.propagate(e);
}
mWorkerAddress =
new NetAddress(workerAddress.getAddress().getCanonicalHostName(), mPort, mDataPort);
mWorkerStorage.initialize(mWorkerAddress);
}
/**
* Creates a new instance of {@link BlockWorker}.
*
* @throws ConnectionFailedException if network connection failed
* @throws IOException for other exceptions
*/
public BlockWorker() throws IOException, ConnectionFailedException {
super(
Executors.newFixedThreadPool(
4, ThreadFactoryUtils.build("block-worker-heartbeat-%d", true)));
mTachyonConf = WorkerContext.getConf();
mStartTimeMs = System.currentTimeMillis();
// Setup MasterClientBase
mBlockMasterClient =
new BlockMasterClient(
NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, mTachyonConf),
mTachyonConf);
mFileSystemMasterClient =
new FileSystemMasterClient(
NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, mTachyonConf),
mTachyonConf);
// Set up BlockDataManager
WorkerSource workerSource = new WorkerSource();
mBlockDataManager =
new BlockDataManager(
workerSource, mBlockMasterClient, mFileSystemMasterClient, new TieredBlockStore());
// Setup metrics collection
mWorkerMetricsSystem = new MetricsSystem("worker", mTachyonConf);
workerSource.registerGauges(mBlockDataManager);
mWorkerMetricsSystem.registerSource(workerSource);
// Setup DataServer
mDataServer =
DataServer.Factory.create(
NetworkAddressUtils.getBindAddress(ServiceType.WORKER_DATA, mTachyonConf),
mBlockDataManager,
mTachyonConf);
// Reset data server port
mTachyonConf.set(Constants.WORKER_DATA_PORT, Integer.toString(mDataServer.getPort()));
// Setup RPC Server
mServiceHandler = new BlockWorkerClientServiceHandler(mBlockDataManager);
mThriftServerSocket = createThriftServerSocket();
mPort = NetworkAddressUtils.getThriftPort(mThriftServerSocket);
// Reset worker RPC port
mTachyonConf.set(Constants.WORKER_RPC_PORT, Integer.toString(mPort));
mThriftServer = createThriftServer();
// Setup web server
mWebServer =
new WorkerUIWebServer(
ServiceType.WORKER_WEB,
NetworkAddressUtils.getBindAddress(ServiceType.WORKER_WEB, mTachyonConf),
mBlockDataManager,
NetworkAddressUtils.getConnectAddress(ServiceType.WORKER_RPC, mTachyonConf),
mStartTimeMs,
mTachyonConf);
mWorkerMetricsSystem.start();
// Add the metrics servlet to the web server, this must be done after the metrics system starts
mWebServer.addHandler(mWorkerMetricsSystem.getServletHandler());
mWebServer.startWebServer();
int webPort = mWebServer.getLocalPort();
// Get the worker id
mWorkerNetAddress =
new NetAddress(
NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, mTachyonConf),
mPort,
mDataServer.getPort(),
webPort);
WorkerIdRegistry.registerWithBlockMaster(mBlockMasterClient, mWorkerNetAddress);
mBlockMasterSync =
new BlockMasterSync(mBlockDataManager, mWorkerNetAddress, mBlockMasterClient);
// Setup PinListSyncer
mPinListSync = new PinListSync(mBlockDataManager, mFileSystemMasterClient);
// Setup session cleaner
mSessionCleanerThread = new SessionCleaner(mBlockDataManager);
// Setup space reserver
if (mTachyonConf.getBoolean(Constants.WORKER_TIERED_STORE_RESERVER_ENABLED)) {
mSpaceReserver = new SpaceReserver(mBlockDataManager);
}
}
/** @return the worker data service port */
public int getDataLocalPort() {
return mDataServer.getPort();
}
/** @return the worker data service bind host */
public String getDataBindHost() {
return mDataServer.getBindHost();
}