private void stopServing() {
mThriftServer.stop();
mThriftServerSocket.close();
mWorkerMetricsSystem.stop();
try {
mWebServer.shutdownWebServer();
} catch (Exception e) {
LOG.error("Failed to stop web server", e);
}
mWorkerMetricsSystem.stop();
}
/**
* Stops the block worker. This method should only be called to terminate the worker.
*
* @throws IOException if the data server fails to close
*/
public void stop() throws IOException {
mDataServer.close();
mThriftServer.stop();
mThriftServerSocket.close();
mSessionCleanerThread.stop();
mBlockMasterClient.close();
if (mSpaceReserver != null) {
mSpaceReserver.stop();
}
mFileSystemMasterClient.close();
// Use shutdownNow because HeartbeatThreads never finish until they are interrupted
getExecutorService().shutdownNow();
mWorkerMetricsSystem.stop();
try {
mWebServer.shutdownWebServer();
} catch (Exception e) {
LOG.error("Failed to stop web server", e);
}
mBlockDataManager.stop();
while (!mDataServer.isClosed() || mThriftServer.isServing()) {
// The reason to stop and close again is due to some issues in Thrift.
mDataServer.close();
mThriftServer.stop();
mThriftServerSocket.close();
CommonUtils.sleepMs(100);
}
}
/** Constructor of {@link TachyonWorker}. */
public TachyonWorker() {
try {
mStartTimeMs = System.currentTimeMillis();
mTachyonConf = WorkerContext.getConf();
mBlockWorker = new BlockWorker();
mFileSystemWorker = new FileSystemWorker(mBlockWorker);
mAdditionalWorkers = Lists.newArrayList();
List<? extends Worker> workers = Lists.newArrayList(mBlockWorker, mFileSystemWorker);
// Discover and register the available factories
// NOTE: ClassLoader is explicitly specified so we don't need to set ContextClassLoader
ServiceLoader<WorkerFactory> discoveredMasterFactories =
ServiceLoader.load(WorkerFactory.class, WorkerFactory.class.getClassLoader());
for (WorkerFactory factory : discoveredMasterFactories) {
Worker worker = factory.create(workers);
if (worker != null) {
mAdditionalWorkers.add(worker);
}
}
// Setup metrics collection system
mWorkerMetricsSystem = new MetricsSystem("worker", mTachyonConf);
WorkerSource workerSource = WorkerContext.getWorkerSource();
workerSource.registerGauges(mBlockWorker);
mWorkerMetricsSystem.registerSource(workerSource);
// Setup web server
mWebServer =
new WorkerUIWebServer(
ServiceType.WORKER_WEB,
NetworkAddressUtils.getBindAddress(ServiceType.WORKER_WEB, mTachyonConf),
mBlockWorker,
NetworkAddressUtils.getConnectAddress(ServiceType.WORKER_RPC, mTachyonConf),
mStartTimeMs,
mTachyonConf);
// Setup Thrift server
mThriftServerSocket = createThriftServerSocket();
mRPCPort = NetworkAddressUtils.getThriftPort(mThriftServerSocket);
// Reset worker RPC port based on assigned port number
mTachyonConf.set(Constants.WORKER_RPC_PORT, Integer.toString(mRPCPort));
mThriftServer = createThriftServer();
mWorkerAddress =
NetworkAddressUtils.getConnectAddress(
NetworkAddressUtils.ServiceType.WORKER_RPC, mTachyonConf);
} catch (Exception e) {
LOG.error("Failed to initialize {}", this.getClass().getName(), e);
System.exit(-1);
}
}
/**
* Starts the Tachyon worker server.
*
* @throws Exception if the workers fail to start
*/
public void start() throws Exception {
// NOTE: the order to start different services is sensitive. If you change it, do it cautiously.
// Start serving metrics system, this will not block
mWorkerMetricsSystem.start();
// Start serving the web server, this will not block
// Requirement: metrics system started so we could add the metrics servlet to the web server
// Consequence: when starting webserver, the webport will be updated.
mWebServer.addHandler(mWorkerMetricsSystem.getServletHandler());
mWebServer.startWebServer();
// Set updated net address for this worker in context
// Requirement: RPC, web, and dataserver ports are updated
// Consequence: create a NetAddress object and set it into WorkerContext
mNetAddress =
new NetAddress(
NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, mTachyonConf),
mTachyonConf.getInt(Constants.WORKER_RPC_PORT),
getDataLocalPort(),
mTachyonConf.getInt(Constants.WORKER_WEB_PORT));
WorkerContext.setWorkerNetAddress(mNetAddress);
// Start each worker
// Requirement: NetAddress set in WorkerContext, so block worker can initialize BlockMasterSync
// Consequence: worker id is granted
startWorkers();
LOG.info("Started worker with id {}", WorkerIdRegistry.getWorkerId());
mIsServingRPC = true;
// Start serving RPC, this will block
LOG.info("Tachyon Worker version {} started @ {}", Version.VERSION, mWorkerAddress);
mThriftServer.serve();
LOG.info("Tachyon Worker version {} ended @ {}", Version.VERSION, mWorkerAddress);
}
/**
* Creates a new instance of {@link BlockWorker}.
*
* @throws ConnectionFailedException if network connection failed
* @throws IOException for other exceptions
*/
public BlockWorker() throws IOException, ConnectionFailedException {
super(
Executors.newFixedThreadPool(
4, ThreadFactoryUtils.build("block-worker-heartbeat-%d", true)));
mTachyonConf = WorkerContext.getConf();
mStartTimeMs = System.currentTimeMillis();
// Setup MasterClientBase
mBlockMasterClient =
new BlockMasterClient(
NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, mTachyonConf),
mTachyonConf);
mFileSystemMasterClient =
new FileSystemMasterClient(
NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, mTachyonConf),
mTachyonConf);
// Set up BlockDataManager
WorkerSource workerSource = new WorkerSource();
mBlockDataManager =
new BlockDataManager(
workerSource, mBlockMasterClient, mFileSystemMasterClient, new TieredBlockStore());
// Setup metrics collection
mWorkerMetricsSystem = new MetricsSystem("worker", mTachyonConf);
workerSource.registerGauges(mBlockDataManager);
mWorkerMetricsSystem.registerSource(workerSource);
// Setup DataServer
mDataServer =
DataServer.Factory.create(
NetworkAddressUtils.getBindAddress(ServiceType.WORKER_DATA, mTachyonConf),
mBlockDataManager,
mTachyonConf);
// Reset data server port
mTachyonConf.set(Constants.WORKER_DATA_PORT, Integer.toString(mDataServer.getPort()));
// Setup RPC Server
mServiceHandler = new BlockWorkerClientServiceHandler(mBlockDataManager);
mThriftServerSocket = createThriftServerSocket();
mPort = NetworkAddressUtils.getThriftPort(mThriftServerSocket);
// Reset worker RPC port
mTachyonConf.set(Constants.WORKER_RPC_PORT, Integer.toString(mPort));
mThriftServer = createThriftServer();
// Setup web server
mWebServer =
new WorkerUIWebServer(
ServiceType.WORKER_WEB,
NetworkAddressUtils.getBindAddress(ServiceType.WORKER_WEB, mTachyonConf),
mBlockDataManager,
NetworkAddressUtils.getConnectAddress(ServiceType.WORKER_RPC, mTachyonConf),
mStartTimeMs,
mTachyonConf);
mWorkerMetricsSystem.start();
// Add the metrics servlet to the web server, this must be done after the metrics system starts
mWebServer.addHandler(mWorkerMetricsSystem.getServletHandler());
mWebServer.startWebServer();
int webPort = mWebServer.getLocalPort();
// Get the worker id
mWorkerNetAddress =
new NetAddress(
NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, mTachyonConf),
mPort,
mDataServer.getPort(),
webPort);
WorkerIdRegistry.registerWithBlockMaster(mBlockMasterClient, mWorkerNetAddress);
mBlockMasterSync =
new BlockMasterSync(mBlockDataManager, mWorkerNetAddress, mBlockMasterClient);
// Setup PinListSyncer
mPinListSync = new PinListSync(mBlockDataManager, mFileSystemMasterClient);
// Setup session cleaner
mSessionCleanerThread = new SessionCleaner(mBlockDataManager);
// Setup space reserver
if (mTachyonConf.getBoolean(Constants.WORKER_TIERED_STORE_RESERVER_ENABLED)) {
mSpaceReserver = new SpaceReserver(mBlockDataManager);
}
}