/**
* Creates a new instance of {@link FileSystemWorker}.
*
* @param blockDataManager a block data manager handle
* @throws IOException if an I/O error occurs
*/
public FileSystemWorker(BlockDataManager blockDataManager) throws IOException {
super(
Executors.newFixedThreadPool(
3, ThreadFactoryUtils.build("file-system-worker-heartbeat-%d", true)));
Preconditions.checkState(WorkerIdRegistry.getWorkerId() != 0, "Failed to register worker");
mTachyonConf = WorkerContext.getConf();
mFileDataManager = new FileDataManager(Preconditions.checkNotNull(blockDataManager));
// Setup MasterClientBase
mFileSystemMasterWorkerClient =
new FileSystemMasterClient(
NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, mTachyonConf),
mTachyonConf);
}
private TachyonFS(TachyonConf tachyonConf) {
super(tachyonConf);
mMasterAddress = NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, tachyonConf);
mZookeeperMode = mTachyonConf.getBoolean(Constants.USE_ZOOKEEPER);
mExecutorService =
Executors.newFixedThreadPool(2, ThreadFactoryUtils.build("client-heartbeat-%d", true));
mFSMasterClient = mCloser.register(FileSystemContext.INSTANCE.acquireMasterClient());
mBlockMasterClient = mCloser.register(BlockStoreContext.INSTANCE.acquireMasterClient());
mWorkerClient = mCloser.register(BlockStoreContext.INSTANCE.acquireWorkerClient());
mUserFailedSpaceRequestLimits = mTachyonConf.getInt(Constants.USER_FAILED_SPACE_REQUEST_LIMITS);
String scheme = mZookeeperMode ? Constants.SCHEME_FT : Constants.SCHEME;
String authority = mMasterAddress.getHostName() + ":" + mMasterAddress.getPort();
mRootUri = new TachyonURI(scheme, authority, TachyonURI.SEPARATOR);
}
/**
* Creates a new instance of {@link BlockWorker}.
*
* @throws ConnectionFailedException if network connection failed
* @throws IOException for other exceptions
*/
public BlockWorker() throws IOException, ConnectionFailedException {
super(
Executors.newFixedThreadPool(
4, ThreadFactoryUtils.build("block-worker-heartbeat-%d", true)));
mTachyonConf = WorkerContext.getConf();
mStartTimeMs = System.currentTimeMillis();
// Setup MasterClientBase
mBlockMasterClient =
new BlockMasterClient(
NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, mTachyonConf),
mTachyonConf);
mFileSystemMasterClient =
new FileSystemMasterClient(
NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, mTachyonConf),
mTachyonConf);
// Set up BlockDataManager
WorkerSource workerSource = new WorkerSource();
mBlockDataManager =
new BlockDataManager(
workerSource, mBlockMasterClient, mFileSystemMasterClient, new TieredBlockStore());
// Setup metrics collection
mWorkerMetricsSystem = new MetricsSystem("worker", mTachyonConf);
workerSource.registerGauges(mBlockDataManager);
mWorkerMetricsSystem.registerSource(workerSource);
// Setup DataServer
mDataServer =
DataServer.Factory.create(
NetworkAddressUtils.getBindAddress(ServiceType.WORKER_DATA, mTachyonConf),
mBlockDataManager,
mTachyonConf);
// Reset data server port
mTachyonConf.set(Constants.WORKER_DATA_PORT, Integer.toString(mDataServer.getPort()));
// Setup RPC Server
mServiceHandler = new BlockWorkerClientServiceHandler(mBlockDataManager);
mThriftServerSocket = createThriftServerSocket();
mPort = NetworkAddressUtils.getThriftPort(mThriftServerSocket);
// Reset worker RPC port
mTachyonConf.set(Constants.WORKER_RPC_PORT, Integer.toString(mPort));
mThriftServer = createThriftServer();
// Setup web server
mWebServer =
new WorkerUIWebServer(
ServiceType.WORKER_WEB,
NetworkAddressUtils.getBindAddress(ServiceType.WORKER_WEB, mTachyonConf),
mBlockDataManager,
NetworkAddressUtils.getConnectAddress(ServiceType.WORKER_RPC, mTachyonConf),
mStartTimeMs,
mTachyonConf);
mWorkerMetricsSystem.start();
// Add the metrics servlet to the web server, this must be done after the metrics system starts
mWebServer.addHandler(mWorkerMetricsSystem.getServletHandler());
mWebServer.startWebServer();
int webPort = mWebServer.getLocalPort();
// Get the worker id
mWorkerNetAddress =
new NetAddress(
NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, mTachyonConf),
mPort,
mDataServer.getPort(),
webPort);
WorkerIdRegistry.registerWithBlockMaster(mBlockMasterClient, mWorkerNetAddress);
mBlockMasterSync =
new BlockMasterSync(mBlockDataManager, mWorkerNetAddress, mBlockMasterClient);
// Setup PinListSyncer
mPinListSync = new PinListSync(mBlockDataManager, mFileSystemMasterClient);
// Setup session cleaner
mSessionCleanerThread = new SessionCleaner(mBlockDataManager);
// Setup space reserver
if (mTachyonConf.getBoolean(Constants.WORKER_TIERED_STORE_RESERVER_ENABLED)) {
mSpaceReserver = new SpaceReserver(mBlockDataManager);
}
}
/** Integration tests for tachyon.worker.DataServer. */
@RunWith(Parameterized.class)
public class DataServerIntegrationTest {
private static final int WORKER_CAPACITY_BYTES = 1000;
private static final int USER_QUOTA_UNIT_BYTES = 100;
@Parameterized.Parameters
public static Collection<Object[]> data() {
// Creates a new instance of DataServerIntegrationTest for different combinations of parameters.
List<Object[]> list = new ArrayList<Object[]>();
list.add(
new Object[] {
IntegrationTestConstants.NETTY_DATA_SERVER,
IntegrationTestConstants.MAPPED_TRANSFER,
IntegrationTestConstants.TCP_BLOCK_READER
});
list.add(
new Object[] {
IntegrationTestConstants.NETTY_DATA_SERVER,
IntegrationTestConstants.MAPPED_TRANSFER,
IntegrationTestConstants.NETTY_BLOCK_READER
});
list.add(
new Object[] {
IntegrationTestConstants.NETTY_DATA_SERVER,
IntegrationTestConstants.FILE_CHANNEL_TRANSFER,
IntegrationTestConstants.TCP_BLOCK_READER
});
list.add(
new Object[] {
IntegrationTestConstants.NETTY_DATA_SERVER,
IntegrationTestConstants.FILE_CHANNEL_TRANSFER,
IntegrationTestConstants.NETTY_BLOCK_READER
});
// The transfer type is not applicable to the NIODataServer.
list.add(
new Object[] {
IntegrationTestConstants.NIO_DATA_SERVER,
IntegrationTestConstants.UNUSED_TRANSFER,
IntegrationTestConstants.TCP_BLOCK_READER
});
list.add(
new Object[] {
IntegrationTestConstants.NIO_DATA_SERVER,
IntegrationTestConstants.UNUSED_TRANSFER,
IntegrationTestConstants.NETTY_BLOCK_READER
});
return list;
}
private final String mDataServerClass;
private final String mNettyTransferType;
private final String mBlockReader;
private final ExecutorService mExecutorService =
Executors.newFixedThreadPool(2, ThreadFactoryUtils.build("test-executor-%d", true));
private LocalTachyonCluster mLocalTachyonCluster = null;
private TachyonFileSystem mTFS = null;
private TachyonConf mWorkerTachyonConf;
private BlockMasterClient mBlockMasterClient;
public DataServerIntegrationTest(String className, String nettyTransferType, String blockReader) {
mDataServerClass = className;
mNettyTransferType = nettyTransferType;
mBlockReader = blockReader;
}
@After
public final void after() throws Exception {
mTFS.close();
mBlockMasterClient.close();
mLocalTachyonCluster.stop();
System.clearProperty(Constants.WORKER_DATA_SERVER);
System.clearProperty(Constants.WORKER_NETTY_FILE_TRANSFER_TYPE);
System.clearProperty(Constants.USER_REMOTE_BLOCK_READER);
}
/** Asserts that the message back matches the block response protocols for the error case. */
private void assertError(final DataServerMessage msg, final long blockId) {
Assert.assertEquals(blockId, msg.getBlockId());
Assert.assertEquals(0, msg.getLength());
Assert.assertNotEquals(msg.getStatus().getId(), RPCResponse.Status.SUCCESS.getId());
}
/** Asserts that the message back matches the block response protocols. */
private void assertValid(
final DataServerMessage msg,
final ByteBuffer expectedData,
final long blockId,
final long offset,
final long length) {
Assert.assertEquals(expectedData, msg.getReadOnlyData());
Assert.assertEquals(blockId, msg.getBlockId());
Assert.assertEquals(offset, msg.getOffset());
Assert.assertEquals(length, msg.getLength());
}
/** Asserts that the message back matches the block response protocols. */
private void assertValid(
final DataServerMessage msg,
final int expectedSize,
final long blockId,
final long offset,
final long length) {
assertValid(msg, BufferUtils.getIncreasingByteBuffer(expectedSize), blockId, offset, length);
}
@Before
public final void before() throws Exception {
TachyonConf tachyonConf = new TachyonConf();
tachyonConf.set(Constants.USER_FILE_BUFFER_BYTES, String.valueOf(100));
System.setProperty(Constants.WORKER_DATA_SERVER, mDataServerClass);
System.setProperty(Constants.WORKER_NETTY_FILE_TRANSFER_TYPE, mNettyTransferType);
System.setProperty(Constants.USER_REMOTE_BLOCK_READER, mBlockReader);
mLocalTachyonCluster =
new LocalTachyonCluster(WORKER_CAPACITY_BYTES, USER_QUOTA_UNIT_BYTES, Constants.GB);
mLocalTachyonCluster.start(tachyonConf);
mWorkerTachyonConf = mLocalTachyonCluster.getWorkerTachyonConf();
mTFS = mLocalTachyonCluster.getClient();
mBlockMasterClient =
new BlockMasterClient(
new InetSocketAddress(
mLocalTachyonCluster.getMasterHostname(), mLocalTachyonCluster.getMasterPort()),
mExecutorService,
mWorkerTachyonConf);
}
@Test
public void lengthTooSmall() throws IOException, TException {
final int length = 20;
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/readTooLarge", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block = getFirstBlockInfo(file);
DataServerMessage recvMsg = request(block, 0, length * -2);
assertError(recvMsg, block.blockId);
}
@Test
public void multiReadTest() throws IOException, TException {
final int length = 20;
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/multiReadTest", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block = getFirstBlockInfo(file);
for (int i = 0; i < 10; i++) {
DataServerMessage recvMsg = request(block);
assertValid(recvMsg, length, block.getBlockId(), 0, length);
}
}
@Test
public void negativeOffset() throws IOException, TException {
final int length = 10;
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/readTooLarge", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block = getFirstBlockInfo(file);
DataServerMessage recvMsg = request(block, length * -2, 1);
assertError(recvMsg, block.blockId);
}
@Test
public void readMultiFiles() throws IOException, TException {
final int length = WORKER_CAPACITY_BYTES / 2 + 1;
TachyonFile file1 =
TachyonFSTestUtils.createByteFile(
mTFS, "/readFile1", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block1 = getFirstBlockInfo(file1);
DataServerMessage recvMsg1 = request(block1);
assertValid(recvMsg1, length, block1.getBlockId(), 0, length);
TachyonFile file2 =
TachyonFSTestUtils.createByteFile(
mTFS, "/readFile2", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block2 = getFirstBlockInfo(file2);
DataServerMessage recvMsg2 = request(block2);
assertValid(recvMsg2, length, block2.getBlockId(), 0, length);
CommonUtils.sleepMs(
mWorkerTachyonConf.getInt(Constants.WORKER_TO_MASTER_HEARTBEAT_INTERVAL_MS) * 2 + 10);
FileInfo fileInfo = mTFS.getInfo(mTFS.open(new TachyonURI("/readFile1")));
Assert.assertEquals(0, fileInfo.inMemoryPercentage);
}
@Test
public void readPartialTest1() throws TException, IOException {
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
BlockInfo block = getFirstBlockInfo(file);
final int offset = 0;
final int length = 6;
DataServerMessage recvMsg = request(block, offset, length);
assertValid(recvMsg, length, block.getBlockId(), offset, length);
}
@Test
public void readPartialTest2() throws TException, IOException {
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
BlockInfo block = getFirstBlockInfo(file);
final int offset = 2;
final int length = 6;
DataServerMessage recvMsg = request(block, offset, length);
assertValid(
recvMsg,
BufferUtils.getIncreasingByteBuffer(offset, length),
block.getBlockId(),
offset,
length);
}
@Test
public void readTest() throws IOException, TException {
final int length = 10;
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block = getFirstBlockInfo(file);
DataServerMessage recvMsg = request(block);
assertValid(recvMsg, length, block.getBlockId(), 0, length);
}
@Test
public void readThroughClientTest() throws IOException, TException {
final int length = 10;
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block = getFirstBlockInfo(file);
RemoteBlockReader client =
RemoteBlockReader.Factory.createRemoteBlockReader(mWorkerTachyonConf);
ByteBuffer result =
client.readRemoteBlock(
new InetSocketAddress(
block.getLocations().get(0).getWorkerAddress().getHost(),
block.getLocations().get(0).getWorkerAddress().getDataPort()),
block.getBlockId(),
0,
length);
Assert.assertEquals(BufferUtils.getIncreasingByteBuffer(length), result);
}
// TODO(calvin): Make this work with the new BlockReader.
// @Test
public void readThroughClientNonExistentTest() throws IOException, TException {
final int length = 10;
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block = getFirstBlockInfo(file);
// Get the maximum block id, for use in determining a non-existent block id.
FileInfo fileInfo = mTFS.getInfo(file);
long maxBlockId = block.getBlockId();
for (long blockId : fileInfo.blockIds) {
if (blockId > maxBlockId) {
maxBlockId = blockId;
}
}
RemoteBlockReader client =
RemoteBlockReader.Factory.createRemoteBlockReader(mWorkerTachyonConf);
ByteBuffer result =
client.readRemoteBlock(
new InetSocketAddress(
block.getLocations().get(0).getWorkerAddress().getHost(),
block.getLocations().get(0).getWorkerAddress().getDataPort()),
maxBlockId + 1,
0,
length);
Assert.assertNull(result);
}
@Test
public void readTooLarge() throws IOException, TException {
final int length = 20;
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/readTooLarge", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block = getFirstBlockInfo(file);
DataServerMessage recvMsg = request(block, 0, length * 2);
assertError(recvMsg, block.blockId);
}
@Test
public void tooLargeOffset() throws IOException, TException {
final int length = 10;
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS, "/readTooLarge", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, length);
BlockInfo block = getFirstBlockInfo(file);
DataServerMessage recvMsg = request(block, length * 2, 1);
assertError(recvMsg, block.blockId);
}
/** Requests a block from the server. This call will read the full block. */
private DataServerMessage request(final BlockInfo block) throws IOException, TException {
return request(block, 0, -1);
}
/**
* Create a new socket to the data port and send a block request. The returned value is the
* response from the server.
*/
private DataServerMessage request(final BlockInfo block, final long offset, final long length)
throws IOException, TException {
DataServerMessage sendMsg =
DataServerMessage.createBlockRequestMessage(block.blockId, offset, length);
SocketChannel socketChannel =
SocketChannel.open(
new InetSocketAddress(
block.getLocations().get(0).getWorkerAddress().getHost(),
block.getLocations().get(0).getWorkerAddress().getDataPort()));
try {
while (!sendMsg.finishSending()) {
sendMsg.send(socketChannel);
}
DataServerMessage recvMsg =
DataServerMessage.createBlockResponseMessage(false, block.blockId, offset, length, null);
while (!recvMsg.isMessageReady()) {
int numRead = recvMsg.recv(socketChannel);
if (numRead == -1) {
break;
}
}
return recvMsg;
} finally {
socketChannel.close();
}
}
/**
* Returns the MasterBlockInfo of the first block of the file
*
* @param tachyonFile the file to get the first MasterBlockInfro for
* @return the MasterBlockInfo of the first block in the file
* @throws IOException if the block does not exist
* @throws TException
*/
private BlockInfo getFirstBlockInfo(TachyonFile tachyonFile) throws IOException, TException {
FileInfo fileInfo = mTFS.getInfo(tachyonFile);
return mBlockMasterClient.getBlockInfo(fileInfo.blockIds.get(0));
}
}
/** Entry point for a worker daemon. */
public class TachyonWorker implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_TYPE);
/**
* Create a new TachyonWorker
*
* @param masterAddress The TachyonMaster's address
* @param workerAddress This TachyonWorker's address
* @param dataPort This TachyonWorker's data server's port
* @param selectorThreads The number of selector threads of the worker's thrift server
* @param acceptQueueSizePerThreads The accept queue size per thread of the worker's thrift server
* @param workerThreads The number of threads of the worker's thrift server
* @param localFolder This TachyonWorker's local folder's path
* @param spaceLimitBytes The maximum memory space this TachyonWorker can use, in bytes
* @return The new TachyonWorker
*/
public static synchronized TachyonWorker createWorker(
InetSocketAddress masterAddress,
InetSocketAddress workerAddress,
int dataPort,
int selectorThreads,
int acceptQueueSizePerThreads,
int workerThreads,
String localFolder,
long spaceLimitBytes) {
return new TachyonWorker(
masterAddress,
workerAddress,
dataPort,
selectorThreads,
acceptQueueSizePerThreads,
workerThreads,
localFolder,
spaceLimitBytes);
}
/**
* Create a new TachyonWorker
*
* @param masterAddress The TachyonMaster's address. e.g., localhost:19998
* @param workerAddress This TachyonWorker's address. e.g., localhost:29998
* @param dataPort This TachyonWorker's data server's port
* @param selectorThreads The number of selector threads of the worker's thrift server
* @param acceptQueueSizePerThreads The accept queue size per thread of the worker's thrift server
* @param workerThreads The number of threads of the worker's thrift server
* @param localFolder This TachyonWorker's local folder's path
* @param spaceLimitBytes The maximum memory space this TachyonWorker can use, in bytes
* @return The new TachyonWorker
*/
public static synchronized TachyonWorker createWorker(
String masterAddress,
String workerAddress,
int dataPort,
int selectorThreads,
int acceptQueueSizePerThreads,
int workerThreads,
String localFolder,
long spaceLimitBytes) {
String[] address = masterAddress.split(":");
InetSocketAddress master = new InetSocketAddress(address[0], Integer.parseInt(address[1]));
address = workerAddress.split(":");
InetSocketAddress worker = new InetSocketAddress(address[0], Integer.parseInt(address[1]));
return new TachyonWorker(
master,
worker,
dataPort,
selectorThreads,
acceptQueueSizePerThreads,
workerThreads,
localFolder,
spaceLimitBytes);
}
private static String getMasterLocation(String[] args) {
WorkerConf wConf = WorkerConf.get();
String confFileMasterLoc = wConf.MASTER_HOSTNAME + ":" + wConf.MASTER_PORT;
String masterLocation;
if (args.length < 1) {
masterLocation = confFileMasterLoc;
} else {
masterLocation = args[0];
if (masterLocation.indexOf(":") == -1) {
masterLocation += ":" + wConf.MASTER_PORT;
}
if (!masterLocation.equals(confFileMasterLoc)) {
LOG.warn(
"Master Address in configuration file("
+ confFileMasterLoc
+ ") is different "
+ "from the command line one("
+ masterLocation
+ ").");
}
}
return masterLocation;
}
public static void main(String[] args) throws UnknownHostException {
if (args.length > 1) {
LOG.info(
"Usage: java -cp target/tachyon-"
+ Version.VERSION
+ "-jar-with-dependencies.jar "
+ "tachyon.Worker [<MasterHost:Port>]");
System.exit(-1);
}
WorkerConf wConf = WorkerConf.get();
String resolvedWorkerHost = NetworkUtils.getLocalHostName();
LOG.info("Resolved local TachyonWorker host to " + resolvedWorkerHost);
TachyonWorker worker =
TachyonWorker.createWorker(
getMasterLocation(args),
resolvedWorkerHost + ":" + wConf.PORT,
wConf.DATA_PORT,
wConf.SELECTOR_THREADS,
wConf.QUEUE_SIZE_PER_SELECTOR,
wConf.SERVER_THREADS,
wConf.DATA_FOLDER,
wConf.MEMORY_SIZE);
try {
worker.start();
} catch (Exception e) {
LOG.error("Uncaught exception terminating worker", e);
throw new RuntimeException(e);
}
}
private final InetSocketAddress mMasterAddress;
private final NetAddress mWorkerAddress;
private TServer mServer;
private TNonblockingServerSocket mServerTNonblockingServerSocket;
private final WorkerStorage mWorkerStorage;
private final WorkerServiceHandler mWorkerServiceHandler;
private final DataServer mDataServer;
private final Thread mHeartbeatThread;
private volatile boolean mStop = false;
private final int mPort;
private final int mDataPort;
private final ExecutorService mExecutorService =
Executors.newFixedThreadPool(1, ThreadFactoryUtils.daemon("heartbeat-worker-%d"));
/**
* @param masterAddress The TachyonMaster's address.
* @param workerAddress This TachyonWorker's address.
* @param dataPort This TachyonWorker's data server's port
* @param selectorThreads The number of selector threads of the worker's thrift server
* @param acceptQueueSizePerThreads The accept queue size per thread of the worker's thrift server
* @param workerThreads The number of threads of the worker's thrift server
* @param dataFolder This TachyonWorker's local folder's path
* @param memoryCapacityBytes The maximum memory space this TachyonWorker can use, in bytes
*/
private TachyonWorker(
InetSocketAddress masterAddress,
InetSocketAddress workerAddress,
int dataPort,
int selectorThreads,
int acceptQueueSizePerThreads,
int workerThreads,
String dataFolder,
long memoryCapacityBytes) {
CommonConf.assertValidPort(masterAddress);
CommonConf.assertValidPort(workerAddress);
CommonConf.assertValidPort(dataPort);
mMasterAddress = masterAddress;
mWorkerStorage =
new WorkerStorage(mMasterAddress, dataFolder, memoryCapacityBytes, mExecutorService);
mWorkerServiceHandler = new WorkerServiceHandler(mWorkerStorage);
// Extract the port from the generated socket.
// When running tests, its great to use port '0' so the system will figure out what port to use
// (any random free port).
// In a production or any real deployment setup, port '0' should not be used as it will make
// deployment more complicated.
InetSocketAddress dataAddress = new InetSocketAddress(workerAddress.getHostName(), dataPort);
BlocksLocker blockLocker = new BlocksLocker(mWorkerStorage, Users.DATASERVER_USER_ID);
mDataServer = createDataServer(dataAddress, blockLocker);
mDataPort = mDataServer.getPort();
mHeartbeatThread = new Thread(this);
try {
LOG.info("Tachyon Worker version " + Version.VERSION + " tries to start @ " + workerAddress);
WorkerService.Processor<WorkerServiceHandler> processor =
new WorkerService.Processor<WorkerServiceHandler>(mWorkerServiceHandler);
mServerTNonblockingServerSocket = new TNonblockingServerSocket(workerAddress);
mPort = NetworkUtils.getPort(mServerTNonblockingServerSocket);
mServer =
new TThreadedSelectorServer(
new TThreadedSelectorServer.Args(mServerTNonblockingServerSocket)
.processor(processor)
.selectorThreads(selectorThreads)
.acceptQueueSizePerThread(acceptQueueSizePerThreads)
.workerThreads(workerThreads));
} catch (TTransportException e) {
LOG.error(e.getMessage(), e);
throw Throwables.propagate(e);
}
mWorkerAddress =
new NetAddress(workerAddress.getAddress().getCanonicalHostName(), mPort, mDataPort);
mWorkerStorage.initialize(mWorkerAddress);
}
private DataServer createDataServer(
final InetSocketAddress dataAddress, final BlocksLocker blockLocker) {
switch (WorkerConf.get().NETWORK_TYPE) {
case NIO:
return new NIODataServer(dataAddress, blockLocker);
case NETTY:
return new NettyDataServer(dataAddress, blockLocker);
default:
throw new AssertionError("Unknown network type: " + WorkerConf.get().NETWORK_TYPE);
}
}
/** Gets the data port of the worker. For unit tests only. */
public int getDataPort() {
return mDataPort;
}
/** Gets the metadata port of the worker. For unit tests only. */
public int getMetaPort() {
return mPort;
}
/**
* Get the worker server handler class. This is for unit test only.
*
* @return the WorkerServiceHandler
*/
WorkerServiceHandler getWorkerServiceHandler() {
return mWorkerServiceHandler;
}
private void login() throws IOException {
WorkerConf wConf = WorkerConf.get();
if (wConf.KEYTAB == null || wConf.PRINCIPAL == null) {
return;
}
UnderFileSystem ufs = UnderFileSystem.get(CommonConf.get().UNDERFS_ADDRESS);
if (ufs instanceof UnderFileSystemHdfs) {
((UnderFileSystemHdfs) ufs)
.login(
wConf.KEYTAB_KEY,
wConf.KEYTAB,
wConf.PRINCIPAL_KEY,
wConf.PRINCIPAL,
NetworkUtils.getFqdnHost(mWorkerAddress));
}
}
@Override
public void run() {
long lastHeartbeatMs = System.currentTimeMillis();
Command cmd = null;
while (!mStop) {
long diff = System.currentTimeMillis() - lastHeartbeatMs;
if (diff < WorkerConf.get().TO_MASTER_HEARTBEAT_INTERVAL_MS) {
LOG.debug("Heartbeat process takes {} ms.", diff);
CommonUtils.sleepMs(LOG, WorkerConf.get().TO_MASTER_HEARTBEAT_INTERVAL_MS - diff);
} else {
LOG.error("Heartbeat process takes " + diff + " ms.");
}
try {
cmd = mWorkerStorage.heartbeat();
lastHeartbeatMs = System.currentTimeMillis();
} catch (IOException e) {
LOG.error(e.getMessage(), e);
mWorkerStorage.resetMasterClient();
CommonUtils.sleepMs(LOG, Constants.SECOND_MS);
cmd = null;
if (System.currentTimeMillis() - lastHeartbeatMs >= WorkerConf.get().HEARTBEAT_TIMEOUT_MS) {
throw new RuntimeException(
"Timebeat timeout " + (System.currentTimeMillis() - lastHeartbeatMs) + "ms");
}
}
if (cmd != null) {
switch (cmd.mCommandType) {
case Unknown:
LOG.error("Unknown command: " + cmd);
break;
case Nothing:
LOG.debug("Nothing command: {}", cmd);
break;
case Register:
LOG.info("Register command: " + cmd);
mWorkerStorage.register();
break;
case Free:
mWorkerStorage.freeBlocks(cmd.mData);
LOG.info("Free command: " + cmd);
break;
case Delete:
LOG.info("Delete command: " + cmd);
break;
default:
throw new RuntimeException("Un-recognized command from master " + cmd.toString());
}
}
mWorkerStorage.checkStatus();
}
}
/** Start the data server thread and heartbeat thread of this TachyonWorker. */
public void start() throws IOException {
login();
mHeartbeatThread.start();
LOG.info("The worker server started @ " + mWorkerAddress);
mServer.serve();
LOG.info("The worker server ends @ " + mWorkerAddress);
}
/**
* Stop this TachyonWorker. Stop all the threads belong to this TachyonWorker.
*
* @throws IOException
* @throws InterruptedException
*/
public void stop() throws IOException, InterruptedException {
mStop = true;
mWorkerStorage.stop();
mDataServer.close();
mServer.stop();
mServerTNonblockingServerSocket.close();
mExecutorService.shutdown();
while (!mDataServer.isClosed() || mServer.isServing() || mHeartbeatThread.isAlive()) {
// TODO The reason to stop and close again is due to some issues in Thrift.
mServer.stop();
mServerTNonblockingServerSocket.close();
CommonUtils.sleepMs(null, 100);
}
mHeartbeatThread.join();
}
}