public static void assertValidPort(final int port, TachyonConf tachyonConf) {
Preconditions.checkArgument(port < 65536, "Port must be less than 65536");
if (!tachyonConf.getBoolean(Constants.IN_TEST_MODE)) {
Preconditions.checkArgument(port > 0, "Port is only allowed to be zero in test mode.");
}
}
public S3UnderFileSystem(String bucketName, TachyonConf tachyonConf) throws ServiceException {
super(tachyonConf);
Preconditions.checkArgument(
tachyonConf.containsKey(Constants.S3_ACCESS_KEY),
"Property " + Constants.S3_ACCESS_KEY + " is required to connect to S3");
Preconditions.checkArgument(
tachyonConf.containsKey(Constants.S3_SECRET_KEY),
"Property " + Constants.S3_SECRET_KEY + " is required to connect to S3");
AWSCredentials awsCredentials =
new AWSCredentials(
tachyonConf.get(Constants.S3_ACCESS_KEY), tachyonConf.get(Constants.S3_SECRET_KEY));
mBucketName = bucketName;
Jets3tProperties props = new Jets3tProperties();
if (tachyonConf.containsKey(Constants.UNDERFS_S3_PROXY_HOST)) {
props.setProperty("httpclient.proxy-autodetect", "false");
props.setProperty("httpclient.proxy-host", tachyonConf.get(Constants.UNDERFS_S3_PROXY_HOST));
props.setProperty("httpclient.proxy-port", tachyonConf.get(Constants.UNDERFS_S3_PROXY_PORT));
}
if (tachyonConf.containsKey(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
props.setProperty(
"s3service.https-only",
Boolean.toString(tachyonConf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)));
}
LOG.debug("Initializing S3 underFs with properties: " + props.getProperties());
mClient = new RestS3Service(awsCredentials, null, null, props);
mBucketPrefix = Constants.HEADER_S3N + mBucketName + PATH_SEPARATOR;
}
/**
* Validates the path, verifying that it contains the {@link Constants#HEADER} or {@link
* Constants#HEADER_FT} and a hostname:port specified.
*
* @param path the path to be verified
* @param tachyonConf the instance of {@link tachyon.conf.TachyonConf} to be used
* @return the verified path in a form like tachyon://host:port/dir. If only the "/dir" or "dir"
* part is provided, the host and port are retrieved from property, tachyon.master.hostname
* and tachyon.master.port, respectively.
* @throws IOException if the given path is not valid
*/
public static String validatePath(String path, TachyonConf tachyonConf) throws IOException {
if (path.startsWith(Constants.HEADER) || path.startsWith(Constants.HEADER_FT)) {
if (!path.contains(":")) {
throw new IOException(
"Invalid Path: "
+ path
+ ". Use "
+ Constants.HEADER
+ "host:port/ ,"
+ Constants.HEADER_FT
+ "host:port/"
+ " , or /file");
} else {
return path;
}
} else {
String hostname = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, tachyonConf);
int port = tachyonConf.getInt(Constants.MASTER_PORT);
if (tachyonConf.getBoolean(Constants.ZOOKEEPER_ENABLED)) {
return PathUtils.concatPath(Constants.HEADER_FT + hostname + ":" + port, path);
}
return PathUtils.concatPath(Constants.HEADER + hostname + ":" + port, path);
}
}
/**
* Creates a new instance of {@link BlockWorker}.
*
* @throws ConnectionFailedException if network connection failed
* @throws IOException for other exceptions
*/
public BlockWorker() throws IOException, ConnectionFailedException {
super(
Executors.newFixedThreadPool(
4, ThreadFactoryUtils.build("block-worker-heartbeat-%d", true)));
mTachyonConf = WorkerContext.getConf();
mStartTimeMs = System.currentTimeMillis();
// Setup MasterClientBase
mBlockMasterClient =
new BlockMasterClient(
NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, mTachyonConf),
mTachyonConf);
mFileSystemMasterClient =
new FileSystemMasterClient(
NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, mTachyonConf),
mTachyonConf);
// Set up BlockDataManager
WorkerSource workerSource = new WorkerSource();
mBlockDataManager =
new BlockDataManager(
workerSource, mBlockMasterClient, mFileSystemMasterClient, new TieredBlockStore());
// Setup metrics collection
mWorkerMetricsSystem = new MetricsSystem("worker", mTachyonConf);
workerSource.registerGauges(mBlockDataManager);
mWorkerMetricsSystem.registerSource(workerSource);
// Setup DataServer
mDataServer =
DataServer.Factory.create(
NetworkAddressUtils.getBindAddress(ServiceType.WORKER_DATA, mTachyonConf),
mBlockDataManager,
mTachyonConf);
// Reset data server port
mTachyonConf.set(Constants.WORKER_DATA_PORT, Integer.toString(mDataServer.getPort()));
// Setup RPC Server
mServiceHandler = new BlockWorkerClientServiceHandler(mBlockDataManager);
mThriftServerSocket = createThriftServerSocket();
mPort = NetworkAddressUtils.getThriftPort(mThriftServerSocket);
// Reset worker RPC port
mTachyonConf.set(Constants.WORKER_RPC_PORT, Integer.toString(mPort));
mThriftServer = createThriftServer();
// Setup web server
mWebServer =
new WorkerUIWebServer(
ServiceType.WORKER_WEB,
NetworkAddressUtils.getBindAddress(ServiceType.WORKER_WEB, mTachyonConf),
mBlockDataManager,
NetworkAddressUtils.getConnectAddress(ServiceType.WORKER_RPC, mTachyonConf),
mStartTimeMs,
mTachyonConf);
mWorkerMetricsSystem.start();
// Add the metrics servlet to the web server, this must be done after the metrics system starts
mWebServer.addHandler(mWorkerMetricsSystem.getServletHandler());
mWebServer.startWebServer();
int webPort = mWebServer.getLocalPort();
// Get the worker id
mWorkerNetAddress =
new NetAddress(
NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, mTachyonConf),
mPort,
mDataServer.getPort(),
webPort);
WorkerIdRegistry.registerWithBlockMaster(mBlockMasterClient, mWorkerNetAddress);
mBlockMasterSync =
new BlockMasterSync(mBlockDataManager, mWorkerNetAddress, mBlockMasterClient);
// Setup PinListSyncer
mPinListSync = new PinListSync(mBlockDataManager, mFileSystemMasterClient);
// Setup session cleaner
mSessionCleanerThread = new SessionCleaner(mBlockDataManager);
// Setup space reserver
if (mTachyonConf.getBoolean(Constants.WORKER_TIERED_STORE_RESERVER_ENABLED)) {
mSpaceReserver = new SpaceReserver(mBlockDataManager);
}
}