/**
* Creates a <code>TachyonFS</code> handler for the given Tachyon URI and configuration.
*
* @param tachyonURI a Tachyon URI to indicate master address. e.g., tachyon://localhost:19998,
* tachyon://localhost:19998/ab/c.txt
* @param tachyonConf The TachyonConf instance.
* @return the corresponding TachyonFS handler
*/
public static synchronized TachyonFS get(final TachyonURI tachyonURI, TachyonConf tachyonConf) {
Preconditions.checkArgument(tachyonConf != null, "TachyonConf cannot be null.");
Preconditions.checkArgument(
tachyonURI != null,
"Tachyon URI cannot be null. Use "
+ Constants.HEADER
+ "host:port/ ,"
+ Constants.HEADER_FT
+ "host:port/.");
String scheme = tachyonURI.getScheme();
Preconditions.checkNotNull(
scheme,
"Tachyon scheme cannot be null. Use "
+ Constants.SCHEME
+ " or "
+ Constants.SCHEME_FT
+ ".");
Preconditions.checkNotNull(tachyonURI.getHost(), "Tachyon hostname cannot be null.");
Preconditions.checkState(tachyonURI.getPort() != -1, "Tachyon URI must have a port number.");
Preconditions.checkState(
(Constants.SCHEME.equals(scheme) || Constants.SCHEME_FT.equals(scheme)),
"Tachyon scheme must be either " + Constants.SCHEME + " or " + Constants.SCHEME_FT + ".");
boolean useZookeeper = scheme.equals(Constants.SCHEME_FT);
tachyonConf.set(Constants.USE_ZOOKEEPER, Boolean.toString(useZookeeper));
tachyonConf.set(Constants.MASTER_HOSTNAME, tachyonURI.getHost());
tachyonConf.set(Constants.MASTER_PORT, Integer.toString(tachyonURI.getPort()));
return get(tachyonConf);
}
@Before
public final void before() throws Exception {
TachyonConf conf = WorkerContext.getConf();
conf.set(Constants.WORKER_DATA_SERVER, mDataServerClass);
conf.set(Constants.WORKER_NETWORK_NETTY_FILE_TRANSFER_TYPE, mNettyTransferType);
conf.set(Constants.USER_REMOTE_BLOCK_READER, mRemoteReaderClass);
conf.set(Constants.USER_REMOTE_READ_BUFFER_SIZE_BYTE, "100");
mLocalTachyonCluster = new LocalTachyonCluster(Constants.GB, Constants.KB, Constants.GB);
mLocalTachyonCluster.start();
mTachyonConf = mLocalTachyonCluster.getMasterTachyonConf();
mTfs = mLocalTachyonCluster.getClient();
mWriteTachyon =
new OutStreamOptions.Builder(mTachyonConf)
.setTachyonStorageType(TachyonStorageType.STORE)
.setUnderStorageType(UnderStorageType.NO_PERSIST)
.build();
mWriteUnderStore =
new OutStreamOptions.Builder(mTachyonConf)
.setTachyonStorageType(TachyonStorageType.NO_STORE)
.setUnderStorageType(UnderStorageType.PERSIST)
.build();
mReadCache =
new InStreamOptions.Builder(mTachyonConf)
.setTachyonStorageType(TachyonStorageType.STORE)
.build();
mReadNoCache =
new InStreamOptions.Builder(mTachyonConf)
.setTachyonStorageType(TachyonStorageType.NO_STORE)
.build();
}
/**
* Creates a <code>TachyonFS</code> handler for the given hostname, port, and Zookeeper mode.
*
* @param masterHost master host details
* @param masterPort port master listens on
* @param zkMode use zookeeper
* @return the corresponding TachyonFS handler
*/
public static synchronized TachyonFS get(String masterHost, int masterPort, boolean zkMode) {
TachyonConf tachyonConf = new TachyonConf();
tachyonConf.set(Constants.MASTER_HOSTNAME, masterHost);
tachyonConf.set(Constants.MASTER_PORT, Integer.toString(masterPort));
tachyonConf.set(Constants.USE_ZOOKEEPER, Boolean.toString(zkMode));
return get(tachyonConf);
}
/**
* Sets up all dependencies before a test runs.
*
* @throws Exception if setting up the meta manager, the lock manager or the evictor fails
*/
@Before
public final void before() throws Exception {
File tempFolder = mTestFolder.newFolder();
mMetaManager = TieredBlockStoreTestUtils.defaultMetadataManager(tempFolder.getAbsolutePath());
mManagerView =
new BlockMetadataManagerView(
mMetaManager, Collections.<Long>emptySet(), Collections.<Long>emptySet());
TachyonConf conf = WorkerContext.getConf();
conf.set(Constants.WORKER_EVICTOR_CLASS, LRFUEvictor.class.getName());
conf.set(Constants.WORKER_ALLOCATOR_CLASS, MaxFreeAllocator.class.getName());
mAllocator = Allocator.Factory.create(conf, mManagerView);
mStepFactor = conf.getDouble(Constants.WORKER_EVICTOR_LRFU_STEP_FACTOR);
mAttenuationFactor = conf.getDouble(Constants.WORKER_EVICTOR_LRFU_ATTENUATION_FACTOR);
mEvictor = Evictor.Factory.create(conf, mManagerView, mAllocator);
}
@Before
public final void before() throws Exception {
mLocalTachyonCluster =
new LocalTachyonCluster(MEM_CAPACITY_BYTES, USER_QUOTA_UNIT_BYTES, MEM_CAPACITY_BYTES / 2);
TachyonConf conf = new TachyonConf();
conf.set(Constants.WORKER_TO_MASTER_HEARTBEAT_INTERVAL_MS, HEARTBEAT_INTERVAL_MS + "");
conf.set(Constants.WORKER_MAX_TIERED_STORAGE_LEVEL, "2");
conf.set(String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_ALIAS_FORMAT, 1), "HDD");
conf.set(String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, 1), "/disk1");
conf.set(
String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_QUOTA_FORMAT, 1),
DISK_CAPACITY_BYTES + "");
mLocalTachyonCluster.start(conf);
mTFS = mLocalTachyonCluster.getClient();
}
/**
* Tests that allocation happens when the RAM, SSD and HDD size is lower than the default size.
*
* @throws Exception if a block cannot be allocated
*/
@Test
public void shouldAllocateTest() throws Exception {
TachyonConf conf = WorkerContext.getConf();
for (String strategyName : mStrategies) {
conf.set(Constants.WORKER_ALLOCATOR_CLASS, strategyName);
resetManagerView();
Allocator tierAllocator = Allocator.Factory.create(conf, getManagerView());
for (int i = 0; i < DEFAULT_RAM_NUM; i++) {
assertTempBlockMeta(tierAllocator, mAnyDirInTierLoc1, DEFAULT_RAM_SIZE - 1, true);
}
for (int i = 0; i < DEFAULT_SSD_NUM; i++) {
assertTempBlockMeta(tierAllocator, mAnyDirInTierLoc2, DEFAULT_SSD_SIZE - 1, true);
}
for (int i = 0; i < DEFAULT_HDD_NUM; i++) {
assertTempBlockMeta(tierAllocator, mAnyDirInTierLoc3, DEFAULT_HDD_SIZE - 1, true);
}
resetManagerView();
Allocator anyAllocator = Allocator.Factory.create(conf, getManagerView());
for (int i = 0; i < DEFAULT_RAM_NUM; i++) {
assertTempBlockMeta(anyAllocator, mAnyTierLoc, DEFAULT_RAM_SIZE - 1, true);
}
for (int i = 0; i < DEFAULT_SSD_NUM; i++) {
assertTempBlockMeta(anyAllocator, mAnyTierLoc, DEFAULT_SSD_SIZE - 1, true);
}
for (int i = 0; i < DEFAULT_HDD_NUM; i++) {
assertTempBlockMeta(anyAllocator, mAnyTierLoc, DEFAULT_HDD_SIZE - 1, true);
}
}
}
/**
* This method adds Aliyun credentials from system properties to the Tachyon Conf if they are not
* already present.
*
* @param tachyonConf the conf to check and add credentials to
* @return true if both access and secret key are present, false otherwise
*/
private boolean addAndCheckOSSCredentials(TachyonConf tachyonConf) {
String accessKeyConf = Constants.OSS_ACCESS_KEY;
if (System.getProperty(accessKeyConf) != null && tachyonConf.get(accessKeyConf) == null) {
tachyonConf.set(accessKeyConf, System.getProperty(accessKeyConf));
}
String secretKeyConf = Constants.OSS_SECRET_KEY;
if (System.getProperty(secretKeyConf) != null && tachyonConf.get(secretKeyConf) == null) {
tachyonConf.set(secretKeyConf, System.getProperty(secretKeyConf));
}
String endPointConf = Constants.OSS_ENDPOINT_KEY;
if (System.getProperty(endPointConf) != null && tachyonConf.get(endPointConf) == null) {
tachyonConf.set(endPointConf, System.getProperty(endPointConf));
}
return tachyonConf.get(accessKeyConf) != null
&& tachyonConf.get(secretKeyConf) != null
&& tachyonConf.get(endPointConf) != null;
}
/**
* Creates a new local tachyon master with a isolated home and port.
*
* @throws IOException when unable to do file operation or listen on port
* @return an instance of Tachyon master
*/
public static LocalTachyonMaster create() throws IOException {
final String tachyonHome = uniquePath();
TachyonConf tachyonConf = MasterContext.getConf();
UnderFileSystemUtils.deleteDir(tachyonHome, tachyonConf);
UnderFileSystemUtils.mkdirIfNotExists(tachyonHome, tachyonConf);
// Update Tachyon home in the passed TachyonConf instance.
tachyonConf.set(Constants.TACHYON_HOME, tachyonHome);
return new LocalTachyonMaster(tachyonHome);
}
/** Constructor of {@link TachyonWorker}. */
public TachyonWorker() {
try {
mStartTimeMs = System.currentTimeMillis();
mTachyonConf = WorkerContext.getConf();
mBlockWorker = new BlockWorker();
mFileSystemWorker = new FileSystemWorker(mBlockWorker);
mAdditionalWorkers = Lists.newArrayList();
List<? extends Worker> workers = Lists.newArrayList(mBlockWorker, mFileSystemWorker);
// Discover and register the available factories
// NOTE: ClassLoader is explicitly specified so we don't need to set ContextClassLoader
ServiceLoader<WorkerFactory> discoveredMasterFactories =
ServiceLoader.load(WorkerFactory.class, WorkerFactory.class.getClassLoader());
for (WorkerFactory factory : discoveredMasterFactories) {
Worker worker = factory.create(workers);
if (worker != null) {
mAdditionalWorkers.add(worker);
}
}
// Setup metrics collection system
mWorkerMetricsSystem = new MetricsSystem("worker", mTachyonConf);
WorkerSource workerSource = WorkerContext.getWorkerSource();
workerSource.registerGauges(mBlockWorker);
mWorkerMetricsSystem.registerSource(workerSource);
// Setup web server
mWebServer =
new WorkerUIWebServer(
ServiceType.WORKER_WEB,
NetworkAddressUtils.getBindAddress(ServiceType.WORKER_WEB, mTachyonConf),
mBlockWorker,
NetworkAddressUtils.getConnectAddress(ServiceType.WORKER_RPC, mTachyonConf),
mStartTimeMs,
mTachyonConf);
// Setup Thrift server
mThriftServerSocket = createThriftServerSocket();
mRPCPort = NetworkAddressUtils.getThriftPort(mThriftServerSocket);
// Reset worker RPC port based on assigned port number
mTachyonConf.set(Constants.WORKER_RPC_PORT, Integer.toString(mRPCPort));
mThriftServer = createThriftServer();
mWorkerAddress =
NetworkAddressUtils.getConnectAddress(
NetworkAddressUtils.ServiceType.WORKER_RPC, mTachyonConf);
} catch (Exception e) {
LOG.error("Failed to initialize {}", this.getClass().getName(), e);
System.exit(-1);
}
}
@Before
public final void before() throws Exception {
TachyonConf tachyonConf = MasterContext.getConf();
tachyonConf.set(Constants.USER_FILE_BUFFER_BYTES, String.valueOf(100));
mLocalTachyonCluster =
new LocalTachyonCluster(MEM_CAPACITY_BYTES, USER_QUOTA_UNIT_BYTES, Constants.GB);
mLocalTachyonCluster.start();
mTFS = mLocalTachyonCluster.getClient();
mWorkerConf = mLocalTachyonCluster.getWorkerTachyonConf();
mWorkerToMasterHeartbeatIntervalMs =
mWorkerConf.getInt(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS);
mSetPinned = new SetStateOptions.Builder(mWorkerConf).setPinned(true).build();
mSetUnpinned = new SetStateOptions.Builder(mWorkerConf).setPinned(false).build();
}
@Before
public final void before() throws Exception {
TachyonConf tachyonConf = new TachyonConf();
tachyonConf.set(Constants.USER_FILE_BUFFER_BYTES, String.valueOf(100));
System.setProperty(Constants.WORKER_DATA_SERVER, mDataServerClass);
System.setProperty(Constants.WORKER_NETTY_FILE_TRANSFER_TYPE, mNettyTransferType);
System.setProperty(Constants.USER_REMOTE_BLOCK_READER, mBlockReader);
mLocalTachyonCluster =
new LocalTachyonCluster(WORKER_CAPACITY_BYTES, USER_QUOTA_UNIT_BYTES, Constants.GB);
mLocalTachyonCluster.start(tachyonConf);
mWorkerTachyonConf = mLocalTachyonCluster.getWorkerTachyonConf();
mTFS = mLocalTachyonCluster.getClient();
mBlockMasterClient =
new BlockMasterClient(
new InetSocketAddress(
mLocalTachyonCluster.getMasterHostname(), mLocalTachyonCluster.getMasterPort()),
mExecutorService,
mWorkerTachyonConf);
}
private void getBindAddress(ServiceType service) throws Exception {
TachyonConf conf = new TachyonConf();
String localHostName = NetworkAddressUtils.getLocalHostName(conf);
InetSocketAddress workerAddress;
// all default
workerAddress = NetworkAddressUtils.getBindAddress(service, conf);
Assert.assertEquals(
new InetSocketAddress(NetworkAddressUtils.WILDCARD_ADDRESS, service.getDefaultPort()),
workerAddress);
// bind host only
conf.set(service.getBindHostKey(), "bind.host");
workerAddress = NetworkAddressUtils.getBindAddress(service, conf);
Assert.assertEquals(
new InetSocketAddress("bind.host", service.getDefaultPort()), workerAddress);
// connect host and bind host
conf.set(service.getHostNameKey(), "connect.host");
workerAddress = NetworkAddressUtils.getBindAddress(service, conf);
Assert.assertEquals(
new InetSocketAddress("bind.host", service.getDefaultPort()), workerAddress);
// wildcard connect host and bind host
conf.set(service.getHostNameKey(), NetworkAddressUtils.WILDCARD_ADDRESS);
workerAddress = NetworkAddressUtils.getBindAddress(service, conf);
Assert.assertEquals(
new InetSocketAddress("bind.host", service.getDefaultPort()), workerAddress);
// wildcard connect host and wildcard bind host
conf.set(service.getBindHostKey(), NetworkAddressUtils.WILDCARD_ADDRESS);
workerAddress = NetworkAddressUtils.getBindAddress(service, conf);
Assert.assertEquals(
new InetSocketAddress(NetworkAddressUtils.WILDCARD_ADDRESS, service.getDefaultPort()),
workerAddress);
// connect host and wildcard bind host
conf.set(service.getHostNameKey(), "connect.host");
workerAddress = NetworkAddressUtils.getBindAddress(service, conf);
Assert.assertEquals(
new InetSocketAddress(NetworkAddressUtils.WILDCARD_ADDRESS, service.getDefaultPort()),
workerAddress);
// connect host and wildcard bind host with port
switch (service) {
case MASTER_RPC:
conf.set(Constants.MASTER_RPC_PORT, "20000");
break;
case MASTER_WEB:
conf.set(Constants.MASTER_WEB_PORT, "20000");
break;
case WORKER_RPC:
conf.set(Constants.WORKER_RPC_PORT, "20000");
break;
case WORKER_DATA:
conf.set(Constants.WORKER_DATA_PORT, "20000");
break;
case WORKER_WEB:
conf.set(Constants.WORKER_WEB_PORT, "20000");
break;
default:
Assert.fail("Unrecognized service type: " + service.toString());
break;
}
workerAddress = NetworkAddressUtils.getBindAddress(service, conf);
Assert.assertEquals(
new InetSocketAddress(NetworkAddressUtils.WILDCARD_ADDRESS, 20000), workerAddress);
// connect host and bind host with port
conf.set(service.getBindHostKey(), "bind.host");
workerAddress = NetworkAddressUtils.getBindAddress(service, conf);
Assert.assertEquals(new InetSocketAddress("bind.host", 20000), workerAddress);
// empty connect host and bind host with port
conf.set(service.getHostNameKey(), "");
workerAddress = NetworkAddressUtils.getBindAddress(service, conf);
Assert.assertEquals(new InetSocketAddress("bind.host", 20000), workerAddress);
// empty connect host and wildcard bind host with port
conf.set(service.getBindHostKey(), NetworkAddressUtils.WILDCARD_ADDRESS);
workerAddress = NetworkAddressUtils.getBindAddress(service, conf);
Assert.assertEquals(
new InetSocketAddress(NetworkAddressUtils.WILDCARD_ADDRESS, 20000), workerAddress);
// empty connect host and empty bind host with port
conf.set(service.getBindHostKey(), "");
workerAddress = NetworkAddressUtils.getBindAddress(service, conf);
Assert.assertEquals(new InetSocketAddress(localHostName, 20000), workerAddress);
}
@Test
public void allocateBlockTest() throws Exception {
TachyonConf conf = new TachyonConf();
conf.set(Constants.WORKER_ALLOCATE_STRATEGY_CLASS, RoundRobinAllocator.class.getName());
mAllocator = Allocator.Factory.createAllocator(conf, mManagerView);
//
// idx | tier1 | tier2 | tier3
// 0 1000
// 0 ├───── 2000
// 1 └───── 2000
// 0 ├─── 3000
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyTierLoc, 500, true, 1, 0);
//
// idx | tier1 | tier2 | tier3
// 0 500 <--- alloc
// 0 ├───── 2000
// 1 └───── 2000
// 0 ├─── 3000
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyTierLoc, 600, true, 2, 0);
//
// idx | tier1 | tier2 | tier3
// 0 500
// 0 ├───── 1400 <--- alloc
// 1 └───── 2000
// 0 ├─── 3000
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyDirInTierLoc2, 700, true, 2, 1);
//
// idx | tier1 | tier2 | tier3
// 0 500
// 0 ├───── 1400
// 1 └───── 1300 <--- alloc
// 0 ├─── 3000
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyTierLoc, 700, true, 2, 0);
//
// idx | tier1 | tier2 | tier3
// 0 500
// 0 ├───── 700 <--- alloc
// 1 └───── 1300
// 0 ├─── 3000
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyTierLoc, 1000, true, 2, 1);
//
// idx | tier1 | tier2 | tier3
// 0 500
// 0 ├───── 700
// 1 └───── 300 <--- alloc
// 0 ├─── 3000
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyTierLoc, 700, true, 2, 0);
//
// idx | tier1 | tier2 | tier3
// 0 500
// 0 ├───── 0 <--- alloc
// 1 └───── 300
// 0 ├─── 3000
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyTierLoc, 700, true, 3, 0);
//
// idx | tier1 | tier2 | tier3
// 0 500
// 0 ├───── 0
// 1 └───── 300
// 0 ├─── 2300 <--- alloc
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyDirInTierLoc1, 200, true, 1, 0);
//
// idx | tier1 | tier2 | tier3
// 0 300 <--- alloc
// 0 ├───── 0
// 1 └───── 300
// 0 ├─── 2300
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyDirInTierLoc1, 100, true, 1, 0);
//
// idx | tier1 | tier2 | tier3
// 0 200 <--- alloc
// 0 ├───── 0
// 1 └───── 300
// 0 ├─── 2300
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyDirInTierLoc1, 700, false, 0, 0);
//
// idx | tier1 | tier2 | tier3
// 0 200
// 0 ├───── 0
// 1 └───── 300
// 0 ├─── 2300
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyDirInTierLoc2, 100, true, 2, 1);
//
// idx | tier1 | tier2 | tier3
// 0 200
// 0 ├───── 0
// 1 └───── 200 <--- alloc
// 0 ├─── 2300
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyDirInTierLoc2, 100, true, 2, 1);
//
// idx | tier1 | tier2 | tier3
// 0 200
// 0 ├───── 0
// 1 └───── 100 <--- alloc
// 0 ├─── 2300
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyDirInTierLoc2, 1500, false, 0, 0);
//
// idx | tier1 | tier2 | tier3
// 0 200
// 0 ├───── 0
// 1 └───── 100
// 0 ├─── 2300
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyDirInTierLoc3, 2000, true, 3, 1);
//
// idx | tier1 | tier2 | tier3
// 0 200
// 0 ├───── 0
// 1 └───── 100
// 0 ├─── 2300
// 1 ├─── 1000 <--- alloc
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyDirInTierLoc3, 3000, true, 3, 2);
//
// idx | tier1 | tier2 | tier3
// 0 200
// 0 ├───── 0
// 1 └───── 100
// 0 ├─── 2300
// 1 ├─── 1000
// 2 └─── 0 <--- alloc
//
assertTempBlockMeta(mAllocator, mAnyDirInTierLoc3, 500, true, 3, 0);
//
// idx | tier1 | tier2 | tier3
// 0 200
// 0 ├───── 0
// 1 └───── 100
// 0 ├─── 1800 <--- alloc
// 1 ├─── 1000
// 2 └─── 0
//
assertTempBlockMeta(mAllocator, mAnyDirInTierLoc3, 2000, false, 0, 0);
//
// idx | tier1 | tier2 | tier3
// 0 200
// 0 ├───── 0
// 1 └───── 100
// 0 ├─── 1800
// 1 ├─── 1000
// 2 └─── 0
//
// tier 3, dir 0, remain 0
assertTempBlockMeta(mAllocator, mAnyDirInTierLoc3, 300, true, 3, 1);
// idx | tier1 | tier2 | tier3
// 0 200
// 0 ├───── 0
// 1 └───── 100
// 0 ├─── 1800
// 1 ├─── 700 <--- alloc
// 2 └─── 0
//
}
private LocalTachyonMaster(final String tachyonHome) throws IOException {
mTachyonHome = tachyonHome;
TachyonConf tachyonConf = MasterContext.getConf();
mHostname = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, tachyonConf);
// To start the UFS either for integration or unit test. If it targets the unit test, UFS is
// setup over the local file system (see also {@link LocalFilesystemCluster} - under folder of
// "mTachyonHome/tachyon*". Otherwise, it starts some distributed file system cluster e.g.,
// miniDFSCluster (see also {@link tachyon.LocalMiniDFScluster} and setup the folder like
// "hdfs://xxx:xxx/tachyon*".
mUnderFSCluster = UnderFileSystemCluster.get(mTachyonHome + "/dfs", tachyonConf);
mUnderFSFolder = mUnderFSCluster.getUnderFilesystemAddress() + "/tachyon_underfs_folder";
// To setup the journalFolder under either local file system or distributed ufs like
// miniDFSCluster
mJournalFolder = mUnderFSCluster.getUnderFilesystemAddress() + "/journal";
UnderFileSystemUtils.mkdirIfNotExists(mJournalFolder, tachyonConf);
String[] masterServiceNames =
new String[] {
Constants.BLOCK_MASTER_SERVICE_NAME,
Constants.FILE_SYSTEM_MASTER_SERVICE_NAME,
Constants.RAW_TABLE_MASTER_SERVICE_NAME,
};
for (String masterServiceName : masterServiceNames) {
UnderFileSystemUtils.mkdirIfNotExists(
PathUtils.concatPath(mJournalFolder, masterServiceName), tachyonConf);
}
UnderFileSystemUtils.touch(
mJournalFolder + "/_format_" + System.currentTimeMillis(), tachyonConf);
tachyonConf.set(Constants.MASTER_JOURNAL_FOLDER, mJournalFolder);
tachyonConf.set(Constants.UNDERFS_ADDRESS, mUnderFSFolder);
tachyonConf.set(Constants.MASTER_MIN_WORKER_THREADS, "1");
tachyonConf.set(Constants.MASTER_MAX_WORKER_THREADS, "100");
// If tests fail to connect they should fail early rather than using the default ridiculously
// high retries
tachyonConf.set(Constants.MASTER_RETRY_COUNT, "3");
// Since tests are always running on a single host keep the resolution timeout low as otherwise
// people running with strange network configurations will see very slow tests
tachyonConf.set(Constants.HOST_RESOLUTION_TIMEOUT_MS, "250");
tachyonConf.set(Constants.WEB_THREAD_COUNT, "1");
tachyonConf.set(
Constants.WEB_RESOURCES,
PathUtils.concatPath(System.getProperty("user.dir"), "../servers/src/main/webapp"));
mTachyonMaster = TachyonMaster.Factory.createMaster();
// Reset the master port
tachyonConf.set(Constants.MASTER_PORT, Integer.toString(getRPCLocalPort()));
Runnable runMaster =
new Runnable() {
@Override
public void run() {
try {
mTachyonMaster.start();
} catch (Exception e) {
throw new RuntimeException(e + " \n Start Master Error \n" + e.getMessage(), e);
}
}
};
mMasterThread = new Thread(runMaster);
}
public void start() throws IOException {
int maxLevel = 1;
mTachyonHome =
File.createTempFile("Tachyon", "U" + System.currentTimeMillis()).getAbsolutePath();
mWorkerDataFolder = "/datastore";
mHostname = NetworkAddressUtils.getLocalHostName(100);
mMasterConf = MasterContext.getConf();
mMasterConf.set(Constants.IN_TEST_MODE, "true");
mMasterConf.set(Constants.TACHYON_HOME, mTachyonHome);
mMasterConf.set(Constants.USE_ZOOKEEPER, "true");
mMasterConf.set(Constants.MASTER_HOSTNAME, mHostname);
mMasterConf.set(Constants.MASTER_BIND_HOST, mHostname);
mMasterConf.set(Constants.MASTER_PORT, "0");
mMasterConf.set(Constants.MASTER_WEB_BIND_HOST, mHostname);
mMasterConf.set(Constants.MASTER_WEB_PORT, "0");
mMasterConf.set(Constants.ZOOKEEPER_ADDRESS, mCuratorServer.getConnectString());
mMasterConf.set(Constants.ZOOKEEPER_ELECTION_PATH, "/election");
mMasterConf.set(Constants.ZOOKEEPER_LEADER_PATH, "/leader");
mMasterConf.set(Constants.USER_QUOTA_UNIT_BYTES, "10000");
mMasterConf.set(Constants.USER_DEFAULT_BLOCK_SIZE_BYTE, Integer.toString(mUserBlockSize));
// Since tests are always running on a single host keep the resolution timeout low as otherwise
// people running with strange network configurations will see very slow tests
mMasterConf.set(Constants.HOST_RESOLUTION_TIMEOUT_MS, "250");
// Disable hdfs client caching to avoid file system close() affecting other clients
System.setProperty("fs.hdfs.impl.disable.cache", "true");
// re-build the dir to set permission to 777
deleteDir(mTachyonHome);
mkdir(mTachyonHome);
for (int k = 0; k < mNumOfMasters; k++) {
final LocalTachyonMaster master = LocalTachyonMaster.create(mTachyonHome);
master.start();
LOG.info(
"master NO."
+ k
+ " started, isServing: "
+ master.isServing()
+ ", address: "
+ master.getAddress());
mMasters.add(master);
// Each master should generate a new port for binding
mMasterConf.set(Constants.MASTER_PORT, "0");
}
// Create the directories for the data and workers after LocalTachyonMaster construction,
// because LocalTachyonMaster sets the UNDERFS_DATA_FOLDER and UNDERFS_WORKERS_FOLDER.
mkdir(mMasterConf.get(Constants.UNDERFS_DATA_FOLDER));
mkdir(mMasterConf.get(Constants.UNDERFS_WORKERS_FOLDER));
LOG.info("all " + mNumOfMasters + " masters started.");
LOG.info("waiting for a leader.");
boolean hasLeader = false;
while (!hasLeader) {
for (int i = 0; i < mMasters.size(); i++) {
if (mMasters.get(i).isServing()) {
LOG.info(
"master NO."
+ i
+ " is selected as leader. address: "
+ mMasters.get(i).getAddress());
hasLeader = true;
break;
}
}
}
// Use first master port
mMasterConf.set(Constants.MASTER_PORT, getMasterPort() + "");
CommonUtils.sleepMs(10);
mWorkerConf = WorkerContext.getConf();
mWorkerConf.merge(mMasterConf);
mWorkerConf.set(Constants.WORKER_DATA_FOLDER, mWorkerDataFolder);
mWorkerConf.set(Constants.WORKER_MEMORY_SIZE, mWorkerCapacityBytes + "");
mWorkerConf.set(Constants.WORKER_TO_MASTER_HEARTBEAT_INTERVAL_MS, 15 + "");
// Setup conf for worker
mWorkerConf.set(Constants.WORKER_MAX_TIERED_STORAGE_LEVEL, Integer.toString(maxLevel));
mWorkerConf.set(String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_ALIAS_FORMAT, 0), "MEM");
mWorkerConf.set(
String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, 0),
mTachyonHome + "/ramdisk");
mWorkerConf.set(
String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_QUOTA_FORMAT, 0),
mWorkerCapacityBytes + "");
// Since tests are always running on a single host keep the resolution timeout low as otherwise
// people running with strange network configurations will see very slow tests
mWorkerConf.set(Constants.HOST_RESOLUTION_TIMEOUT_MS, "250");
for (int level = 1; level < maxLevel; level++) {
String tierLevelDirPath =
String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, level);
String[] dirPaths = mWorkerConf.get(tierLevelDirPath).split(",");
String newPath = "";
for (String dirPath : dirPaths) {
newPath += mTachyonHome + dirPath + ",";
}
mWorkerConf.set(
String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, level),
newPath.substring(0, newPath.length() - 1));
}
mWorkerConf.set(Constants.WORKER_BIND_HOST, mHostname);
mWorkerConf.set(Constants.WORKER_PORT, "0");
mWorkerConf.set(Constants.WORKER_DATA_BIND_HOST, mHostname);
mWorkerConf.set(Constants.WORKER_DATA_PORT, "0");
mWorkerConf.set(Constants.WORKER_WEB_BIND_HOST, mHostname);
mWorkerConf.set(Constants.WORKER_WEB_PORT, "0");
mWorkerConf.set(Constants.WORKER_MIN_WORKER_THREADS, "1");
mWorkerConf.set(Constants.WORKER_MAX_WORKER_THREADS, "100");
// Perform immediate shutdown of data server. Graceful shutdown is unnecessary and slow
mWorkerConf.set(Constants.WORKER_NETWORK_NETTY_SHUTDOWN_QUIET_PERIOD, Integer.toString(0));
mWorkerConf.set(Constants.WORKER_NETWORK_NETTY_SHUTDOWN_TIMEOUT, Integer.toString(0));
mWorker = new BlockWorker();
Runnable runWorker =
new Runnable() {
@Override
public void run() {
try {
mWorker.process();
} catch (Exception e) {
throw new RuntimeException(e + " \n Start Master Error \n" + e.getMessage(), e);
}
}
};
mWorkerThread = new Thread(runWorker);
mWorkerThread.start();
// The client context should reflect the updates to the conf.
if (sReinitializer == null) {
ClientContext.accessReinitializer(sReinitializerAccesser);
}
sReinitializer.reinitializeWithConf(mWorkerConf);
}
/**
* Creates a new instance of {@link BlockWorker}.
*
* @throws ConnectionFailedException if network connection failed
* @throws IOException for other exceptions
*/
public BlockWorker() throws IOException, ConnectionFailedException {
super(
Executors.newFixedThreadPool(
4, ThreadFactoryUtils.build("block-worker-heartbeat-%d", true)));
mTachyonConf = WorkerContext.getConf();
mStartTimeMs = System.currentTimeMillis();
// Setup MasterClientBase
mBlockMasterClient =
new BlockMasterClient(
NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, mTachyonConf),
mTachyonConf);
mFileSystemMasterClient =
new FileSystemMasterClient(
NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, mTachyonConf),
mTachyonConf);
// Set up BlockDataManager
WorkerSource workerSource = new WorkerSource();
mBlockDataManager =
new BlockDataManager(
workerSource, mBlockMasterClient, mFileSystemMasterClient, new TieredBlockStore());
// Setup metrics collection
mWorkerMetricsSystem = new MetricsSystem("worker", mTachyonConf);
workerSource.registerGauges(mBlockDataManager);
mWorkerMetricsSystem.registerSource(workerSource);
// Setup DataServer
mDataServer =
DataServer.Factory.create(
NetworkAddressUtils.getBindAddress(ServiceType.WORKER_DATA, mTachyonConf),
mBlockDataManager,
mTachyonConf);
// Reset data server port
mTachyonConf.set(Constants.WORKER_DATA_PORT, Integer.toString(mDataServer.getPort()));
// Setup RPC Server
mServiceHandler = new BlockWorkerClientServiceHandler(mBlockDataManager);
mThriftServerSocket = createThriftServerSocket();
mPort = NetworkAddressUtils.getThriftPort(mThriftServerSocket);
// Reset worker RPC port
mTachyonConf.set(Constants.WORKER_RPC_PORT, Integer.toString(mPort));
mThriftServer = createThriftServer();
// Setup web server
mWebServer =
new WorkerUIWebServer(
ServiceType.WORKER_WEB,
NetworkAddressUtils.getBindAddress(ServiceType.WORKER_WEB, mTachyonConf),
mBlockDataManager,
NetworkAddressUtils.getConnectAddress(ServiceType.WORKER_RPC, mTachyonConf),
mStartTimeMs,
mTachyonConf);
mWorkerMetricsSystem.start();
// Add the metrics servlet to the web server, this must be done after the metrics system starts
mWebServer.addHandler(mWorkerMetricsSystem.getServletHandler());
mWebServer.startWebServer();
int webPort = mWebServer.getLocalPort();
// Get the worker id
mWorkerNetAddress =
new NetAddress(
NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, mTachyonConf),
mPort,
mDataServer.getPort(),
webPort);
WorkerIdRegistry.registerWithBlockMaster(mBlockMasterClient, mWorkerNetAddress);
mBlockMasterSync =
new BlockMasterSync(mBlockDataManager, mWorkerNetAddress, mBlockMasterClient);
// Setup PinListSyncer
mPinListSync = new PinListSync(mBlockDataManager, mFileSystemMasterClient);
// Setup session cleaner
mSessionCleanerThread = new SessionCleaner(mBlockDataManager);
// Setup space reserver
if (mTachyonConf.getBoolean(Constants.WORKER_TIERED_STORE_RESERVER_ENABLED)) {
mSpaceReserver = new SpaceReserver(mBlockDataManager);
}
}
private void getConnectAddress(ServiceType service) throws Exception {
TachyonConf conf = new TachyonConf();
String localHostName = NetworkAddressUtils.getLocalHostName(conf);
InetSocketAddress masterAddress;
// all default
masterAddress = NetworkAddressUtils.getConnectAddress(service, conf);
Assert.assertEquals(
new InetSocketAddress(localHostName, service.getDefaultPort()), masterAddress);
// bind host only
conf.set(service.getHostNameKey(), "");
conf.set(service.getBindHostKey(), "bind.host");
masterAddress = NetworkAddressUtils.getConnectAddress(service, conf);
Assert.assertEquals(
new InetSocketAddress("bind.host", service.getDefaultPort()), masterAddress);
// connect host and bind host
conf.set(service.getHostNameKey(), "connect.host");
masterAddress = NetworkAddressUtils.getConnectAddress(service, conf);
Assert.assertEquals(
new InetSocketAddress("connect.host", service.getDefaultPort()), masterAddress);
// wildcard connect host and bind host
conf.set(service.getHostNameKey(), NetworkAddressUtils.WILDCARD_ADDRESS);
masterAddress = NetworkAddressUtils.getConnectAddress(service, conf);
Assert.assertEquals(
new InetSocketAddress("bind.host", service.getDefaultPort()), masterAddress);
// wildcard connect host and wildcard bind host
conf.set(service.getBindHostKey(), NetworkAddressUtils.WILDCARD_ADDRESS);
masterAddress = NetworkAddressUtils.getConnectAddress(service, conf);
Assert.assertEquals(
new InetSocketAddress(localHostName, service.getDefaultPort()), masterAddress);
// connect host and wildcard bind host
conf.set(service.getHostNameKey(), "connect.host");
masterAddress = NetworkAddressUtils.getConnectAddress(service, conf);
Assert.assertEquals(
new InetSocketAddress("connect.host", service.getDefaultPort()), masterAddress);
// connect host and wildcard bind host with port
conf.set(service.getPortKey(), "10000");
masterAddress = NetworkAddressUtils.getConnectAddress(service, conf);
Assert.assertEquals(new InetSocketAddress("connect.host", 10000), masterAddress);
// connect host and bind host with port
conf.set(service.getBindHostKey(), "bind.host");
masterAddress = NetworkAddressUtils.getConnectAddress(service, conf);
Assert.assertEquals(new InetSocketAddress("connect.host", 10000), masterAddress);
// empty connect host and bind host with port
conf.set(service.getHostNameKey(), "");
masterAddress = NetworkAddressUtils.getConnectAddress(service, conf);
Assert.assertEquals(new InetSocketAddress("bind.host", 10000), masterAddress);
// empty connect host and wildcard bind host with port
conf.set(service.getBindHostKey(), NetworkAddressUtils.WILDCARD_ADDRESS);
masterAddress = NetworkAddressUtils.getConnectAddress(service, conf);
Assert.assertEquals(new InetSocketAddress(localHostName, 10000), masterAddress);
}
