private void initStorageTier() throws AlreadyExistsException, IOException, OutOfSpaceException { String workerDataFolder = WorkerContext.getConf().get(Constants.WORKER_DATA_FOLDER, Constants.DEFAULT_DATA_FOLDER); String tierDirPathConf = String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, mTierLevel); String[] dirPaths = WorkerContext.getConf().get(tierDirPathConf, "/mnt/ramdisk").split(","); // Add the worker data folder path after each storage directory, the final path will be like // /mnt/ramdisk/tachyonworker for (int i = 0; i < dirPaths.length; i++) { dirPaths[i] = PathUtils.concatPath(dirPaths[i].trim(), workerDataFolder); } String tierDirCapacityConf = String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_QUOTA_FORMAT, mTierLevel); String[] dirQuotas = WorkerContext.getConf().get(tierDirCapacityConf, "0").split(","); mDirs = new ArrayList<StorageDir>(dirPaths.length); long totalCapacity = 0; for (int i = 0; i < dirPaths.length; i++) { int index = i >= dirQuotas.length ? dirQuotas.length - 1 : i; long capacity = FormatUtils.parseSpaceSize(dirQuotas[index]); totalCapacity += capacity; mDirs.add(StorageDir.newStorageDir(this, i, capacity, dirPaths[i])); } mCapacityBytes = totalCapacity; }
/** * Runs the block worker. The thread calling this will be blocked until the thrift server shuts * down. */ public void process() { getExecutorService() .submit( new HeartbeatThread( HeartbeatContext.WORKER_BLOCK_SYNC, mBlockMasterSync, WorkerContext.getConf().getInt(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS))); // Start the pinlist syncer to perform the periodical fetching getExecutorService() .submit( new HeartbeatThread( HeartbeatContext.WORKER_PIN_LIST_SYNC, mPinListSync, WorkerContext.getConf().getInt(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS))); // Start the session cleanup checker to perform the periodical checking getExecutorService().submit(mSessionCleanerThread); // Start the space reserver if (mSpaceReserver != null) { getExecutorService().submit(mSpaceReserver); } mThriftServer.serve(); }
@Test public void reserveTest() throws Exception { // Reserve on top tier long blockId = 100; BlockStoreLocation tier0 = BlockStoreLocation.anyDirInTier(StorageLevelAlias.MEM.getValue()); for (int i = 0; i < 3; i++) { TieredBlockStoreTestUtils.cache(SESSION_ID, blockId++, BLOCK_SIZE, mBlockStore, tier0); } CommonUtils.sleepMs( WorkerContext.getConf().getLong(Constants.WORKER_SPACE_RESERVER_INTERVAL_MS)); BlockStoreMeta storeMeta = mBlockStore.getBlockStoreMeta(); Assert.assertEquals(3 * BLOCK_SIZE, storeMeta.getUsedBytes()); List<Long> usedBytesOnTiers = storeMeta.getUsedBytesOnTiers(); Assert.assertEquals( 2 * BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.MEM.getValue() - 1)); Assert.assertEquals( BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.HDD.getValue() - 1)); // Reserve on under tier for (int i = 0; i < 7; i++) { TieredBlockStoreTestUtils.cache(SESSION_ID, blockId++, BLOCK_SIZE, mBlockStore, tier0); } CommonUtils.sleepMs( WorkerContext.getConf().getLong(Constants.WORKER_SPACE_RESERVER_INTERVAL_MS)); storeMeta = mBlockStore.getBlockStoreMeta(); Assert.assertEquals(9 * BLOCK_SIZE, storeMeta.getUsedBytes()); usedBytesOnTiers = storeMeta.getUsedBytesOnTiers(); Assert.assertEquals( 2 * BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.MEM.getValue() - 1)); Assert.assertEquals( 7 * BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.HDD.getValue() - 1)); }
/** * Tests that allocation happens when the RAM, SSD and HDD size is lower than the default size. * * @throws Exception if a block cannot be allocated */ @Test public void shouldAllocateTest() throws Exception { TachyonConf conf = WorkerContext.getConf(); for (String strategyName : mStrategies) { conf.set(Constants.WORKER_ALLOCATOR_CLASS, strategyName); resetManagerView(); Allocator tierAllocator = Allocator.Factory.create(conf, getManagerView()); for (int i = 0; i < DEFAULT_RAM_NUM; i++) { assertTempBlockMeta(tierAllocator, mAnyDirInTierLoc1, DEFAULT_RAM_SIZE - 1, true); } for (int i = 0; i < DEFAULT_SSD_NUM; i++) { assertTempBlockMeta(tierAllocator, mAnyDirInTierLoc2, DEFAULT_SSD_SIZE - 1, true); } for (int i = 0; i < DEFAULT_HDD_NUM; i++) { assertTempBlockMeta(tierAllocator, mAnyDirInTierLoc3, DEFAULT_HDD_SIZE - 1, true); } resetManagerView(); Allocator anyAllocator = Allocator.Factory.create(conf, getManagerView()); for (int i = 0; i < DEFAULT_RAM_NUM; i++) { assertTempBlockMeta(anyAllocator, mAnyTierLoc, DEFAULT_RAM_SIZE - 1, true); } for (int i = 0; i < DEFAULT_SSD_NUM; i++) { assertTempBlockMeta(anyAllocator, mAnyTierLoc, DEFAULT_SSD_SIZE - 1, true); } for (int i = 0; i < DEFAULT_HDD_NUM; i++) { assertTempBlockMeta(anyAllocator, mAnyTierLoc, DEFAULT_HDD_SIZE - 1, true); } } }
/** * Helper method to create a {@link org.apache.thrift.server.TThreadPoolServer} for handling * incoming RPC requests. * * @return a thrift server */ private TThreadPoolServer createThriftServer() { int minWorkerThreads = mTachyonConf.getInt(Constants.WORKER_WORKER_BLOCK_THREADS_MIN); int maxWorkerThreads = mTachyonConf.getInt(Constants.WORKER_WORKER_BLOCK_THREADS_MAX); BlockWorkerClientService.Processor<BlockWorkerClientServiceHandler> processor = new BlockWorkerClientService.Processor<BlockWorkerClientServiceHandler>(mServiceHandler); TTransportFactory tTransportFactory; try { tTransportFactory = AuthenticationUtils.getServerTransportFactory(mTachyonConf); } catch (IOException ioe) { throw Throwables.propagate(ioe); } Args args = new TThreadPoolServer.Args(mThriftServerSocket) .minWorkerThreads(minWorkerThreads) .maxWorkerThreads(maxWorkerThreads) .processor(processor) .transportFactory(tTransportFactory) .protocolFactory(new TBinaryProtocol.Factory(true, true)); if (WorkerContext.getConf().getBoolean(Constants.IN_TEST_MODE)) { args.stopTimeoutVal = 0; } else { args.stopTimeoutVal = Constants.THRIFT_STOP_TIMEOUT_SECONDS; } return new TThreadPoolServer(args); }
@Before public final void before() throws Exception { TachyonConf conf = WorkerContext.getConf(); conf.set(Constants.WORKER_DATA_SERVER, mDataServerClass); conf.set(Constants.WORKER_NETWORK_NETTY_FILE_TRANSFER_TYPE, mNettyTransferType); conf.set(Constants.USER_REMOTE_BLOCK_READER, mRemoteReaderClass); conf.set(Constants.USER_REMOTE_READ_BUFFER_SIZE_BYTE, "100"); mLocalTachyonCluster = new LocalTachyonCluster(Constants.GB, Constants.KB, Constants.GB); mLocalTachyonCluster.start(); mTachyonConf = mLocalTachyonCluster.getMasterTachyonConf(); mTfs = mLocalTachyonCluster.getClient(); mWriteTachyon = new OutStreamOptions.Builder(mTachyonConf) .setTachyonStorageType(TachyonStorageType.STORE) .setUnderStorageType(UnderStorageType.NO_PERSIST) .build(); mWriteUnderStore = new OutStreamOptions.Builder(mTachyonConf) .setTachyonStorageType(TachyonStorageType.NO_STORE) .setUnderStorageType(UnderStorageType.PERSIST) .build(); mReadCache = new InStreamOptions.Builder(mTachyonConf) .setTachyonStorageType(TachyonStorageType.STORE) .build(); mReadNoCache = new InStreamOptions.Builder(mTachyonConf) .setTachyonStorageType(TachyonStorageType.NO_STORE) .build(); }
private StorageTier(int tierLevel) { mTierLevel = tierLevel; String tierLevelAliasProp = String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_ALIAS_FORMAT, tierLevel); StorageLevelAlias alias = WorkerContext.getConf().getEnum(tierLevelAliasProp, StorageLevelAlias.class); mTierAlias = alias.getValue(); }
public SpaceReserver(BlockDataManager blockManager) { mBlockManager = blockManager; mStorageTierAssoc = new WorkerStorageTierAssoc(WorkerContext.getConf()); Map<String, Long> capOnTiers = blockManager.getStoreMeta().getCapacityBytesOnTiers(); long lastTierReservedBytes = 0; for (int ordinal = 0; ordinal < mStorageTierAssoc.size(); ordinal++) { String tierReservedSpaceProp = String.format(Constants.WORKER_TIERED_STORE_LEVEL_RESERVED_RATIO_FORMAT, ordinal); String tierAlias = mStorageTierAssoc.getAlias(ordinal); long reservedSpaceBytes = (long) (capOnTiers.get(tierAlias) * WorkerContext.getConf().getDouble(tierReservedSpaceProp)); mBytesToReserveOnTiers.put(tierAlias, reservedSpaceBytes + lastTierReservedBytes); lastTierReservedBytes += reservedSpaceBytes; } mCheckIntervalMs = WorkerContext.getConf().getInt(Constants.WORKER_TIERED_STORE_RESERVER_INTERVAL_MS); mRunning = true; }
public void stopTFS() throws Exception { mClientPool.close(); mWorker.stop(); if (LineageUtils.isLineageEnabled(WorkerContext.getConf())) { mLineageWorker.stop(); } for (int k = 0; k < mNumOfMasters; k++) { mMasters.get(k).stop(); } mCuratorServer.stop(); }
@Before public void before() throws Exception { WorkerFileSystemMasterClient workerFileSystemMasterClient = PowerMockito.mock(WorkerFileSystemMasterClient.class); WorkerSource workerSource = PowerMockito.mock(WorkerSource.class); WorkerBlockMasterClient blockMasterClient = PowerMockito.mock(WorkerBlockMasterClient.class); String baseDir = mTempFolder.newFolder().getAbsolutePath(); TieredBlockStoreTestUtils.setupTachyonConfWithMultiTier( baseDir, TIER_LEVEL, TIER_ALIAS, TIER_PATH, TIER_CAPACITY_BYTES, null); mBlockStore = new TieredBlockStore(); BlockDataManager blockDataManager = new BlockDataManager( workerSource, blockMasterClient, workerFileSystemMasterClient, mBlockStore); String reserveRatioProp = String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_RESERVED_RATIO_FORMAT, 0); WorkerContext.getConf().set(reserveRatioProp, "0.2"); reserveRatioProp = String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_RESERVED_RATIO_FORMAT, 1); WorkerContext.getConf().set(reserveRatioProp, "0.2"); mSpaceReserver = new SpaceReserver(blockDataManager); mExecutorService.submit(mSpaceReserver); }
/** * Creates a new instance of {@link FileSystemWorker}. * * @param blockDataManager a block data manager handle * @throws IOException if an I/O error occurs */ public FileSystemWorker(BlockDataManager blockDataManager) throws IOException { super( Executors.newFixedThreadPool( 3, ThreadFactoryUtils.build("file-system-worker-heartbeat-%d", true))); Preconditions.checkState(WorkerIdRegistry.getWorkerId() != 0, "Failed to register worker"); mTachyonConf = WorkerContext.getConf(); mFileDataManager = new FileDataManager(Preconditions.checkNotNull(blockDataManager)); // Setup MasterClientBase mFileSystemMasterWorkerClient = new FileSystemMasterClient( NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, mTachyonConf), mTachyonConf); }
/** * Sets up all dependencies before a test runs. * * @throws Exception if setting up the meta manager, the lock manager or the evictor fails */ @Before public final void before() throws Exception { File tempFolder = mTestFolder.newFolder(); mMetaManager = TieredBlockStoreTestUtils.defaultMetadataManager(tempFolder.getAbsolutePath()); mManagerView = new BlockMetadataManagerView( mMetaManager, Collections.<Long>emptySet(), Collections.<Long>emptySet()); TachyonConf conf = WorkerContext.getConf(); conf.set(Constants.WORKER_EVICTOR_CLASS, LRFUEvictor.class.getName()); conf.set(Constants.WORKER_ALLOCATOR_CLASS, MaxFreeAllocator.class.getName()); mAllocator = Allocator.Factory.create(conf, mManagerView); mStepFactor = conf.getDouble(Constants.WORKER_EVICTOR_LRFU_STEP_FACTOR); mAttenuationFactor = conf.getDouble(Constants.WORKER_EVICTOR_LRFU_ATTENUATION_FACTOR); mEvictor = Evictor.Factory.create(conf, mManagerView, mAllocator); }
public void start() throws IOException { int maxLevel = 1; mTachyonHome = File.createTempFile("Tachyon", "U" + System.currentTimeMillis()).getAbsolutePath(); mWorkerDataFolder = "/datastore"; mHostname = NetworkAddressUtils.getLocalHostName(100); mMasterConf = MasterContext.getConf(); mMasterConf.set(Constants.IN_TEST_MODE, "true"); mMasterConf.set(Constants.TACHYON_HOME, mTachyonHome); mMasterConf.set(Constants.USE_ZOOKEEPER, "true"); mMasterConf.set(Constants.MASTER_HOSTNAME, mHostname); mMasterConf.set(Constants.MASTER_BIND_HOST, mHostname); mMasterConf.set(Constants.MASTER_PORT, "0"); mMasterConf.set(Constants.MASTER_WEB_BIND_HOST, mHostname); mMasterConf.set(Constants.MASTER_WEB_PORT, "0"); mMasterConf.set(Constants.ZOOKEEPER_ADDRESS, mCuratorServer.getConnectString()); mMasterConf.set(Constants.ZOOKEEPER_ELECTION_PATH, "/election"); mMasterConf.set(Constants.ZOOKEEPER_LEADER_PATH, "/leader"); mMasterConf.set(Constants.USER_QUOTA_UNIT_BYTES, "10000"); mMasterConf.set(Constants.USER_DEFAULT_BLOCK_SIZE_BYTE, Integer.toString(mUserBlockSize)); // Since tests are always running on a single host keep the resolution timeout low as otherwise // people running with strange network configurations will see very slow tests mMasterConf.set(Constants.HOST_RESOLUTION_TIMEOUT_MS, "250"); // Disable hdfs client caching to avoid file system close() affecting other clients System.setProperty("fs.hdfs.impl.disable.cache", "true"); // re-build the dir to set permission to 777 deleteDir(mTachyonHome); mkdir(mTachyonHome); for (int k = 0; k < mNumOfMasters; k++) { final LocalTachyonMaster master = LocalTachyonMaster.create(mTachyonHome); master.start(); LOG.info( "master NO." + k + " started, isServing: " + master.isServing() + ", address: " + master.getAddress()); mMasters.add(master); // Each master should generate a new port for binding mMasterConf.set(Constants.MASTER_PORT, "0"); } // Create the directories for the data and workers after LocalTachyonMaster construction, // because LocalTachyonMaster sets the UNDERFS_DATA_FOLDER and UNDERFS_WORKERS_FOLDER. mkdir(mMasterConf.get(Constants.UNDERFS_DATA_FOLDER)); mkdir(mMasterConf.get(Constants.UNDERFS_WORKERS_FOLDER)); LOG.info("all " + mNumOfMasters + " masters started."); LOG.info("waiting for a leader."); boolean hasLeader = false; while (!hasLeader) { for (int i = 0; i < mMasters.size(); i++) { if (mMasters.get(i).isServing()) { LOG.info( "master NO." + i + " is selected as leader. address: " + mMasters.get(i).getAddress()); hasLeader = true; break; } } } // Use first master port mMasterConf.set(Constants.MASTER_PORT, getMasterPort() + ""); CommonUtils.sleepMs(10); mWorkerConf = WorkerContext.getConf(); mWorkerConf.merge(mMasterConf); mWorkerConf.set(Constants.WORKER_DATA_FOLDER, mWorkerDataFolder); mWorkerConf.set(Constants.WORKER_MEMORY_SIZE, mWorkerCapacityBytes + ""); mWorkerConf.set(Constants.WORKER_TO_MASTER_HEARTBEAT_INTERVAL_MS, 15 + ""); // Setup conf for worker mWorkerConf.set(Constants.WORKER_MAX_TIERED_STORAGE_LEVEL, Integer.toString(maxLevel)); mWorkerConf.set(String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_ALIAS_FORMAT, 0), "MEM"); mWorkerConf.set( String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, 0), mTachyonHome + "/ramdisk"); mWorkerConf.set( String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_QUOTA_FORMAT, 0), mWorkerCapacityBytes + ""); // Since tests are always running on a single host keep the resolution timeout low as otherwise // people running with strange network configurations will see very slow tests mWorkerConf.set(Constants.HOST_RESOLUTION_TIMEOUT_MS, "250"); for (int level = 1; level < maxLevel; level++) { String tierLevelDirPath = String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, level); String[] dirPaths = mWorkerConf.get(tierLevelDirPath).split(","); String newPath = ""; for (String dirPath : dirPaths) { newPath += mTachyonHome + dirPath + ","; } mWorkerConf.set( String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, level), newPath.substring(0, newPath.length() - 1)); } mWorkerConf.set(Constants.WORKER_BIND_HOST, mHostname); mWorkerConf.set(Constants.WORKER_PORT, "0"); mWorkerConf.set(Constants.WORKER_DATA_BIND_HOST, mHostname); mWorkerConf.set(Constants.WORKER_DATA_PORT, "0"); mWorkerConf.set(Constants.WORKER_WEB_BIND_HOST, mHostname); mWorkerConf.set(Constants.WORKER_WEB_PORT, "0"); mWorkerConf.set(Constants.WORKER_MIN_WORKER_THREADS, "1"); mWorkerConf.set(Constants.WORKER_MAX_WORKER_THREADS, "100"); // Perform immediate shutdown of data server. Graceful shutdown is unnecessary and slow mWorkerConf.set(Constants.WORKER_NETWORK_NETTY_SHUTDOWN_QUIET_PERIOD, Integer.toString(0)); mWorkerConf.set(Constants.WORKER_NETWORK_NETTY_SHUTDOWN_TIMEOUT, Integer.toString(0)); mWorker = new BlockWorker(); Runnable runWorker = new Runnable() { @Override public void run() { try { mWorker.process(); } catch (Exception e) { throw new RuntimeException(e + " \n Start Master Error \n" + e.getMessage(), e); } } }; mWorkerThread = new Thread(runWorker); mWorkerThread.start(); // The client context should reflect the updates to the conf. if (sReinitializer == null) { ClientContext.accessReinitializer(sReinitializerAccesser); } sReinitializer.reinitializeWithConf(mWorkerConf); }
/** * Creates a new instance of {@link BlockWorker}. * * @throws ConnectionFailedException if network connection failed * @throws IOException for other exceptions */ public BlockWorker() throws IOException, ConnectionFailedException { super( Executors.newFixedThreadPool( 4, ThreadFactoryUtils.build("block-worker-heartbeat-%d", true))); mTachyonConf = WorkerContext.getConf(); mStartTimeMs = System.currentTimeMillis(); // Setup MasterClientBase mBlockMasterClient = new BlockMasterClient( NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, mTachyonConf), mTachyonConf); mFileSystemMasterClient = new FileSystemMasterClient( NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC, mTachyonConf), mTachyonConf); // Set up BlockDataManager WorkerSource workerSource = new WorkerSource(); mBlockDataManager = new BlockDataManager( workerSource, mBlockMasterClient, mFileSystemMasterClient, new TieredBlockStore()); // Setup metrics collection mWorkerMetricsSystem = new MetricsSystem("worker", mTachyonConf); workerSource.registerGauges(mBlockDataManager); mWorkerMetricsSystem.registerSource(workerSource); // Setup DataServer mDataServer = DataServer.Factory.create( NetworkAddressUtils.getBindAddress(ServiceType.WORKER_DATA, mTachyonConf), mBlockDataManager, mTachyonConf); // Reset data server port mTachyonConf.set(Constants.WORKER_DATA_PORT, Integer.toString(mDataServer.getPort())); // Setup RPC Server mServiceHandler = new BlockWorkerClientServiceHandler(mBlockDataManager); mThriftServerSocket = createThriftServerSocket(); mPort = NetworkAddressUtils.getThriftPort(mThriftServerSocket); // Reset worker RPC port mTachyonConf.set(Constants.WORKER_RPC_PORT, Integer.toString(mPort)); mThriftServer = createThriftServer(); // Setup web server mWebServer = new WorkerUIWebServer( ServiceType.WORKER_WEB, NetworkAddressUtils.getBindAddress(ServiceType.WORKER_WEB, mTachyonConf), mBlockDataManager, NetworkAddressUtils.getConnectAddress(ServiceType.WORKER_RPC, mTachyonConf), mStartTimeMs, mTachyonConf); mWorkerMetricsSystem.start(); // Add the metrics servlet to the web server, this must be done after the metrics system starts mWebServer.addHandler(mWorkerMetricsSystem.getServletHandler()); mWebServer.startWebServer(); int webPort = mWebServer.getLocalPort(); // Get the worker id mWorkerNetAddress = new NetAddress( NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, mTachyonConf), mPort, mDataServer.getPort(), webPort); WorkerIdRegistry.registerWithBlockMaster(mBlockMasterClient, mWorkerNetAddress); mBlockMasterSync = new BlockMasterSync(mBlockDataManager, mWorkerNetAddress, mBlockMasterClient); // Setup PinListSyncer mPinListSync = new PinListSync(mBlockDataManager, mFileSystemMasterClient); // Setup session cleaner mSessionCleanerThread = new SessionCleaner(mBlockDataManager); // Setup space reserver if (mTachyonConf.getBoolean(Constants.WORKER_TIERED_STORE_RESERVER_ENABLED)) { mSpaceReserver = new SpaceReserver(mBlockDataManager); } }