/**
* Gets a stream to write data to a block. The stream can only be backed by Alluxio storage.
*
* @param blockId the block to write
* @param blockSize the standard block size to write, or -1 if the block already exists (and this
* stream is just storing the block in Alluxio again)
* @param address the address of the worker to write the block to, fails if the worker cannot
* serve the request
* @return a {@link BufferedBlockOutStream} which can be used to write data to the block in a
* streaming fashion
* @throws IOException if the block cannot be written
*/
public BufferedBlockOutStream getOutStream(long blockId, long blockSize, WorkerNetAddress address)
throws IOException {
if (blockSize == -1) {
try (CloseableResource<BlockMasterClient> blockMasterClientResource =
mContext.acquireMasterClientResource()) {
blockSize = blockMasterClientResource.get().getBlockInfo(blockId).getLength();
} catch (AlluxioException e) {
throw new IOException(e);
}
}
// No specified location to write to.
if (address == null) {
throw new RuntimeException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage());
}
// Location is local.
if (NetworkAddressUtils.getLocalHostName(ClientContext.getConf()).equals(address.getHost())) {
if (mContext.hasLocalWorker()) {
return new LocalBlockOutStream(blockId, blockSize);
} else {
throw new IOException(ExceptionMessage.NO_LOCAL_WORKER.getMessage("write"));
}
}
// Location is specified and it is remote.
return new RemoteBlockOutStream(blockId, blockSize, address);
}
/**
* Creates a new instance of {@link FileSystemWorker}.
*
* @param blockWorker the block worker handle
* @param workerId a reference to the id of this worker
* @throws IOException if an I/O error occurs
*/
public DefaultFileSystemWorker(BlockWorker blockWorker, AtomicReference<Long> workerId)
throws IOException {
super(
Executors.newFixedThreadPool(
3, ThreadFactoryUtils.build("file-system-worker-heartbeat-%d", true)));
mWorkerId = workerId;
mSessions = new Sessions();
UnderFileSystem ufs = UnderFileSystem.get(Configuration.get(PropertyKey.UNDERFS_ADDRESS));
mFileDataManager =
new FileDataManager(
Preconditions.checkNotNull(blockWorker),
ufs,
RateLimiter.create(Configuration.getBytes(PropertyKey.WORKER_FILE_PERSIST_RATE_LIMIT)));
mUnderFileSystemManager = new UnderFileSystemManager();
// Setup AbstractMasterClient
mFileSystemMasterWorkerClient =
new FileSystemMasterClient(NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC));
// Setup session cleaner
mSessionCleaner =
new SessionCleaner(
new SessionCleanupCallback() {
/** Cleans up after sessions, to prevent zombie sessions holding ufs resources. */
@Override
public void cleanupSessions() {
for (long session : mSessions.getTimedOutSessions()) {
mSessions.removeSession(session);
mUnderFileSystemManager.cleanupSession(session);
}
}
});
mServiceHandler = new FileSystemWorkerClientServiceHandler(this);
}
private LocalAlluxioMaster() throws IOException {
mHostname = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC);
mJournalFolder = Configuration.get(Constants.MASTER_JOURNAL_FOLDER);
mAlluxioMaster = AlluxioMaster.Factory.create();
Whitebox.setInternalState(AlluxioMaster.class, "sAlluxioMaster", mAlluxioMaster);
// Reset the master port
Configuration.set(Constants.MASTER_RPC_PORT, Integer.toString(getRPCLocalPort()));
Runnable runMaster =
new Runnable() {
@Override
public void run() {
try {
mAlluxioMaster.start();
} catch (Exception e) {
throw new RuntimeException(e + " \n Start Master Error \n" + e.getMessage(), e);
}
}
};
mMasterThread = new Thread(runMaster);
}
/**
* Initializes {@link #mLocalBlockWorkerClientPoolMap}. This method is supposed be called in a
* lazy manner.
*/
private synchronized void initializeLocalBlockWorkerClientPool() {
if (!mLocalBlockWorkerClientPoolInitialized) {
for (WorkerNetAddress localWorkerAddress :
getWorkerAddresses(NetworkAddressUtils.getLocalHostName())) {
mLocalBlockWorkerClientPoolMap.put(
localWorkerAddress, new BlockWorkerClientPool(localWorkerAddress));
}
mLocalBlockWorkerClientPoolInitialized = true;
}
}
/**
* Obtains a client for a remote based on the given network address. Illegal argument exception is
* thrown if the hostname is the local hostname. Runtime exception is thrown if the client cannot
* be created with a connection to the hostname.
*
* @param address the address of the worker
* @return a worker client with a connection to the specified hostname
*/
private BlockWorkerClient acquireRemoteWorkerClient(WorkerNetAddress address) {
// If we couldn't find a worker, crash.
if (address == null) {
// TODO(calvin): Better exception usage.
throw new RuntimeException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage());
}
Preconditions.checkArgument(
!address.getHost().equals(NetworkAddressUtils.getLocalHostName()),
PreconditionMessage.REMOTE_CLIENT_BUT_LOCAL_HOSTNAME);
long clientId = IdUtils.getRandomNonNegativeLong();
return new RetryHandlingBlockWorkerClient(
address, ClientContext.getBlockClientExecutorService(), clientId, false);
}
/**
* Obtains a client for a worker with the given address.
*
* @param address the address of the worker to get a client to
* @return a {@link BlockWorkerClient} connected to the worker with the given hostname
* @throws IOException if no Alluxio worker is available for the given hostname
*/
public BlockWorkerClient acquireWorkerClient(WorkerNetAddress address) throws IOException {
BlockWorkerClient client;
if (address == null) {
throw new RuntimeException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage());
}
if (address.getHost().equals(NetworkAddressUtils.getLocalHostName())) {
client = acquireLocalWorkerClient(address);
if (client == null) {
throw new IOException(ExceptionMessage.NO_WORKER_AVAILABLE_ON_ADDRESS.getMessage(address));
}
} else {
client = acquireRemoteWorkerClient(address);
}
return client;
}
/** Tests the default properties for the master. */
@Test
public void masterDefaultTest() {
String alluxioHome = sDefaultConfiguration.get(Constants.HOME);
Assert.assertNotNull(alluxioHome);
Assert.assertEquals("/mnt/alluxio_default_home", alluxioHome);
String value = sDefaultConfiguration.get(Constants.MASTER_JOURNAL_FOLDER);
Assert.assertNotNull(value);
Assert.assertEquals(alluxioHome + "/journal/", value);
value = sDefaultConfiguration.get(Constants.MASTER_HOSTNAME);
Assert.assertNotNull(value);
Assert.assertEquals(NetworkAddressUtils.getLocalHostName(100), value);
value = sDefaultConfiguration.get(Constants.MASTER_FORMAT_FILE_PREFIX);
Assert.assertNotNull(value);
Assert.assertEquals(Constants.FORMAT_FILE_PREFIX, value);
value = sDefaultConfiguration.get(Constants.MASTER_ADDRESS);
Assert.assertNotNull(value);
value = sDefaultConfiguration.get(Constants.MASTER_BIND_HOST);
Assert.assertNotNull(value);
Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);
int intValue = sDefaultConfiguration.getInt(Constants.MASTER_RPC_PORT);
Assert.assertEquals(19998, intValue);
value = sDefaultConfiguration.get(Constants.MASTER_WEB_BIND_HOST);
Assert.assertNotNull(value);
Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);
intValue = sDefaultConfiguration.getInt(Constants.MASTER_WEB_PORT);
Assert.assertEquals(19999, intValue);
intValue = sDefaultConfiguration.getInt(Constants.WEB_THREAD_COUNT);
Assert.assertEquals(1, intValue);
intValue = sDefaultConfiguration.getInt(Constants.MASTER_HEARTBEAT_INTERVAL_MS);
Assert.assertEquals(Constants.SECOND_MS, intValue);
intValue = sDefaultConfiguration.getInt(Constants.MASTER_WORKER_THREADS_MIN);
Assert.assertEquals(Runtime.getRuntime().availableProcessors(), intValue);
intValue = sDefaultConfiguration.getInt(Constants.MASTER_WORKER_TIMEOUT_MS);
Assert.assertEquals(10 * Constants.SECOND_MS, intValue);
}
/**
* Gets a stream to read the data of a block. The stream is backed by Alluxio storage.
*
* @param blockId the block to read from
* @return a {@link BlockInStream} which can be used to read the data in a streaming fashion
* @throws IOException if the block does not exist
*/
public BufferedBlockInStream getInStream(long blockId) throws IOException {
BlockInfo blockInfo;
try (CloseableResource<BlockMasterClient> masterClientResource =
mContext.acquireMasterClientResource()) {
blockInfo = masterClientResource.get().getBlockInfo(blockId);
} catch (AlluxioException e) {
throw new IOException(e);
}
if (blockInfo.getLocations().isEmpty()) {
throw new IOException("Block " + blockId + " is not available in Alluxio");
}
// TODO(calvin): Get location via a policy.
// Although blockInfo.locations are sorted by tier, we prefer reading from the local worker.
// But when there is no local worker or there are no local blocks, we prefer the first
// location in blockInfo.locations that is nearest to memory tier.
// Assuming if there is no local worker, there are no local blocks in blockInfo.locations.
// TODO(cc): Check mContext.hasLocalWorker before finding for a local block when the TODO
// for hasLocalWorker is fixed.
String localHostName = NetworkAddressUtils.getLocalHostName(ClientContext.getConf());
for (BlockLocation location : blockInfo.getLocations()) {
WorkerNetAddress workerNetAddress = location.getWorkerAddress();
if (workerNetAddress.getHost().equals(localHostName)) {
// There is a local worker and the block is local.
try {
return new LocalBlockInStream(blockId, blockInfo.getLength());
} catch (IOException e) {
LOG.warn("Failed to open local stream for block " + blockId + ". " + e.getMessage());
// Getting a local stream failed, do not try again
break;
}
}
}
// No local worker/block, get the first location since it's nearest to memory tier.
WorkerNetAddress workerNetAddress = blockInfo.getLocations().get(0).getWorkerAddress();
return new RemoteBlockInStream(blockId, blockInfo.getLength(), workerNetAddress);
}
/**
* Constructor with a flag to indicate whether system properties should be included. When the flag
* is set to false, it is used for {@link Configuration} test class.
*
* @param sitePropertiesFile site-wide properties
* @param includeSystemProperties whether to include the system properties
*/
private static void init(String sitePropertiesFile, boolean includeSystemProperties) {
// Load default
Properties defaultProps = ConfigurationUtils.loadPropertiesFromResource(DEFAULT_PROPERTIES);
if (defaultProps == null) {
throw new RuntimeException(
ExceptionMessage.DEFAULT_PROPERTIES_FILE_DOES_NOT_EXIST.getMessage());
}
// Override runtime default
defaultProps.setProperty(Constants.MASTER_HOSTNAME, NetworkAddressUtils.getLocalHostName(250));
defaultProps.setProperty(
Constants.WORKER_NETWORK_NETTY_CHANNEL, String.valueOf(ChannelType.defaultType()));
defaultProps.setProperty(
Constants.USER_NETWORK_NETTY_CHANNEL, String.valueOf(ChannelType.defaultType()));
String confPaths;
// If site conf is overwritten in system properties, overwrite the default setting
if (System.getProperty(Constants.SITE_CONF_DIR) != null) {
confPaths = System.getProperty(Constants.SITE_CONF_DIR);
} else {
confPaths = defaultProps.getProperty(Constants.SITE_CONF_DIR);
}
String[] confPathList = confPaths.split(",");
// Load site specific properties file
Properties siteProps =
ConfigurationUtils.searchPropertiesFile(sitePropertiesFile, confPathList);
// Load system properties
Properties systemProps = new Properties();
if (includeSystemProperties) {
systemProps.putAll(System.getProperties());
}
// Now lets combine, order matters here
PROPERTIES.putAll(defaultProps);
if (siteProps != null) {
PROPERTIES.putAll(siteProps);
}
PROPERTIES.putAll(systemProps);
String masterHostname = PROPERTIES.getProperty(Constants.MASTER_HOSTNAME);
String masterPort = PROPERTIES.getProperty(Constants.MASTER_RPC_PORT);
boolean useZk = Boolean.parseBoolean(PROPERTIES.getProperty(Constants.ZOOKEEPER_ENABLED));
String masterAddress =
(useZk ? Constants.HEADER_FT : Constants.HEADER) + masterHostname + ":" + masterPort;
PROPERTIES.setProperty(Constants.MASTER_ADDRESS, masterAddress);
checkUserFileBufferBytes();
// Make sure the user hasn't set worker ports when there may be multiple workers per host
int maxWorkersPerHost = getInt(Constants.INTEGRATION_YARN_WORKERS_PER_HOST_MAX);
if (maxWorkersPerHost > 1) {
String message =
"%s cannot be specified when allowing multiple workers per host with "
+ Constants.INTEGRATION_YARN_WORKERS_PER_HOST_MAX
+ "="
+ maxWorkersPerHost;
Preconditions.checkState(
System.getProperty(Constants.WORKER_DATA_PORT) == null,
String.format(message, Constants.WORKER_DATA_PORT));
Preconditions.checkState(
System.getProperty(Constants.WORKER_RPC_PORT) == null,
String.format(message, Constants.WORKER_RPC_PORT));
Preconditions.checkState(
System.getProperty(Constants.WORKER_WEB_PORT) == null,
String.format(message, Constants.WORKER_WEB_PORT));
PROPERTIES.setProperty(Constants.WORKER_DATA_PORT, "0");
PROPERTIES.setProperty(Constants.WORKER_RPC_PORT, "0");
PROPERTIES.setProperty(Constants.WORKER_WEB_PORT, "0");
}
}
/**
* Builds unique metric registry names with unique ID (set to host name). The pattern is
* instance.hostname.metricName.
*
* @param instance the instance name
* @param name the metric name
* @return the metric registry name
*/
public static String getMetricNameWithUniqueId(String instance, String name) {
return Joiner.on(".")
.join(instance, NetworkAddressUtils.getLocalHostName().replace('.', '_'), name);
}