/** Tests the default properties for the user. */
@Test
public void userDefaultTest() {
int intValue = sDefaultConfiguration.getInt(Constants.USER_FAILED_SPACE_REQUEST_LIMITS);
Assert.assertEquals(3, intValue);
intValue = sDefaultConfiguration.getInt(Constants.USER_HEARTBEAT_INTERVAL_MS);
Assert.assertEquals(Constants.SECOND_MS, intValue);
long longValue = sDefaultConfiguration.getBytes(Constants.USER_FILE_BUFFER_BYTES);
Assert.assertEquals(Constants.MB, longValue);
longValue = sDefaultConfiguration.getBytes(Constants.USER_BLOCK_REMOTE_READ_BUFFER_SIZE_BYTES);
Assert.assertEquals(8 * Constants.MB, longValue);
}
/** Tests the default properties for the worker. */
@Test
public void workerDefaultTest() {
String value = sDefaultConfiguration.get(Constants.WORKER_DATA_FOLDER);
Assert.assertNotNull(value);
Assert.assertEquals("/alluxioworker/", value);
value = sDefaultConfiguration.get(Constants.WORKER_BIND_HOST);
Assert.assertNotNull(value);
Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);
int intValue = sDefaultConfiguration.getInt(Constants.WORKER_RPC_PORT);
Assert.assertEquals(29998, intValue);
value = sDefaultConfiguration.get(Constants.WORKER_DATA_BIND_HOST);
Assert.assertNotNull(value);
Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_DATA_PORT);
Assert.assertEquals(29999, intValue);
value = sDefaultConfiguration.get(Constants.WORKER_WEB_BIND_HOST);
Assert.assertNotNull(value);
Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_WEB_PORT);
Assert.assertEquals(30000, intValue);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_BLOCK_HEARTBEAT_TIMEOUT_MS);
Assert.assertEquals(10 * Constants.SECOND_MS, intValue);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS);
Assert.assertEquals(Constants.SECOND_MS, intValue);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_WORKER_BLOCK_THREADS_MIN);
Assert.assertEquals(1, intValue);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_SESSION_TIMEOUT_MS);
Assert.assertEquals(10 * Constants.SECOND_MS, intValue);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_NETWORK_NETTY_BOSS_THREADS);
Assert.assertEquals(1, intValue);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_NETWORK_NETTY_WORKER_THREADS);
Assert.assertEquals(0, intValue);
long longValue = sDefaultConfiguration.getBytes(Constants.WORKER_MEMORY_SIZE);
Assert.assertEquals(128 * Constants.MB, longValue);
}
/** Tests the default common properties. */
@Test
public void commonDefaultTest() {
String alluxioHome = sDefaultConfiguration.get(Constants.HOME);
Assert.assertNotNull(alluxioHome);
Assert.assertEquals("/mnt/alluxio_default_home", alluxioHome);
String ufsAddress = sDefaultConfiguration.get(Constants.UNDERFS_ADDRESS);
Assert.assertNotNull(ufsAddress);
Assert.assertEquals(alluxioHome + "/underFSStorage", ufsAddress);
String value = sDefaultConfiguration.get(Constants.WEB_RESOURCES);
Assert.assertNotNull(value);
Assert.assertEquals(alluxioHome + "/core/server/src/main/webapp", value);
value = sDefaultConfiguration.get(Constants.UNDERFS_HDFS_IMPL);
Assert.assertNotNull(value);
Assert.assertEquals("org.apache.hadoop.hdfs.DistributedFileSystem", value);
value = sDefaultConfiguration.get(Constants.UNDERFS_HDFS_PREFIXS);
Assert.assertNotNull(value);
Assert.assertEquals(DEFAULT_HADOOP_UFS_PREFIX, value);
value = sDefaultConfiguration.get(Constants.UNDERFS_GLUSTERFS_IMPL);
Assert.assertNotNull(value);
Assert.assertEquals("org.apache.hadoop.fs.glusterfs.GlusterFileSystem", value);
boolean booleanValue = sDefaultConfiguration.getBoolean(Constants.ZOOKEEPER_ENABLED);
Assert.assertFalse(booleanValue);
booleanValue = sDefaultConfiguration.getBoolean(Constants.IN_TEST_MODE);
Assert.assertFalse(booleanValue);
int intValue = sDefaultConfiguration.getInt(Constants.NETWORK_HOST_RESOLUTION_TIMEOUT_MS);
Assert.assertEquals(Constants.DEFAULT_HOST_RESOLUTION_TIMEOUT_MS, intValue);
long longBytesValue =
sDefaultConfiguration.getBytes(Constants.USER_BLOCK_REMOTE_READ_BUFFER_SIZE_BYTES);
Assert.assertEquals(Constants.MB * 8, longBytesValue);
int maxTry = sDefaultConfiguration.getInt(Constants.ZOOKEEPER_LEADER_INQUIRY_RETRY_COUNT);
Assert.assertEquals(10, maxTry);
}
/**
* Tests writing to a file for longer than HEARTBEAT_INTERVAL_MS to make sure the sessionId
* doesn't change. Tracks [ALLUXIO-171].
*/
@Test
public void longWrite() throws Exception {
AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
final int length = 2;
FileOutStream os =
mFileSystem.createFile(
filePath, CreateFileOptions.defaults().setWriteType(WriteType.THROUGH));
os.write((byte) 0);
Thread.sleep(Configuration.getInt(PropertyKey.USER_HEARTBEAT_INTERVAL_MS) * 2);
os.write((byte) 1);
os.close();
checkWrite(filePath, UnderStorageType.SYNC_PERSIST, length, length);
}
@Override
public void start(boolean isLeader) throws IOException {
super.start(isLeader);
mGlobalStorageTierAssoc = new MasterStorageTierAssoc();
if (isLeader) {
mLostWorkerDetectionService =
getExecutorService()
.submit(
new HeartbeatThread(
HeartbeatContext.MASTER_LOST_WORKER_DETECTION,
new LostWorkerDetectionHeartbeatExecutor(),
Configuration.getInt(Constants.MASTER_HEARTBEAT_INTERVAL_MS)));
}
}
@Override
public void start() {
mFilePersistenceService =
getExecutorService()
.submit(
new HeartbeatThread(
HeartbeatContext.WORKER_FILESYSTEM_MASTER_SYNC,
new FileWorkerMasterSyncExecutor(
mFileDataManager, mFileSystemMasterWorkerClient, mWorkerId),
Configuration.getInt(PropertyKey.WORKER_FILESYSTEM_HEARTBEAT_INTERVAL_MS)));
// Start the session cleanup checker to perform the periodical checking
getExecutorService().submit(mSessionCleaner);
}
@Override
public void heartbeat() {
int masterWorkerTimeoutMs = Configuration.getInt(Constants.MASTER_WORKER_TIMEOUT_MS);
for (MasterWorkerInfo worker : mWorkers) {
synchronized (worker) {
final long lastUpdate = CommonUtils.getCurrentMs() - worker.getLastUpdatedTimeMs();
if (lastUpdate > masterWorkerTimeoutMs) {
LOG.error(
"The worker {} timed out after {}ms without a heartbeat!", worker, lastUpdate);
mLostWorkers.add(worker);
mWorkers.remove(worker);
processWorkerRemovedBlocks(worker, worker.getBlocks());
}
}
}
}
/** Tests the default properties for the master. */
@Test
public void masterDefaultTest() {
String alluxioHome = sDefaultConfiguration.get(Constants.HOME);
Assert.assertNotNull(alluxioHome);
Assert.assertEquals("/mnt/alluxio_default_home", alluxioHome);
String value = sDefaultConfiguration.get(Constants.MASTER_JOURNAL_FOLDER);
Assert.assertNotNull(value);
Assert.assertEquals(alluxioHome + "/journal/", value);
value = sDefaultConfiguration.get(Constants.MASTER_HOSTNAME);
Assert.assertNotNull(value);
Assert.assertEquals(NetworkAddressUtils.getLocalHostName(100), value);
value = sDefaultConfiguration.get(Constants.MASTER_FORMAT_FILE_PREFIX);
Assert.assertNotNull(value);
Assert.assertEquals(Constants.FORMAT_FILE_PREFIX, value);
value = sDefaultConfiguration.get(Constants.MASTER_ADDRESS);
Assert.assertNotNull(value);
value = sDefaultConfiguration.get(Constants.MASTER_BIND_HOST);
Assert.assertNotNull(value);
Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);
int intValue = sDefaultConfiguration.getInt(Constants.MASTER_RPC_PORT);
Assert.assertEquals(19998, intValue);
value = sDefaultConfiguration.get(Constants.MASTER_WEB_BIND_HOST);
Assert.assertNotNull(value);
Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);
intValue = sDefaultConfiguration.getInt(Constants.MASTER_WEB_PORT);
Assert.assertEquals(19999, intValue);
intValue = sDefaultConfiguration.getInt(Constants.WEB_THREAD_COUNT);
Assert.assertEquals(1, intValue);
intValue = sDefaultConfiguration.getInt(Constants.MASTER_HEARTBEAT_INTERVAL_MS);
Assert.assertEquals(Constants.SECOND_MS, intValue);
intValue = sDefaultConfiguration.getInt(Constants.MASTER_WORKER_THREADS_MIN);
Assert.assertEquals(Runtime.getRuntime().availableProcessors(), intValue);
intValue = sDefaultConfiguration.getInt(Constants.MASTER_WORKER_TIMEOUT_MS);
Assert.assertEquals(10 * Constants.SECOND_MS, intValue);
}
/**
* Alluxio stores data into an under layer file system. Any file system implementing this interface
* can be a valid under layer file system
*/
@ThreadSafe
public abstract class UnderFileSystem {
private static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_TYPE);
/** The UFS {@link AlluxioURI} used to create this {@link UnderFileSystem}. */
protected final AlluxioURI mUri;
/** A map of property names to values. */
protected HashMap<String, String> mProperties = new HashMap<>();
/**
* This variable indicates whether the underFS actually provides storage. Most UnderFS should
* provide storage, but a dummyFS for example does not.
*/
private boolean mProvidesStorage = true;
/** Maximum length for a single listing query. */
private static final int MAX_LISTING_LENGTH = 1000;
/** Length of each list request. */
protected static final int LISTING_LENGTH =
Configuration.getInt(PropertyKey.UNDERFS_LISTING_LENGTH) > MAX_LISTING_LENGTH
? MAX_LISTING_LENGTH
: Configuration.getInt(PropertyKey.UNDERFS_LISTING_LENGTH);
private static final Cache UFS_CACHE = new Cache();
/**
* The different types of space indicate the total space, the free space and the space used in the
* under file system.
*/
public enum SpaceType {
/** Indicates the storage capacity of the under file system. */
SPACE_TOTAL(0),
/** Indicates the amount of free space available in the under file system. */
SPACE_FREE(1),
/** Indicates the amount of space used in the under file system. */
SPACE_USED(2),
;
private final int mValue;
SpaceType(int value) {
mValue = value;
}
/** @return the integer value of this enum value */
public int getValue() {
return mValue;
}
}
/** A class used to cache UnderFileSystems. */
@ThreadSafe
private static class Cache {
/** Maps from {@link Key} to {@link UnderFileSystem} instances. */
private final ConcurrentHashMap<Key, UnderFileSystem> mUnderFileSystemMap =
new ConcurrentHashMap<>();
Cache() {}
/**
* Gets a UFS instance from the cache if exists. Otherwise, creates a new instance and adds that
* to the cache.
*
* @param path the ufs path
* @param ufsConf the ufs configuration
* @return the UFS instance
*/
UnderFileSystem get(String path, Object ufsConf) {
Key key = new Key(new AlluxioURI(path));
UnderFileSystem cachedFs = mUnderFileSystemMap.get(key);
if (cachedFs != null) {
return cachedFs;
}
UnderFileSystem fs = UnderFileSystemRegistry.create(path, ufsConf);
cachedFs = mUnderFileSystemMap.putIfAbsent(key, fs);
if (cachedFs == null) {
return fs;
}
try {
fs.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
return cachedFs;
}
void clear() {
mUnderFileSystemMap.clear();
}
}
/** The key of the UFS cache. */
private static class Key {
private final String mScheme;
private final String mAuthority;
Key(AlluxioURI uri) {
mScheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase();
mAuthority = uri.getAuthority() == null ? "" : uri.getAuthority().toLowerCase();
}
@Override
public int hashCode() {
return Objects.hashCode(mScheme, mAuthority);
}
@Override
public boolean equals(Object object) {
if (object == this) {
return true;
}
if (!(object instanceof Key)) {
return false;
}
Key that = (Key) object;
return Objects.equal(mScheme, that.mScheme) && Objects.equal(mAuthority, that.mAuthority);
}
@Override
public String toString() {
return mScheme + "://" + mAuthority;
}
}
/**
* Gets the UnderFileSystem instance according to its schema.
*
* @param path the file path storing over the ufs
* @return instance of the under layer file system
*/
public static UnderFileSystem get(String path) {
return get(path, null);
}
/**
* Gets the UnderFileSystem instance according to its scheme and configuration.
*
* @param path the file path storing over the ufs
* @param ufsConf the configuration object for ufs only
* @return instance of the under layer file system
*/
public static UnderFileSystem get(String path, Object ufsConf) {
Preconditions.checkArgument(path != null, "path may not be null");
return UFS_CACHE.get(path, ufsConf);
}
/** Clears the under file system cache. */
public static void clearCache() {
UFS_CACHE.clear();
}
/**
* Returns the name of the under filesystem implementation.
*
* <p>The name should be lowercase and not include any spaces, e.g. "hdfs", "s3".
*
* @return name of the under filesystem implementation
*/
public abstract String getUnderFSType();
/**
* Checks whether the underFS provides storage.
*
* @return true if the under filesystem provides storage, false otherwise
*/
public boolean providesStorage() {
return mProvidesStorage;
}
/**
* Constructs an {@link UnderFileSystem}.
*
* @param uri the {@link AlluxioURI} used to create this ufs
*/
protected UnderFileSystem(AlluxioURI uri) {
Preconditions.checkNotNull(uri);
mUri = uri;
}
/**
* Configures and updates the properties. For instance, this method can add new properties or
* modify existing properties specified through {@link #setProperties(Map)}.
*
* <p>The default implementation is a no-op. This should be overridden if a subclass needs
* additional functionality.
*
* @throws IOException if an error occurs during configuration
*/
public void configureProperties() throws IOException {
// Default implementation does not update any properties.
}
/**
* Takes any necessary actions required to establish a connection to the under file system from
* the given master host e.g. logging in
*
* <p>Depending on the implementation this may be a no-op
*
* @param hostname the host that wants to connect to the under file system
* @throws IOException if a non-Alluxio error occurs
*/
public abstract void connectFromMaster(String hostname) throws IOException;
/**
* Takes any necessary actions required to establish a connection to the under file system from
* the given worker host e.g. logging in
*
* <p>Depending on the implementation this may be a no-op
*
* @param hostname the host that wants to connect to the under file system
* @throws IOException if a non-Alluxio error occurs
*/
public abstract void connectFromWorker(String hostname) throws IOException;
/**
* Closes this under file system.
*
* @throws IOException if a non-Alluxio error occurs
*/
public abstract void close() throws IOException;
/**
* Creates a file in the under file system with the indicated name.
*
* @param path the file name
* @return A {@code OutputStream} object
* @throws IOException if a non-Alluxio error occurs
*/
public OutputStream create(String path) throws IOException {
return create(path, new CreateOptions());
}
/**
* Creates a file in the under file system with the specified {@link CreateOptions}.
* Implementations should make sure that the path under creation appears in listings only after a
* successful close and that contents are written in its entirety or not at all.
*
* @param path the file name
* @param options the options for create
* @return A {@code OutputStream} object
* @throws IOException if a non-Alluxio error occurs
*/
public OutputStream create(String path, CreateOptions options) throws IOException {
if (!options.isEnsureAtomic()) {
return createDirect(path, options);
}
return new NonAtomicFileOutputStream(path, options, this);
}
/**
* Creates a file in the under file system with the specified {@link CreateOptions}. This stream
* writes directly to the underlying storage without any atomicity guarantees.
*
* @param path the file name
* @param options the options for create
* @return A {@code OutputStream} object
* @throws IOException if a non-Alluxio error occurs
*/
public abstract OutputStream createDirect(String path, CreateOptions options) throws IOException;
/**
* Deletes a file or folder from the under file system with the indicated name.
*
* @param path the file or folder name
* @param recursive the boolean indicates whether we delete folder and its children
* @return true if succeed, false otherwise
* @throws IOException if a non-Alluxio error occurs
*/
public abstract boolean delete(String path, boolean recursive) throws IOException;
/**
* Checks if a file or folder exists in under file system.
*
* @param path the file name
* @return true if succeed, false otherwise
* @throws IOException if a non-Alluxio error occurs
*/
public abstract boolean exists(String path) throws IOException;
/**
* Gets the block size of a file in under file system, in bytes.
*
* @param path the file name
* @return the block size in bytes
* @throws IOException if a non-Alluxio error occurs
*/
public abstract long getBlockSizeByte(String path) throws IOException;
/**
* Gets the configuration object for UnderFileSystem.
*
* @return configuration object used for concrete ufs instance
*/
public abstract Object getConf();
/**
* Gets the list of locations of the indicated path.
*
* @param path the file name
* @return The list of locations
* @throws IOException if a non-Alluxio error occurs
*/
public abstract List<String> getFileLocations(String path) throws IOException;
/**
* Gets the list of locations of the indicated path given its offset.
*
* @param path the file name
* @param offset the offset in bytes
* @return The list of locations
* @throws IOException if a non-Alluxio error occurs
*/
public abstract List<String> getFileLocations(String path, long offset) throws IOException;
/**
* Gets the file size in bytes.
*
* @param path the file name
* @return the file size in bytes
* @throws IOException if a non-Alluxio error occurs
*/
public abstract long getFileSize(String path) throws IOException;
/**
* Gets the UTC time of when the indicated path was modified recently in ms.
*
* @param path the file or folder name
* @return modification time in milliseconds
* @throws IOException if a non-Alluxio error occurs
*/
public abstract long getModificationTimeMs(String path) throws IOException;
/** @return the property map for this {@link UnderFileSystem} */
public Map<String, String> getProperties() {
return Collections.unmodifiableMap(mProperties);
}
/**
* Queries the under file system about the space of the indicated path (e.g., space left, space
* used and etc).
*
* @param path the path to query
* @param type the type of queries
* @return The space in bytes
* @throws IOException if a non-Alluxio error occurs
*/
public abstract long getSpace(String path, SpaceType type) throws IOException;
/**
* Checks if the indicated path is a file or not.
*
* @param path the path name
* @return true if this is a file, false otherwise
* @throws IOException if a non-Alluxio error occurs
*/
public abstract boolean isFile(String path) throws IOException;
/**
* Returns an array of strings naming the files and directories in the directory denoted by this
* abstract pathname.
*
* <p>If this abstract pathname does not denote a directory, then this method returns {@code
* null}. Otherwise an array of strings is returned, one for each file or directory in the
* directory. Names denoting the directory itself and the directory's parent directory are not
* included in the result. Each string is a file name rather than a complete path.
*
* <p>There is no guarantee that the name strings in the resulting array will appear in any
* specific order; they are not, in particular, guaranteed to appear in alphabetical order.
*
* @param path the abstract pathname to list
* @return An array of strings naming the files and directories in the directory denoted by this
* abstract pathname. The array will be empty if the directory is empty. Returns {@code null}
* if this abstract pathname does not denote a directory.
* @throws IOException if a non-Alluxio error occurs
*/
public abstract String[] list(String path) throws IOException;
/**
* Returns an array of strings naming the files and directories in the directory denoted by this
* abstract pathname, and all of its subdirectories.
*
* <p>If this abstract pathname does not denote a directory, then this method returns {@code
* null}. Otherwise an array of strings is returned, one for each file or directory in the
* directory and its subdirectories. Names denoting the directory itself and the directory's
* parent directory are not included in the result. Each string is a path relative to the given
* directory.
*
* <p>There is no guarantee that the name strings in the resulting array will appear in any
* specific order; they are not, in particular, guaranteed to appear in alphabetical order.
*
* @param path the abstract pathname to list
* @return An array of strings naming the files and directories in the directory denoted by this
* abstract pathname and its subdirectories. The array will be empty if the directory is
* empty. Returns {@code null} if this abstract pathname does not denote a directory.
* @throws IOException if a non-Alluxio error occurs
*/
public String[] listRecursive(String path) throws IOException {
// Clean the path by creating a URI and turning it back to a string
AlluxioURI uri = new AlluxioURI(path);
path = uri.toString();
List<String> returnPaths = new ArrayList<>();
Queue<String> pathsToProcess = new ArrayDeque<>();
// We call list initially, so we can return null if the path doesn't denote a directory
String[] subpaths = list(path);
if (subpaths == null) {
return null;
} else {
for (String subp : subpaths) {
pathsToProcess.add(PathUtils.concatPath(path, subp));
}
}
while (!pathsToProcess.isEmpty()) {
String p = pathsToProcess.remove();
returnPaths.add(p.substring(path.length() + 1));
// Add all of its subpaths
subpaths = list(p);
if (subpaths != null) {
for (String subp : subpaths) {
pathsToProcess.add(PathUtils.concatPath(p, subp));
}
}
}
return returnPaths.toArray(new String[returnPaths.size()]);
}
/**
* Creates the directory named by this abstract pathname. If the folder already exists, the method
* returns false.
*
* @param path the folder to create
* @param createParent if true, the method creates any necessary but nonexistent parent
* directories. Otherwise, the method does not create nonexistent parent directories
* @return {@code true} if and only if the directory was created; {@code false} otherwise
* @throws IOException if a non-Alluxio error occurs
*/
public abstract boolean mkdirs(String path, boolean createParent) throws IOException;
/**
* Creates the directory named by this abstract pathname, with specified {@link MkdirsOptions}. If
* the folder already exists, the method returns false.
*
* @param path the folder to create
* @param options the options for mkdirs
* @return {@code true} if and only if the directory was created; {@code false} otherwise
* @throws IOException if a non-Alluxio error occurs
*/
public abstract boolean mkdirs(String path, MkdirsOptions options) throws IOException;
/**
* Opens an {@link InputStream} at the indicated path.
*
* @param path the file name
* @return The {@code InputStream} object
* @throws IOException if a non-Alluxio error occurs
*/
public abstract InputStream open(String path) throws IOException;
/**
* Renames a file or folder from {@code src} to {@code dst} in under file system.
*
* @param src the source file or folder name
* @param dst the destination file or folder name
* @return true if succeed, false otherwise
* @throws IOException if a non-Alluxio error occurs
*/
public abstract boolean rename(String src, String dst) throws IOException;
/**
* Returns an {@link AlluxioURI} representation for the {@link UnderFileSystem} given a base UFS
* URI, and the Alluxio path from the base.
*
* <p>The default implementation simply concatenates the path to the base URI. This should be
* overridden if a subclass needs alternate functionality.
*
* @param ufsBaseUri the base {@link AlluxioURI} in the ufs
* @param alluxioPath the path in Alluxio from the given base
* @return the UFS {@link AlluxioURI} representing the Alluxio path
*/
public AlluxioURI resolveUri(AlluxioURI ufsBaseUri, String alluxioPath) {
return new AlluxioURI(
ufsBaseUri.getScheme(),
ufsBaseUri.getAuthority(),
PathUtils.concatPath(ufsBaseUri.getPath(), alluxioPath),
ufsBaseUri.getQueryMap());
}
/**
* Sets the configuration object for UnderFileSystem. The conf object is understood by the
* concrete underfs's implementation.
*
* @param conf the configuration object accepted by ufs
*/
public abstract void setConf(Object conf);
/**
* Sets the user and group of the given path. An empty implementation should be provided if
* unsupported.
*
* @param path the path of the file
* @param owner the new owner to set, unchanged if null
* @param group the new group to set, unchanged if null
* @throws IOException if a non-Alluxio error occurs
*/
public abstract void setOwner(String path, String owner, String group) throws IOException;
/**
* Sets the properties for this {@link UnderFileSystem}.
*
* @param properties a {@link Map} of property names to values
*/
public void setProperties(Map<String, String> properties) {
mProperties.clear();
mProperties.putAll(properties);
}
/**
* Changes posix file mode.
*
* @param path the path of the file
* @param mode the mode to set in short format, e.g. 0777
* @throws IOException if a non-Alluxio error occurs
*/
public abstract void setMode(String path, short mode) throws IOException;
/**
* Gets the owner of the given path. An empty implementation should be provided if not supported.
*
* @param path the path of the file
* @return the owner of the file
* @throws IOException if a non-Alluxio error occurs
*/
public abstract String getOwner(String path) throws IOException;
/**
* Gets the group of the given path. An empty implementation should be provided if not supported.
*
* @param path the path of the file
* @return the group of the file
* @throws IOException if a non-Alluxio error occurs
*/
public abstract String getGroup(String path) throws IOException;
/**
* Gets the mode of the given path in short format, e.g 0700. An empty implementation should be
* provided if not supported.
*
* @param path the path of the file
* @return the mode of the file
* @throws IOException if a non-Alluxio error occurs
*/
public abstract short getMode(String path) throws IOException;
/**
* Whether this type of UFS supports flush.
*
* @return true if this type of UFS supports flush, false otherwise
*/
public boolean supportsFlush() {
return true;
}
}
/**
* A netty packet reader that streams a region from a netty data server.
*
* <p>Protocol: 1. The client sends a read request (id, offset, length). 2. Once the server receives
* the request, it streams packets to the client. The streaming pauses if the server's buffer is
* full and resumes if the buffer is not full. 3. The client reads packets from the stream. Reading
* pauses if the client buffer is full and resumes if the buffer is not full. If the client can keep
* up with network speed, the buffer should have at most one packet. 4. The client stops reading if
* it receives an empty packet which signifies the end of the stream. 5. The client can cancel the
* read request at anytime. The cancel request is ignored by the server if everything has been sent
* to channel. 6. If the client wants to reuse the channel, the client must read all the packets in
* the channel before releasing the channel to the channel pool. 7. To make it simple to handle
* errors, the channel is closed if any error occurs.
*/
@NotThreadSafe
public final class NettyPacketReader implements PacketReader {
private static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_TYPE);
private static final boolean CANCEL_ENABLED =
Configuration.getBoolean(PropertyKey.USER_NETWORK_NETTY_READER_CANCEL_ENABLED);
private static final int MAX_PACKETS_IN_FLIGHT =
Configuration.getInt(PropertyKey.USER_NETWORK_NETTY_READER_BUFFER_SIZE_PACKETS);
private static final long READ_TIMEOUT_MS =
Configuration.getLong(PropertyKey.USER_NETWORK_NETTY_TIMEOUT_MS);
private final FileSystemContext mContext;
private final Channel mChannel;
private final Protocol.RequestType mRequestType;
private final InetSocketAddress mAddress;
private final long mId;
private final long mStart;
private final long mBytesToRead;
// TODO(peis): Investigate whether we can remove this lock. The main reason to keep this lock
// is to protect mPacketReaderException.
private final ReentrantLock mLock = new ReentrantLock();
@GuardedBy("mLock")
private final Queue<ByteBuf> mPackets = new LinkedList<>();
@GuardedBy("mLock")
private Throwable mPacketReaderException = null;
/** The condition is met when mPackets.size() > 0 or mPacketReaderException != null. */
private final Condition mNotEmptyOrFailed = mLock.newCondition();
/** The next pos to read. */
private long mPosToRead;
/** This is true only when an empty packet is received. */
private boolean mDone = false;
private boolean mClosed = false;
/**
* Creates an instance of {@link NettyPacketReader}. If this is used to read a block remotely, it
* requires the block to be locked beforehand and the lock ID is passed to this class.
*
* @param context the file system context
* @param address the netty data server network address
* @param id the block ID or UFS file ID
* @param offset the offset
* @param len the length to read
* @param lockId the lock ID
* @param sessionId the session ID
* @param type the request type (block or UFS file)
* @throws IOException if it fails to acquire a netty channel
*/
private NettyPacketReader(
FileSystemContext context,
InetSocketAddress address,
long id,
long offset,
long len,
long lockId,
long sessionId,
Protocol.RequestType type)
throws IOException {
Preconditions.checkArgument(offset >= 0 && len > 0);
mContext = context;
mAddress = address;
mId = id;
mStart = offset;
mPosToRead = offset;
mBytesToRead = len;
mRequestType = type;
mChannel = context.acquireNettyChannel(address);
ChannelPipeline pipeline = mChannel.pipeline();
if (!(pipeline.last() instanceof RPCMessageDecoder)) {
throw new RuntimeException(
String.format(
"Channel pipeline has unexpected handlers %s.",
pipeline.last().getClass().getCanonicalName()));
}
mChannel.pipeline().addLast(new PacketReadHandler());
Protocol.ReadRequest readRequest =
Protocol.ReadRequest.newBuilder()
.setId(id)
.setOffset(offset)
.setLength(len)
.setLockId(lockId)
.setSessionId(sessionId)
.setType(type)
.build();
mChannel
.writeAndFlush(new RPCProtoMessage(readRequest))
.addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
}
@Override
public long pos() {
return mPosToRead;
}
@Override
public ByteBuf readPacket() throws IOException {
Preconditions.checkState(!mClosed, "PacketReader is closed while reading packets.");
ByteBuf buf = null;
mLock.lock();
try {
while (true) {
if (mDone) {
return null;
}
if (mPacketReaderException != null) {
throw new IOException(mPacketReaderException);
}
buf = mPackets.poll();
// TODO(peis): Have a better criteria to resume so that we can have fewer state changes.
if (!tooManyPacketsPending()) {
resume();
}
// Queue is empty.
if (buf == null) {
try {
if (!mNotEmptyOrFailed.await(READ_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
throw new IOException(
String.format("Timeout while reading packet from block %d @ %s.", mId, mAddress));
}
} catch (InterruptedException e) {
throw Throwables.propagate(e);
}
} else {
if (buf.readableBytes() == 0) {
buf.release();
mDone = true;
return null;
}
mPosToRead += buf.readableBytes();
Preconditions.checkState(mPosToRead - mStart <= mBytesToRead);
return buf;
}
}
} catch (Throwable e) {
if (buf != null) {
buf.release();
}
throw e;
} finally {
mLock.unlock();
}
}
@Override
public void close() {
try {
if (mDone) {
return;
}
if (!mChannel.isOpen()) {
return;
}
try {
if (!CANCEL_ENABLED) {
mChannel.close().sync();
return;
}
if (remaining() > 0) {
Protocol.ReadRequest cancelRequest =
Protocol.ReadRequest.newBuilder()
.setId(mId)
.setCancel(true)
.setType(mRequestType)
.build();
mChannel
.writeAndFlush(new RPCProtoMessage(cancelRequest))
.addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
}
} catch (InterruptedException e) {
mChannel.close();
throw Throwables.propagate(e);
}
while (true) {
try {
ByteBuf buf = readPacket();
// A null packet indicates the end of the stream.
if (buf == null) {
return;
}
buf.release();
} catch (IOException e) {
LOG.warn(
"Failed to close the NettyBlockReader (block: {}, address: {}).", mId, mAddress, e);
try {
mChannel.close().sync();
} catch (InterruptedException ee) {
throw Throwables.propagate(ee);
}
return;
}
}
} finally {
if (mChannel.isOpen()) {
Preconditions.checkState(mChannel.pipeline().last() instanceof PacketReadHandler);
mChannel.pipeline().removeLast();
// Make sure "autoread" is on before realsing the channel.
resume();
}
mContext.releaseNettyChannel(mAddress, mChannel);
mClosed = true;
}
}
/** @return bytes remaining */
private long remaining() {
return mStart + mBytesToRead - mPosToRead;
}
/** @return true if there are too many packets pending */
private boolean tooManyPacketsPending() {
return mPackets.size() >= MAX_PACKETS_IN_FLIGHT;
}
/** The netty handler that reads packets from the channel. */
private class PacketReadHandler extends ChannelInboundHandlerAdapter {
/** Default constructor. */
public PacketReadHandler() {}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) {
Preconditions.checkState(
acceptMessage(msg),
"Incorrect response type %s, %s.",
msg.getClass().getCanonicalName(),
msg);
RPCProtoMessage response = (RPCProtoMessage) msg;
Protocol.Status status = ((Protocol.Response) response.getMessage()).getStatus();
if (!Status.isOk(status)) {
ctx.fireExceptionCaught(
new IOException(
String.format(
"Failed to read block %d from %s with status %s.",
mId, mAddress, status.toString())));
}
mLock.lock();
try {
Preconditions.checkState(mPacketReaderException == null);
DataBuffer dataBuffer = response.getPayloadDataBuffer();
ByteBuf buf;
if (dataBuffer == null) {
buf = ctx.alloc().buffer(0, 0);
} else {
Preconditions.checkState(dataBuffer.getLength() > 0);
assert dataBuffer.getNettyOutput() instanceof ByteBuf;
buf = (ByteBuf) dataBuffer.getNettyOutput();
}
mPackets.offer(buf);
mNotEmptyOrFailed.signal();
if (tooManyPacketsPending()) {
pause();
}
} finally {
mLock.unlock();
}
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
LOG.error("Exception caught while reading response from netty channel.", cause);
mLock.lock();
try {
mPacketReaderException = cause;
mNotEmptyOrFailed.signal();
} finally {
mLock.unlock();
}
ctx.close();
}
@Override
public void channelUnregistered(ChannelHandlerContext ctx) {
mLock.lock();
try {
if (mPacketReaderException == null) {
mPacketReaderException = new IOException("ChannelClosed");
}
mNotEmptyOrFailed.signal();
} finally {
mLock.unlock();
}
ctx.fireChannelUnregistered();
}
/**
* @param msg the message received
* @return true if this message should be processed
*/
private boolean acceptMessage(Object msg) {
if (msg instanceof RPCProtoMessage) {
MessageLite header = ((RPCProtoMessage) msg).getMessage();
return header instanceof Protocol.Response;
}
return false;
}
}
/** Pauses the underlying packet reader. */
private void pause() {
mChannel.config().setAutoRead(false);
}
/** Resumes the underlying packet reader. */
private void resume() {
mChannel.config().setAutoRead(true);
mChannel.read();
}
/** Factory class to create {@link NettyPacketReader}s. */
public static class Factory implements PacketReader.Factory {
private final FileSystemContext mContext;
private final InetSocketAddress mAddress;
private final long mId;
private final long mLockId;
private final long mSessionId;
private final Protocol.RequestType mRequestType;
/**
* Creates an instance of {@link NettyPacketReader.Factory} for block reads.
*
* @param context the file system context
* @param address the worker address
* @param id the block ID or UFS ID
* @param lockId the lock ID
* @param sessionId the session ID
* @param type the request type
*/
public Factory(
FileSystemContext context,
InetSocketAddress address,
long id,
long lockId,
long sessionId,
Protocol.RequestType type) {
mContext = context;
mAddress = address;
mId = id;
mLockId = lockId;
mSessionId = sessionId;
mRequestType = type;
}
@Override
public PacketReader create(long offset, long len) throws IOException {
return new NettyPacketReader(
mContext, mAddress, mId, offset, len, mLockId, mSessionId, mRequestType);
}
}
}
@Override
public void resourceOffers(SchedulerDriver driver, List<Protos.Offer> offers) {
long masterCpu = Configuration.getInt(PropertyKey.INTEGRATION_MASTER_RESOURCE_CPU);
long masterMem =
Configuration.getBytes(PropertyKey.INTEGRATION_MASTER_RESOURCE_MEM) / Constants.MB;
long workerCpu = Configuration.getInt(PropertyKey.INTEGRATION_WORKER_RESOURCE_CPU);
long workerMem =
Configuration.getBytes(PropertyKey.INTEGRATION_WORKER_RESOURCE_MEM) / Constants.MB;
LOG.info(
"Master launched {}, master count {}, "
+ "requested master cpu {} mem {} MB and required master hostname {}",
mMasterLaunched,
mMasterCount,
masterCpu,
masterMem,
mRequiredMasterHostname);
for (Protos.Offer offer : offers) {
Protos.Offer.Operation.Launch.Builder launch = Protos.Offer.Operation.Launch.newBuilder();
double offerCpu = 0;
double offerMem = 0;
for (Protos.Resource resource : offer.getResourcesList()) {
if (resource.getName().equals(Constants.MESOS_RESOURCE_CPUS)) {
offerCpu += resource.getScalar().getValue();
} else if (resource.getName().equals(Constants.MESOS_RESOURCE_MEM)) {
offerMem += resource.getScalar().getValue();
} else {
// Other resources are currently ignored.
}
}
LOG.info(
"Received offer {} on host {} with cpus {} and mem {} MB and hasMasterPorts {}",
offer.getId().getValue(),
offer.getHostname(),
offerCpu,
offerMem,
OfferUtils.hasAvailableMasterPorts(offer));
Protos.ExecutorInfo.Builder executorBuilder = Protos.ExecutorInfo.newBuilder();
List<Protos.Resource> resources;
if (!mMasterLaunched
&& offerCpu >= masterCpu
&& offerMem >= masterMem
&& mMasterCount
< Configuration.getInt(PropertyKey.INTEGRATION_MESOS_ALLUXIO_MASTER_NODE_COUNT)
&& OfferUtils.hasAvailableMasterPorts(offer)
&& (mRequiredMasterHostname == null
|| mRequiredMasterHostname.equals(offer.getHostname()))) {
LOG.debug("Creating Alluxio Master executor");
executorBuilder
.setName("Alluxio Master Executor")
.setSource("master")
.setExecutorId(Protos.ExecutorID.newBuilder().setValue("master"))
.addAllResources(getExecutorResources())
.setCommand(
Protos.CommandInfo.newBuilder()
.setValue(createStartAlluxioCommand("alluxio-master-mesos.sh"))
.addAllUris(getExecutorDependencyURIList())
.setEnvironment(
Protos.Environment.newBuilder()
.addVariables(
Protos.Environment.Variable.newBuilder()
.setName("ALLUXIO_UNDERFS_ADDRESS")
.setValue(Configuration.get(PropertyKey.UNDERFS_ADDRESS))
.build())
.build()));
// pre-build resource list here, then use it to build Protos.Task later.
resources = getMasterRequiredResources(masterCpu, masterMem);
mMasterHostname = offer.getHostname();
mTaskName = Configuration.get(PropertyKey.INTEGRATION_MESOS_ALLUXIO_MASTER_NAME);
mMasterCount++;
mMasterTaskId = mLaunchedTasks;
} else if (mMasterLaunched
&& !mWorkers.contains(offer.getHostname())
&& offerCpu >= workerCpu
&& offerMem >= workerMem
&& OfferUtils.hasAvailableWorkerPorts(offer)) {
LOG.debug("Creating Alluxio Worker executor");
final String memSize = FormatUtils.getSizeFromBytes((long) workerMem * Constants.MB);
executorBuilder
.setName("Alluxio Worker Executor")
.setSource("worker")
.setExecutorId(Protos.ExecutorID.newBuilder().setValue("worker"))
.addAllResources(getExecutorResources())
.setCommand(
Protos.CommandInfo.newBuilder()
.setValue(createStartAlluxioCommand("alluxio-worker-mesos.sh"))
.addAllUris(getExecutorDependencyURIList())
.setEnvironment(
Protos.Environment.newBuilder()
.addVariables(
Protos.Environment.Variable.newBuilder()
.setName("ALLUXIO_MASTER_HOSTNAME")
.setValue(mMasterHostname)
.build())
.addVariables(
Protos.Environment.Variable.newBuilder()
.setName("ALLUXIO_WORKER_MEMORY_SIZE")
.setValue(memSize)
.build())
.addVariables(
Protos.Environment.Variable.newBuilder()
.setName("ALLUXIO_UNDERFS_ADDRESS")
.setValue(Configuration.get(PropertyKey.UNDERFS_ADDRESS))
.build())
.build()));
// pre-build resource list here, then use it to build Protos.Task later.
resources = getWorkerRequiredResources(workerCpu, workerMem);
mWorkers.add(offer.getHostname());
mTaskName = Configuration.get(PropertyKey.INTEGRATION_MESOS_ALLUXIO_WORKER_NAME);
} else {
// The resource offer cannot be used to start either master or a worker.
LOG.info("Declining offer {}", offer.getId().getValue());
driver.declineOffer(offer.getId());
continue;
}
Protos.TaskID taskId =
Protos.TaskID.newBuilder().setValue(String.valueOf(mLaunchedTasks)).build();
LOG.info("Launching task {} using offer {}", taskId.getValue(), offer.getId().getValue());
Protos.TaskInfo task =
Protos.TaskInfo.newBuilder()
.setName(mTaskName)
.setTaskId(taskId)
.setSlaveId(offer.getSlaveId())
.addAllResources(resources)
.setExecutor(executorBuilder)
.build();
launch.addTaskInfos(Protos.TaskInfo.newBuilder(task));
mLaunchedTasks++;
// NOTE: We use the new API `acceptOffers` here to launch tasks.
// The 'launchTasks' API will be deprecated.
List<Protos.OfferID> offerIds = new ArrayList<Protos.OfferID>();
offerIds.add(offer.getId());
List<Protos.Offer.Operation> operations = new ArrayList<Protos.Offer.Operation>();
Protos.Offer.Operation operation =
Protos.Offer.Operation.newBuilder()
.setType(Protos.Offer.Operation.Type.LAUNCH)
.setLaunch(launch)
.build();
operations.add(operation);
Protos.Filters filters = Protos.Filters.newBuilder().setRefuseSeconds(1).build();
driver.acceptOffers(offerIds, operations, filters);
}
}
