/** Tests the default properties for the user. */ @Test public void userDefaultTest() { int intValue = sDefaultConfiguration.getInt(Constants.USER_FAILED_SPACE_REQUEST_LIMITS); Assert.assertEquals(3, intValue); intValue = sDefaultConfiguration.getInt(Constants.USER_HEARTBEAT_INTERVAL_MS); Assert.assertEquals(Constants.SECOND_MS, intValue); long longValue = sDefaultConfiguration.getBytes(Constants.USER_FILE_BUFFER_BYTES); Assert.assertEquals(Constants.MB, longValue); longValue = sDefaultConfiguration.getBytes(Constants.USER_BLOCK_REMOTE_READ_BUFFER_SIZE_BYTES); Assert.assertEquals(8 * Constants.MB, longValue); }
/** Tests the default properties for the worker. */ @Test public void workerDefaultTest() { String value = sDefaultConfiguration.get(Constants.WORKER_DATA_FOLDER); Assert.assertNotNull(value); Assert.assertEquals("/alluxioworker/", value); value = sDefaultConfiguration.get(Constants.WORKER_BIND_HOST); Assert.assertNotNull(value); Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value); int intValue = sDefaultConfiguration.getInt(Constants.WORKER_RPC_PORT); Assert.assertEquals(29998, intValue); value = sDefaultConfiguration.get(Constants.WORKER_DATA_BIND_HOST); Assert.assertNotNull(value); Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value); intValue = sDefaultConfiguration.getInt(Constants.WORKER_DATA_PORT); Assert.assertEquals(29999, intValue); value = sDefaultConfiguration.get(Constants.WORKER_WEB_BIND_HOST); Assert.assertNotNull(value); Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value); intValue = sDefaultConfiguration.getInt(Constants.WORKER_WEB_PORT); Assert.assertEquals(30000, intValue); intValue = sDefaultConfiguration.getInt(Constants.WORKER_BLOCK_HEARTBEAT_TIMEOUT_MS); Assert.assertEquals(10 * Constants.SECOND_MS, intValue); intValue = sDefaultConfiguration.getInt(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS); Assert.assertEquals(Constants.SECOND_MS, intValue); intValue = sDefaultConfiguration.getInt(Constants.WORKER_WORKER_BLOCK_THREADS_MIN); Assert.assertEquals(1, intValue); intValue = sDefaultConfiguration.getInt(Constants.WORKER_SESSION_TIMEOUT_MS); Assert.assertEquals(10 * Constants.SECOND_MS, intValue); intValue = sDefaultConfiguration.getInt(Constants.WORKER_NETWORK_NETTY_BOSS_THREADS); Assert.assertEquals(1, intValue); intValue = sDefaultConfiguration.getInt(Constants.WORKER_NETWORK_NETTY_WORKER_THREADS); Assert.assertEquals(0, intValue); long longValue = sDefaultConfiguration.getBytes(Constants.WORKER_MEMORY_SIZE); Assert.assertEquals(128 * Constants.MB, longValue); }
/** Tests the default common properties. */ @Test public void commonDefaultTest() { String alluxioHome = sDefaultConfiguration.get(Constants.HOME); Assert.assertNotNull(alluxioHome); Assert.assertEquals("/mnt/alluxio_default_home", alluxioHome); String ufsAddress = sDefaultConfiguration.get(Constants.UNDERFS_ADDRESS); Assert.assertNotNull(ufsAddress); Assert.assertEquals(alluxioHome + "/underFSStorage", ufsAddress); String value = sDefaultConfiguration.get(Constants.WEB_RESOURCES); Assert.assertNotNull(value); Assert.assertEquals(alluxioHome + "/core/server/src/main/webapp", value); value = sDefaultConfiguration.get(Constants.UNDERFS_HDFS_IMPL); Assert.assertNotNull(value); Assert.assertEquals("org.apache.hadoop.hdfs.DistributedFileSystem", value); value = sDefaultConfiguration.get(Constants.UNDERFS_HDFS_PREFIXS); Assert.assertNotNull(value); Assert.assertEquals(DEFAULT_HADOOP_UFS_PREFIX, value); value = sDefaultConfiguration.get(Constants.UNDERFS_GLUSTERFS_IMPL); Assert.assertNotNull(value); Assert.assertEquals("org.apache.hadoop.fs.glusterfs.GlusterFileSystem", value); boolean booleanValue = sDefaultConfiguration.getBoolean(Constants.ZOOKEEPER_ENABLED); Assert.assertFalse(booleanValue); booleanValue = sDefaultConfiguration.getBoolean(Constants.IN_TEST_MODE); Assert.assertFalse(booleanValue); int intValue = sDefaultConfiguration.getInt(Constants.NETWORK_HOST_RESOLUTION_TIMEOUT_MS); Assert.assertEquals(Constants.DEFAULT_HOST_RESOLUTION_TIMEOUT_MS, intValue); long longBytesValue = sDefaultConfiguration.getBytes(Constants.USER_BLOCK_REMOTE_READ_BUFFER_SIZE_BYTES); Assert.assertEquals(Constants.MB * 8, longBytesValue); int maxTry = sDefaultConfiguration.getInt(Constants.ZOOKEEPER_LEADER_INQUIRY_RETRY_COUNT); Assert.assertEquals(10, maxTry); }
/** * Tests writing to a file for longer than HEARTBEAT_INTERVAL_MS to make sure the sessionId * doesn't change. Tracks [ALLUXIO-171]. */ @Test public void longWrite() throws Exception { AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath()); final int length = 2; FileOutStream os = mFileSystem.createFile( filePath, CreateFileOptions.defaults().setWriteType(WriteType.THROUGH)); os.write((byte) 0); Thread.sleep(Configuration.getInt(PropertyKey.USER_HEARTBEAT_INTERVAL_MS) * 2); os.write((byte) 1); os.close(); checkWrite(filePath, UnderStorageType.SYNC_PERSIST, length, length); }
@Override public void start(boolean isLeader) throws IOException { super.start(isLeader); mGlobalStorageTierAssoc = new MasterStorageTierAssoc(); if (isLeader) { mLostWorkerDetectionService = getExecutorService() .submit( new HeartbeatThread( HeartbeatContext.MASTER_LOST_WORKER_DETECTION, new LostWorkerDetectionHeartbeatExecutor(), Configuration.getInt(Constants.MASTER_HEARTBEAT_INTERVAL_MS))); } }
@Override public void start() { mFilePersistenceService = getExecutorService() .submit( new HeartbeatThread( HeartbeatContext.WORKER_FILESYSTEM_MASTER_SYNC, new FileWorkerMasterSyncExecutor( mFileDataManager, mFileSystemMasterWorkerClient, mWorkerId), Configuration.getInt(PropertyKey.WORKER_FILESYSTEM_HEARTBEAT_INTERVAL_MS))); // Start the session cleanup checker to perform the periodical checking getExecutorService().submit(mSessionCleaner); }
@Override public void heartbeat() { int masterWorkerTimeoutMs = Configuration.getInt(Constants.MASTER_WORKER_TIMEOUT_MS); for (MasterWorkerInfo worker : mWorkers) { synchronized (worker) { final long lastUpdate = CommonUtils.getCurrentMs() - worker.getLastUpdatedTimeMs(); if (lastUpdate > masterWorkerTimeoutMs) { LOG.error( "The worker {} timed out after {}ms without a heartbeat!", worker, lastUpdate); mLostWorkers.add(worker); mWorkers.remove(worker); processWorkerRemovedBlocks(worker, worker.getBlocks()); } } } }
/** Tests the default properties for the master. */ @Test public void masterDefaultTest() { String alluxioHome = sDefaultConfiguration.get(Constants.HOME); Assert.assertNotNull(alluxioHome); Assert.assertEquals("/mnt/alluxio_default_home", alluxioHome); String value = sDefaultConfiguration.get(Constants.MASTER_JOURNAL_FOLDER); Assert.assertNotNull(value); Assert.assertEquals(alluxioHome + "/journal/", value); value = sDefaultConfiguration.get(Constants.MASTER_HOSTNAME); Assert.assertNotNull(value); Assert.assertEquals(NetworkAddressUtils.getLocalHostName(100), value); value = sDefaultConfiguration.get(Constants.MASTER_FORMAT_FILE_PREFIX); Assert.assertNotNull(value); Assert.assertEquals(Constants.FORMAT_FILE_PREFIX, value); value = sDefaultConfiguration.get(Constants.MASTER_ADDRESS); Assert.assertNotNull(value); value = sDefaultConfiguration.get(Constants.MASTER_BIND_HOST); Assert.assertNotNull(value); Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value); int intValue = sDefaultConfiguration.getInt(Constants.MASTER_RPC_PORT); Assert.assertEquals(19998, intValue); value = sDefaultConfiguration.get(Constants.MASTER_WEB_BIND_HOST); Assert.assertNotNull(value); Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value); intValue = sDefaultConfiguration.getInt(Constants.MASTER_WEB_PORT); Assert.assertEquals(19999, intValue); intValue = sDefaultConfiguration.getInt(Constants.WEB_THREAD_COUNT); Assert.assertEquals(1, intValue); intValue = sDefaultConfiguration.getInt(Constants.MASTER_HEARTBEAT_INTERVAL_MS); Assert.assertEquals(Constants.SECOND_MS, intValue); intValue = sDefaultConfiguration.getInt(Constants.MASTER_WORKER_THREADS_MIN); Assert.assertEquals(Runtime.getRuntime().availableProcessors(), intValue); intValue = sDefaultConfiguration.getInt(Constants.MASTER_WORKER_TIMEOUT_MS); Assert.assertEquals(10 * Constants.SECOND_MS, intValue); }
/** * Alluxio stores data into an under layer file system. Any file system implementing this interface * can be a valid under layer file system */ @ThreadSafe public abstract class UnderFileSystem { private static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_TYPE); /** The UFS {@link AlluxioURI} used to create this {@link UnderFileSystem}. */ protected final AlluxioURI mUri; /** A map of property names to values. */ protected HashMap<String, String> mProperties = new HashMap<>(); /** * This variable indicates whether the underFS actually provides storage. Most UnderFS should * provide storage, but a dummyFS for example does not. */ private boolean mProvidesStorage = true; /** Maximum length for a single listing query. */ private static final int MAX_LISTING_LENGTH = 1000; /** Length of each list request. */ protected static final int LISTING_LENGTH = Configuration.getInt(PropertyKey.UNDERFS_LISTING_LENGTH) > MAX_LISTING_LENGTH ? MAX_LISTING_LENGTH : Configuration.getInt(PropertyKey.UNDERFS_LISTING_LENGTH); private static final Cache UFS_CACHE = new Cache(); /** * The different types of space indicate the total space, the free space and the space used in the * under file system. */ public enum SpaceType { /** Indicates the storage capacity of the under file system. */ SPACE_TOTAL(0), /** Indicates the amount of free space available in the under file system. */ SPACE_FREE(1), /** Indicates the amount of space used in the under file system. */ SPACE_USED(2), ; private final int mValue; SpaceType(int value) { mValue = value; } /** @return the integer value of this enum value */ public int getValue() { return mValue; } } /** A class used to cache UnderFileSystems. */ @ThreadSafe private static class Cache { /** Maps from {@link Key} to {@link UnderFileSystem} instances. */ private final ConcurrentHashMap<Key, UnderFileSystem> mUnderFileSystemMap = new ConcurrentHashMap<>(); Cache() {} /** * Gets a UFS instance from the cache if exists. Otherwise, creates a new instance and adds that * to the cache. * * @param path the ufs path * @param ufsConf the ufs configuration * @return the UFS instance */ UnderFileSystem get(String path, Object ufsConf) { Key key = new Key(new AlluxioURI(path)); UnderFileSystem cachedFs = mUnderFileSystemMap.get(key); if (cachedFs != null) { return cachedFs; } UnderFileSystem fs = UnderFileSystemRegistry.create(path, ufsConf); cachedFs = mUnderFileSystemMap.putIfAbsent(key, fs); if (cachedFs == null) { return fs; } try { fs.close(); } catch (IOException e) { throw new RuntimeException(e); } return cachedFs; } void clear() { mUnderFileSystemMap.clear(); } } /** The key of the UFS cache. */ private static class Key { private final String mScheme; private final String mAuthority; Key(AlluxioURI uri) { mScheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase(); mAuthority = uri.getAuthority() == null ? "" : uri.getAuthority().toLowerCase(); } @Override public int hashCode() { return Objects.hashCode(mScheme, mAuthority); } @Override public boolean equals(Object object) { if (object == this) { return true; } if (!(object instanceof Key)) { return false; } Key that = (Key) object; return Objects.equal(mScheme, that.mScheme) && Objects.equal(mAuthority, that.mAuthority); } @Override public String toString() { return mScheme + "://" + mAuthority; } } /** * Gets the UnderFileSystem instance according to its schema. * * @param path the file path storing over the ufs * @return instance of the under layer file system */ public static UnderFileSystem get(String path) { return get(path, null); } /** * Gets the UnderFileSystem instance according to its scheme and configuration. * * @param path the file path storing over the ufs * @param ufsConf the configuration object for ufs only * @return instance of the under layer file system */ public static UnderFileSystem get(String path, Object ufsConf) { Preconditions.checkArgument(path != null, "path may not be null"); return UFS_CACHE.get(path, ufsConf); } /** Clears the under file system cache. */ public static void clearCache() { UFS_CACHE.clear(); } /** * Returns the name of the under filesystem implementation. * * <p>The name should be lowercase and not include any spaces, e.g. "hdfs", "s3". * * @return name of the under filesystem implementation */ public abstract String getUnderFSType(); /** * Checks whether the underFS provides storage. * * @return true if the under filesystem provides storage, false otherwise */ public boolean providesStorage() { return mProvidesStorage; } /** * Constructs an {@link UnderFileSystem}. * * @param uri the {@link AlluxioURI} used to create this ufs */ protected UnderFileSystem(AlluxioURI uri) { Preconditions.checkNotNull(uri); mUri = uri; } /** * Configures and updates the properties. For instance, this method can add new properties or * modify existing properties specified through {@link #setProperties(Map)}. * * <p>The default implementation is a no-op. This should be overridden if a subclass needs * additional functionality. * * @throws IOException if an error occurs during configuration */ public void configureProperties() throws IOException { // Default implementation does not update any properties. } /** * Takes any necessary actions required to establish a connection to the under file system from * the given master host e.g. logging in * * <p>Depending on the implementation this may be a no-op * * @param hostname the host that wants to connect to the under file system * @throws IOException if a non-Alluxio error occurs */ public abstract void connectFromMaster(String hostname) throws IOException; /** * Takes any necessary actions required to establish a connection to the under file system from * the given worker host e.g. logging in * * <p>Depending on the implementation this may be a no-op * * @param hostname the host that wants to connect to the under file system * @throws IOException if a non-Alluxio error occurs */ public abstract void connectFromWorker(String hostname) throws IOException; /** * Closes this under file system. * * @throws IOException if a non-Alluxio error occurs */ public abstract void close() throws IOException; /** * Creates a file in the under file system with the indicated name. * * @param path the file name * @return A {@code OutputStream} object * @throws IOException if a non-Alluxio error occurs */ public OutputStream create(String path) throws IOException { return create(path, new CreateOptions()); } /** * Creates a file in the under file system with the specified {@link CreateOptions}. * Implementations should make sure that the path under creation appears in listings only after a * successful close and that contents are written in its entirety or not at all. * * @param path the file name * @param options the options for create * @return A {@code OutputStream} object * @throws IOException if a non-Alluxio error occurs */ public OutputStream create(String path, CreateOptions options) throws IOException { if (!options.isEnsureAtomic()) { return createDirect(path, options); } return new NonAtomicFileOutputStream(path, options, this); } /** * Creates a file in the under file system with the specified {@link CreateOptions}. This stream * writes directly to the underlying storage without any atomicity guarantees. * * @param path the file name * @param options the options for create * @return A {@code OutputStream} object * @throws IOException if a non-Alluxio error occurs */ public abstract OutputStream createDirect(String path, CreateOptions options) throws IOException; /** * Deletes a file or folder from the under file system with the indicated name. * * @param path the file or folder name * @param recursive the boolean indicates whether we delete folder and its children * @return true if succeed, false otherwise * @throws IOException if a non-Alluxio error occurs */ public abstract boolean delete(String path, boolean recursive) throws IOException; /** * Checks if a file or folder exists in under file system. * * @param path the file name * @return true if succeed, false otherwise * @throws IOException if a non-Alluxio error occurs */ public abstract boolean exists(String path) throws IOException; /** * Gets the block size of a file in under file system, in bytes. * * @param path the file name * @return the block size in bytes * @throws IOException if a non-Alluxio error occurs */ public abstract long getBlockSizeByte(String path) throws IOException; /** * Gets the configuration object for UnderFileSystem. * * @return configuration object used for concrete ufs instance */ public abstract Object getConf(); /** * Gets the list of locations of the indicated path. * * @param path the file name * @return The list of locations * @throws IOException if a non-Alluxio error occurs */ public abstract List<String> getFileLocations(String path) throws IOException; /** * Gets the list of locations of the indicated path given its offset. * * @param path the file name * @param offset the offset in bytes * @return The list of locations * @throws IOException if a non-Alluxio error occurs */ public abstract List<String> getFileLocations(String path, long offset) throws IOException; /** * Gets the file size in bytes. * * @param path the file name * @return the file size in bytes * @throws IOException if a non-Alluxio error occurs */ public abstract long getFileSize(String path) throws IOException; /** * Gets the UTC time of when the indicated path was modified recently in ms. * * @param path the file or folder name * @return modification time in milliseconds * @throws IOException if a non-Alluxio error occurs */ public abstract long getModificationTimeMs(String path) throws IOException; /** @return the property map for this {@link UnderFileSystem} */ public Map<String, String> getProperties() { return Collections.unmodifiableMap(mProperties); } /** * Queries the under file system about the space of the indicated path (e.g., space left, space * used and etc). * * @param path the path to query * @param type the type of queries * @return The space in bytes * @throws IOException if a non-Alluxio error occurs */ public abstract long getSpace(String path, SpaceType type) throws IOException; /** * Checks if the indicated path is a file or not. * * @param path the path name * @return true if this is a file, false otherwise * @throws IOException if a non-Alluxio error occurs */ public abstract boolean isFile(String path) throws IOException; /** * Returns an array of strings naming the files and directories in the directory denoted by this * abstract pathname. * * <p>If this abstract pathname does not denote a directory, then this method returns {@code * null}. Otherwise an array of strings is returned, one for each file or directory in the * directory. Names denoting the directory itself and the directory's parent directory are not * included in the result. Each string is a file name rather than a complete path. * * <p>There is no guarantee that the name strings in the resulting array will appear in any * specific order; they are not, in particular, guaranteed to appear in alphabetical order. * * @param path the abstract pathname to list * @return An array of strings naming the files and directories in the directory denoted by this * abstract pathname. The array will be empty if the directory is empty. Returns {@code null} * if this abstract pathname does not denote a directory. * @throws IOException if a non-Alluxio error occurs */ public abstract String[] list(String path) throws IOException; /** * Returns an array of strings naming the files and directories in the directory denoted by this * abstract pathname, and all of its subdirectories. * * <p>If this abstract pathname does not denote a directory, then this method returns {@code * null}. Otherwise an array of strings is returned, one for each file or directory in the * directory and its subdirectories. Names denoting the directory itself and the directory's * parent directory are not included in the result. Each string is a path relative to the given * directory. * * <p>There is no guarantee that the name strings in the resulting array will appear in any * specific order; they are not, in particular, guaranteed to appear in alphabetical order. * * @param path the abstract pathname to list * @return An array of strings naming the files and directories in the directory denoted by this * abstract pathname and its subdirectories. The array will be empty if the directory is * empty. Returns {@code null} if this abstract pathname does not denote a directory. * @throws IOException if a non-Alluxio error occurs */ public String[] listRecursive(String path) throws IOException { // Clean the path by creating a URI and turning it back to a string AlluxioURI uri = new AlluxioURI(path); path = uri.toString(); List<String> returnPaths = new ArrayList<>(); Queue<String> pathsToProcess = new ArrayDeque<>(); // We call list initially, so we can return null if the path doesn't denote a directory String[] subpaths = list(path); if (subpaths == null) { return null; } else { for (String subp : subpaths) { pathsToProcess.add(PathUtils.concatPath(path, subp)); } } while (!pathsToProcess.isEmpty()) { String p = pathsToProcess.remove(); returnPaths.add(p.substring(path.length() + 1)); // Add all of its subpaths subpaths = list(p); if (subpaths != null) { for (String subp : subpaths) { pathsToProcess.add(PathUtils.concatPath(p, subp)); } } } return returnPaths.toArray(new String[returnPaths.size()]); } /** * Creates the directory named by this abstract pathname. If the folder already exists, the method * returns false. * * @param path the folder to create * @param createParent if true, the method creates any necessary but nonexistent parent * directories. Otherwise, the method does not create nonexistent parent directories * @return {@code true} if and only if the directory was created; {@code false} otherwise * @throws IOException if a non-Alluxio error occurs */ public abstract boolean mkdirs(String path, boolean createParent) throws IOException; /** * Creates the directory named by this abstract pathname, with specified {@link MkdirsOptions}. If * the folder already exists, the method returns false. * * @param path the folder to create * @param options the options for mkdirs * @return {@code true} if and only if the directory was created; {@code false} otherwise * @throws IOException if a non-Alluxio error occurs */ public abstract boolean mkdirs(String path, MkdirsOptions options) throws IOException; /** * Opens an {@link InputStream} at the indicated path. * * @param path the file name * @return The {@code InputStream} object * @throws IOException if a non-Alluxio error occurs */ public abstract InputStream open(String path) throws IOException; /** * Renames a file or folder from {@code src} to {@code dst} in under file system. * * @param src the source file or folder name * @param dst the destination file or folder name * @return true if succeed, false otherwise * @throws IOException if a non-Alluxio error occurs */ public abstract boolean rename(String src, String dst) throws IOException; /** * Returns an {@link AlluxioURI} representation for the {@link UnderFileSystem} given a base UFS * URI, and the Alluxio path from the base. * * <p>The default implementation simply concatenates the path to the base URI. This should be * overridden if a subclass needs alternate functionality. * * @param ufsBaseUri the base {@link AlluxioURI} in the ufs * @param alluxioPath the path in Alluxio from the given base * @return the UFS {@link AlluxioURI} representing the Alluxio path */ public AlluxioURI resolveUri(AlluxioURI ufsBaseUri, String alluxioPath) { return new AlluxioURI( ufsBaseUri.getScheme(), ufsBaseUri.getAuthority(), PathUtils.concatPath(ufsBaseUri.getPath(), alluxioPath), ufsBaseUri.getQueryMap()); } /** * Sets the configuration object for UnderFileSystem. The conf object is understood by the * concrete underfs's implementation. * * @param conf the configuration object accepted by ufs */ public abstract void setConf(Object conf); /** * Sets the user and group of the given path. An empty implementation should be provided if * unsupported. * * @param path the path of the file * @param owner the new owner to set, unchanged if null * @param group the new group to set, unchanged if null * @throws IOException if a non-Alluxio error occurs */ public abstract void setOwner(String path, String owner, String group) throws IOException; /** * Sets the properties for this {@link UnderFileSystem}. * * @param properties a {@link Map} of property names to values */ public void setProperties(Map<String, String> properties) { mProperties.clear(); mProperties.putAll(properties); } /** * Changes posix file mode. * * @param path the path of the file * @param mode the mode to set in short format, e.g. 0777 * @throws IOException if a non-Alluxio error occurs */ public abstract void setMode(String path, short mode) throws IOException; /** * Gets the owner of the given path. An empty implementation should be provided if not supported. * * @param path the path of the file * @return the owner of the file * @throws IOException if a non-Alluxio error occurs */ public abstract String getOwner(String path) throws IOException; /** * Gets the group of the given path. An empty implementation should be provided if not supported. * * @param path the path of the file * @return the group of the file * @throws IOException if a non-Alluxio error occurs */ public abstract String getGroup(String path) throws IOException; /** * Gets the mode of the given path in short format, e.g 0700. An empty implementation should be * provided if not supported. * * @param path the path of the file * @return the mode of the file * @throws IOException if a non-Alluxio error occurs */ public abstract short getMode(String path) throws IOException; /** * Whether this type of UFS supports flush. * * @return true if this type of UFS supports flush, false otherwise */ public boolean supportsFlush() { return true; } }
/** * A netty packet reader that streams a region from a netty data server. * * <p>Protocol: 1. The client sends a read request (id, offset, length). 2. Once the server receives * the request, it streams packets to the client. The streaming pauses if the server's buffer is * full and resumes if the buffer is not full. 3. The client reads packets from the stream. Reading * pauses if the client buffer is full and resumes if the buffer is not full. If the client can keep * up with network speed, the buffer should have at most one packet. 4. The client stops reading if * it receives an empty packet which signifies the end of the stream. 5. The client can cancel the * read request at anytime. The cancel request is ignored by the server if everything has been sent * to channel. 6. If the client wants to reuse the channel, the client must read all the packets in * the channel before releasing the channel to the channel pool. 7. To make it simple to handle * errors, the channel is closed if any error occurs. */ @NotThreadSafe public final class NettyPacketReader implements PacketReader { private static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_TYPE); private static final boolean CANCEL_ENABLED = Configuration.getBoolean(PropertyKey.USER_NETWORK_NETTY_READER_CANCEL_ENABLED); private static final int MAX_PACKETS_IN_FLIGHT = Configuration.getInt(PropertyKey.USER_NETWORK_NETTY_READER_BUFFER_SIZE_PACKETS); private static final long READ_TIMEOUT_MS = Configuration.getLong(PropertyKey.USER_NETWORK_NETTY_TIMEOUT_MS); private final FileSystemContext mContext; private final Channel mChannel; private final Protocol.RequestType mRequestType; private final InetSocketAddress mAddress; private final long mId; private final long mStart; private final long mBytesToRead; // TODO(peis): Investigate whether we can remove this lock. The main reason to keep this lock // is to protect mPacketReaderException. private final ReentrantLock mLock = new ReentrantLock(); @GuardedBy("mLock") private final Queue<ByteBuf> mPackets = new LinkedList<>(); @GuardedBy("mLock") private Throwable mPacketReaderException = null; /** The condition is met when mPackets.size() > 0 or mPacketReaderException != null. */ private final Condition mNotEmptyOrFailed = mLock.newCondition(); /** The next pos to read. */ private long mPosToRead; /** This is true only when an empty packet is received. */ private boolean mDone = false; private boolean mClosed = false; /** * Creates an instance of {@link NettyPacketReader}. If this is used to read a block remotely, it * requires the block to be locked beforehand and the lock ID is passed to this class. * * @param context the file system context * @param address the netty data server network address * @param id the block ID or UFS file ID * @param offset the offset * @param len the length to read * @param lockId the lock ID * @param sessionId the session ID * @param type the request type (block or UFS file) * @throws IOException if it fails to acquire a netty channel */ private NettyPacketReader( FileSystemContext context, InetSocketAddress address, long id, long offset, long len, long lockId, long sessionId, Protocol.RequestType type) throws IOException { Preconditions.checkArgument(offset >= 0 && len > 0); mContext = context; mAddress = address; mId = id; mStart = offset; mPosToRead = offset; mBytesToRead = len; mRequestType = type; mChannel = context.acquireNettyChannel(address); ChannelPipeline pipeline = mChannel.pipeline(); if (!(pipeline.last() instanceof RPCMessageDecoder)) { throw new RuntimeException( String.format( "Channel pipeline has unexpected handlers %s.", pipeline.last().getClass().getCanonicalName())); } mChannel.pipeline().addLast(new PacketReadHandler()); Protocol.ReadRequest readRequest = Protocol.ReadRequest.newBuilder() .setId(id) .setOffset(offset) .setLength(len) .setLockId(lockId) .setSessionId(sessionId) .setType(type) .build(); mChannel .writeAndFlush(new RPCProtoMessage(readRequest)) .addListener(ChannelFutureListener.CLOSE_ON_FAILURE); } @Override public long pos() { return mPosToRead; } @Override public ByteBuf readPacket() throws IOException { Preconditions.checkState(!mClosed, "PacketReader is closed while reading packets."); ByteBuf buf = null; mLock.lock(); try { while (true) { if (mDone) { return null; } if (mPacketReaderException != null) { throw new IOException(mPacketReaderException); } buf = mPackets.poll(); // TODO(peis): Have a better criteria to resume so that we can have fewer state changes. if (!tooManyPacketsPending()) { resume(); } // Queue is empty. if (buf == null) { try { if (!mNotEmptyOrFailed.await(READ_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { throw new IOException( String.format("Timeout while reading packet from block %d @ %s.", mId, mAddress)); } } catch (InterruptedException e) { throw Throwables.propagate(e); } } else { if (buf.readableBytes() == 0) { buf.release(); mDone = true; return null; } mPosToRead += buf.readableBytes(); Preconditions.checkState(mPosToRead - mStart <= mBytesToRead); return buf; } } } catch (Throwable e) { if (buf != null) { buf.release(); } throw e; } finally { mLock.unlock(); } } @Override public void close() { try { if (mDone) { return; } if (!mChannel.isOpen()) { return; } try { if (!CANCEL_ENABLED) { mChannel.close().sync(); return; } if (remaining() > 0) { Protocol.ReadRequest cancelRequest = Protocol.ReadRequest.newBuilder() .setId(mId) .setCancel(true) .setType(mRequestType) .build(); mChannel .writeAndFlush(new RPCProtoMessage(cancelRequest)) .addListener(ChannelFutureListener.CLOSE_ON_FAILURE); } } catch (InterruptedException e) { mChannel.close(); throw Throwables.propagate(e); } while (true) { try { ByteBuf buf = readPacket(); // A null packet indicates the end of the stream. if (buf == null) { return; } buf.release(); } catch (IOException e) { LOG.warn( "Failed to close the NettyBlockReader (block: {}, address: {}).", mId, mAddress, e); try { mChannel.close().sync(); } catch (InterruptedException ee) { throw Throwables.propagate(ee); } return; } } } finally { if (mChannel.isOpen()) { Preconditions.checkState(mChannel.pipeline().last() instanceof PacketReadHandler); mChannel.pipeline().removeLast(); // Make sure "autoread" is on before realsing the channel. resume(); } mContext.releaseNettyChannel(mAddress, mChannel); mClosed = true; } } /** @return bytes remaining */ private long remaining() { return mStart + mBytesToRead - mPosToRead; } /** @return true if there are too many packets pending */ private boolean tooManyPacketsPending() { return mPackets.size() >= MAX_PACKETS_IN_FLIGHT; } /** The netty handler that reads packets from the channel. */ private class PacketReadHandler extends ChannelInboundHandlerAdapter { /** Default constructor. */ public PacketReadHandler() {} @Override public void channelRead(ChannelHandlerContext ctx, Object msg) { Preconditions.checkState( acceptMessage(msg), "Incorrect response type %s, %s.", msg.getClass().getCanonicalName(), msg); RPCProtoMessage response = (RPCProtoMessage) msg; Protocol.Status status = ((Protocol.Response) response.getMessage()).getStatus(); if (!Status.isOk(status)) { ctx.fireExceptionCaught( new IOException( String.format( "Failed to read block %d from %s with status %s.", mId, mAddress, status.toString()))); } mLock.lock(); try { Preconditions.checkState(mPacketReaderException == null); DataBuffer dataBuffer = response.getPayloadDataBuffer(); ByteBuf buf; if (dataBuffer == null) { buf = ctx.alloc().buffer(0, 0); } else { Preconditions.checkState(dataBuffer.getLength() > 0); assert dataBuffer.getNettyOutput() instanceof ByteBuf; buf = (ByteBuf) dataBuffer.getNettyOutput(); } mPackets.offer(buf); mNotEmptyOrFailed.signal(); if (tooManyPacketsPending()) { pause(); } } finally { mLock.unlock(); } } @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { LOG.error("Exception caught while reading response from netty channel.", cause); mLock.lock(); try { mPacketReaderException = cause; mNotEmptyOrFailed.signal(); } finally { mLock.unlock(); } ctx.close(); } @Override public void channelUnregistered(ChannelHandlerContext ctx) { mLock.lock(); try { if (mPacketReaderException == null) { mPacketReaderException = new IOException("ChannelClosed"); } mNotEmptyOrFailed.signal(); } finally { mLock.unlock(); } ctx.fireChannelUnregistered(); } /** * @param msg the message received * @return true if this message should be processed */ private boolean acceptMessage(Object msg) { if (msg instanceof RPCProtoMessage) { MessageLite header = ((RPCProtoMessage) msg).getMessage(); return header instanceof Protocol.Response; } return false; } } /** Pauses the underlying packet reader. */ private void pause() { mChannel.config().setAutoRead(false); } /** Resumes the underlying packet reader. */ private void resume() { mChannel.config().setAutoRead(true); mChannel.read(); } /** Factory class to create {@link NettyPacketReader}s. */ public static class Factory implements PacketReader.Factory { private final FileSystemContext mContext; private final InetSocketAddress mAddress; private final long mId; private final long mLockId; private final long mSessionId; private final Protocol.RequestType mRequestType; /** * Creates an instance of {@link NettyPacketReader.Factory} for block reads. * * @param context the file system context * @param address the worker address * @param id the block ID or UFS ID * @param lockId the lock ID * @param sessionId the session ID * @param type the request type */ public Factory( FileSystemContext context, InetSocketAddress address, long id, long lockId, long sessionId, Protocol.RequestType type) { mContext = context; mAddress = address; mId = id; mLockId = lockId; mSessionId = sessionId; mRequestType = type; } @Override public PacketReader create(long offset, long len) throws IOException { return new NettyPacketReader( mContext, mAddress, mId, offset, len, mLockId, mSessionId, mRequestType); } } }
@Override public void resourceOffers(SchedulerDriver driver, List<Protos.Offer> offers) { long masterCpu = Configuration.getInt(PropertyKey.INTEGRATION_MASTER_RESOURCE_CPU); long masterMem = Configuration.getBytes(PropertyKey.INTEGRATION_MASTER_RESOURCE_MEM) / Constants.MB; long workerCpu = Configuration.getInt(PropertyKey.INTEGRATION_WORKER_RESOURCE_CPU); long workerMem = Configuration.getBytes(PropertyKey.INTEGRATION_WORKER_RESOURCE_MEM) / Constants.MB; LOG.info( "Master launched {}, master count {}, " + "requested master cpu {} mem {} MB and required master hostname {}", mMasterLaunched, mMasterCount, masterCpu, masterMem, mRequiredMasterHostname); for (Protos.Offer offer : offers) { Protos.Offer.Operation.Launch.Builder launch = Protos.Offer.Operation.Launch.newBuilder(); double offerCpu = 0; double offerMem = 0; for (Protos.Resource resource : offer.getResourcesList()) { if (resource.getName().equals(Constants.MESOS_RESOURCE_CPUS)) { offerCpu += resource.getScalar().getValue(); } else if (resource.getName().equals(Constants.MESOS_RESOURCE_MEM)) { offerMem += resource.getScalar().getValue(); } else { // Other resources are currently ignored. } } LOG.info( "Received offer {} on host {} with cpus {} and mem {} MB and hasMasterPorts {}", offer.getId().getValue(), offer.getHostname(), offerCpu, offerMem, OfferUtils.hasAvailableMasterPorts(offer)); Protos.ExecutorInfo.Builder executorBuilder = Protos.ExecutorInfo.newBuilder(); List<Protos.Resource> resources; if (!mMasterLaunched && offerCpu >= masterCpu && offerMem >= masterMem && mMasterCount < Configuration.getInt(PropertyKey.INTEGRATION_MESOS_ALLUXIO_MASTER_NODE_COUNT) && OfferUtils.hasAvailableMasterPorts(offer) && (mRequiredMasterHostname == null || mRequiredMasterHostname.equals(offer.getHostname()))) { LOG.debug("Creating Alluxio Master executor"); executorBuilder .setName("Alluxio Master Executor") .setSource("master") .setExecutorId(Protos.ExecutorID.newBuilder().setValue("master")) .addAllResources(getExecutorResources()) .setCommand( Protos.CommandInfo.newBuilder() .setValue(createStartAlluxioCommand("alluxio-master-mesos.sh")) .addAllUris(getExecutorDependencyURIList()) .setEnvironment( Protos.Environment.newBuilder() .addVariables( Protos.Environment.Variable.newBuilder() .setName("ALLUXIO_UNDERFS_ADDRESS") .setValue(Configuration.get(PropertyKey.UNDERFS_ADDRESS)) .build()) .build())); // pre-build resource list here, then use it to build Protos.Task later. resources = getMasterRequiredResources(masterCpu, masterMem); mMasterHostname = offer.getHostname(); mTaskName = Configuration.get(PropertyKey.INTEGRATION_MESOS_ALLUXIO_MASTER_NAME); mMasterCount++; mMasterTaskId = mLaunchedTasks; } else if (mMasterLaunched && !mWorkers.contains(offer.getHostname()) && offerCpu >= workerCpu && offerMem >= workerMem && OfferUtils.hasAvailableWorkerPorts(offer)) { LOG.debug("Creating Alluxio Worker executor"); final String memSize = FormatUtils.getSizeFromBytes((long) workerMem * Constants.MB); executorBuilder .setName("Alluxio Worker Executor") .setSource("worker") .setExecutorId(Protos.ExecutorID.newBuilder().setValue("worker")) .addAllResources(getExecutorResources()) .setCommand( Protos.CommandInfo.newBuilder() .setValue(createStartAlluxioCommand("alluxio-worker-mesos.sh")) .addAllUris(getExecutorDependencyURIList()) .setEnvironment( Protos.Environment.newBuilder() .addVariables( Protos.Environment.Variable.newBuilder() .setName("ALLUXIO_MASTER_HOSTNAME") .setValue(mMasterHostname) .build()) .addVariables( Protos.Environment.Variable.newBuilder() .setName("ALLUXIO_WORKER_MEMORY_SIZE") .setValue(memSize) .build()) .addVariables( Protos.Environment.Variable.newBuilder() .setName("ALLUXIO_UNDERFS_ADDRESS") .setValue(Configuration.get(PropertyKey.UNDERFS_ADDRESS)) .build()) .build())); // pre-build resource list here, then use it to build Protos.Task later. resources = getWorkerRequiredResources(workerCpu, workerMem); mWorkers.add(offer.getHostname()); mTaskName = Configuration.get(PropertyKey.INTEGRATION_MESOS_ALLUXIO_WORKER_NAME); } else { // The resource offer cannot be used to start either master or a worker. LOG.info("Declining offer {}", offer.getId().getValue()); driver.declineOffer(offer.getId()); continue; } Protos.TaskID taskId = Protos.TaskID.newBuilder().setValue(String.valueOf(mLaunchedTasks)).build(); LOG.info("Launching task {} using offer {}", taskId.getValue(), offer.getId().getValue()); Protos.TaskInfo task = Protos.TaskInfo.newBuilder() .setName(mTaskName) .setTaskId(taskId) .setSlaveId(offer.getSlaveId()) .addAllResources(resources) .setExecutor(executorBuilder) .build(); launch.addTaskInfos(Protos.TaskInfo.newBuilder(task)); mLaunchedTasks++; // NOTE: We use the new API `acceptOffers` here to launch tasks. // The 'launchTasks' API will be deprecated. List<Protos.OfferID> offerIds = new ArrayList<Protos.OfferID>(); offerIds.add(offer.getId()); List<Protos.Offer.Operation> operations = new ArrayList<Protos.Offer.Operation>(); Protos.Offer.Operation operation = Protos.Offer.Operation.newBuilder() .setType(Protos.Offer.Operation.Type.LAUNCH) .setLaunch(launch) .build(); operations.add(operation); Protos.Filters filters = Protos.Filters.newBuilder().setRefuseSeconds(1).build(); driver.acceptOffers(offerIds, operations, filters); } }