/**
* Gets a file status.
*
* @param cache FileInfo cache.
* @param key the key in the cache.
* @param fileId the id of the queried file. If it is -1, uses path.
* @param path the path of the queried file. If fielId is not -1, this parameter is ignored.
* @param useCachedMetaData whether to use the cached data or not.
* @return the clientFileInfo.
* @throws IOException if the underlying master RPC fails
*/
private synchronized <K> FileInfo getFileStatus(
Map<K, FileInfo> cache, K key, long fileId, String path, boolean useCachedMetaData)
throws IOException {
FileInfo info = null;
if (useCachedMetaData) {
info = cache.get(key);
if (info != null) {
return info;
}
}
if (fileId == -1) {
try {
fileId = mFSMasterClient.getFileId(path);
} catch (InvalidPathException e) {
throw new IOException(e);
}
}
if (fileId == -1) {
cache.remove(key);
return null;
}
try {
info = mFSMasterClient.getFileInfo(fileId);
} catch (FileDoesNotExistException e) {
throw new IOException(e);
}
path = info.getPath();
// TODO(hy): LRU
mIdToClientFileInfo.put(fileId, info);
mPathToClientFileInfo.put(path, info);
return info;
}
private long getNextBlockId() throws IOException {
FileSystemMasterClient masterClient = mContext.acquireMasterClient();
try {
return masterClient.getNewBlockIdForFile(mUri);
} catch (AlluxioException e) {
throw new IOException(e);
} finally {
mContext.releaseMasterClient(masterClient);
}
}
/**
* Schedules the async persistence of the current file.
*
* @throws IOException an I/O error occurs
*/
protected void scheduleAsyncPersist() throws IOException {
FileSystemMasterClient masterClient = mContext.acquireMasterClient();
try {
masterClient.scheduleAsyncPersist(mUri);
} catch (AlluxioException e) {
throw new IOException(e);
} finally {
mContext.releaseMasterClient(masterClient);
}
}
private void updateUfsPath() throws IOException {
FileSystemMasterClient client = mContext.acquireMasterClient();
try {
URIStatus status = client.getStatus(mUri);
mUfsPath = status.getUfsPath();
} catch (AlluxioException e) {
throw new IOException(e);
} finally {
mContext.releaseMasterClient(client);
}
}
/**
* Creates a new file in the file system.
*
* @param path The path of the file
* @param ufsPath The path of the file in the under file system. If this is empty, the file does
* not exist in the under file system yet.
* @param blockSizeByte The size of the block in bytes. It is -1 iff ufsPath is non-empty.
* @param recursive Creates necessary parent folders if true, not otherwise.
* @return The file id, which is globally unique.
* @throws IOException if the underlying master RPC fails
*/
@Override
public synchronized long createFile(
TachyonURI path, TachyonURI ufsPath, long blockSizeByte, boolean recursive)
throws IOException {
validateUri(path);
try {
if (blockSizeByte > 0) {
return mFSMasterClient.createFile(path.getPath(), blockSizeByte, recursive);
} else {
return mFSMasterClient.loadFileInfoFromUfs(path.getPath(), ufsPath.toString(), recursive);
}
} catch (TException e) {
throw new IOException(e);
}
}
/**
* Marks the file as complete.
*
* @param fid the file id
* @throws IOException if the underlying master RPC fails
*/
synchronized void completeFile(long fid) throws IOException {
try {
mFSMasterClient.completeFile(fid);
} catch (TException e) {
throw new IOException(e);
}
}
/** Stops the lineage worker service. */
public void stop() {
if (mFilePersistenceService != null) {
mFilePersistenceService.cancel(true);
}
mFileSystemMasterWorkerClient.close();
getExecutorService().shutdown();
}
/**
* Requests the dependency's needed files.
*
* @param depId the dependency id
* @throws IOException if the underlying master RPC fails
*/
public synchronized void requestFilesInDependency(int depId) throws IOException {
try {
mFSMasterClient.requestFilesInDependency(depId);
} catch (DependencyDoesNotExistException e) {
throw new IOException(e);
}
}
/**
* Sets the "pinned" flag for the given file. Pinned files are never evicted by Tachyon until they
* are unpinned.
*
* <p>Calling setPinned() on a folder will recursively set the "pinned" flag on all of that
* folder's children. This may be an expensive operation for folders with many files/subfolders.
*
* @param fid the file id
* @param pinned the target "pinned" flag value
* @throws IOException if the underlying master RPC fails
*/
public synchronized void setPinned(long fid, boolean pinned) throws IOException {
try {
mFSMasterClient.setPinned(fid, pinned);
} catch (FileDoesNotExistException e) {
throw new IOException(e);
}
}
/**
* Reports the lost file to master.
*
* @param fileId the lost file id
* @throws IOException if the underlying master RPC fails
*/
public synchronized void reportLostFile(long fileId) throws IOException {
try {
mFSMasterClient.reportLostFile(fileId);
} catch (FileDoesNotExistException e) {
throw new IOException(e);
}
}
/**
* Frees an in-memory file or folder.
*
* @param fileId The id of the file / folder. If it is not -1, path parameter is ignored.
* Otherwise, the method uses the path parameter.
* @param path The path of the file / folder. It could be empty iff id is not -1.
* @param recursive If fileId or path represents a non-empty folder, free the folder recursively
* or not
* @return true if in-memory free successfully, false otherwise.
* @throws IOException if the underlying master RPC fails
*/
@Override
public synchronized boolean freepath(long fileId, TachyonURI path, boolean recursive)
throws IOException {
validateUri(path);
if (fileId == -1) {
try {
fileId = mFSMasterClient.getFileId(path.getPath());
} catch (InvalidPathException e) {
throw new IOException(e);
}
}
try {
return mFSMasterClient.free(fileId, recursive);
} catch (FileDoesNotExistException e) {
throw new IOException(e);
}
}
/**
* Renames a file or folder to the indicated new path.
*
* @param fileId The id of the source file / folder. If it is not -1, path parameter is ignored.
* Otherwise, the method uses the srcPath parameter.
* @param srcPath The path of the source file / folder. It could be empty iff id is not -1.
* @param dstPath The path of the destination file / folder. It could be empty iff id is not -1.
* @return true if renames successfully, false otherwise.
* @throws IOException if the underlying master RPC fails
*/
@Override
public synchronized boolean rename(long fileId, TachyonURI srcPath, TachyonURI dstPath)
throws IOException {
validateUri(srcPath);
validateUri(dstPath);
if (fileId == -1) {
try {
fileId = mFSMasterClient.getFileId(srcPath.getPath());
} catch (InvalidPathException e) {
throw new IOException(e);
}
}
try {
return mFSMasterClient.renameFile(fileId, dstPath.getPath());
} catch (FileDoesNotExistException e) {
throw new IOException(e);
}
}
/**
* Creates a folder.
*
* @param path the path of the folder to be created
* @param recursive Creates necessary parent folders if true, not otherwise.
* @return true if the folder is created successfully or already existing. false otherwise.
* @throws IOException if the underlying master RPC fails
*/
@Override
public synchronized boolean mkdirs(TachyonURI path, boolean recursive) throws IOException {
validateUri(path);
try {
return mFSMasterClient.createDirectory(path.getPath(), recursive);
} catch (TException e) {
throw new IOException(e);
}
}
/**
* If the <code>path</code> is a directory, returns all the direct entries in it. If the <code>
* path</code> is a file, returns its ClientFileInfo.
*
* @param path the target directory/file path
* @return A list of FileInfo, null if the file or folder does not exist.
* @throws IOException when the underlying master RPC fails
*/
@Override
public synchronized List<FileInfo> listStatus(TachyonURI path) throws IOException {
validateUri(path);
try {
return mFSMasterClient.getFileInfoList(getFileStatus(-1, path).getFileId());
} catch (FileDoesNotExistException e) {
throw new IOException(e);
}
}
/**
* Gets all the blocks' info of the file.
*
* @param fid the file id
* @return the list of the blocks' info
* @throws IOException if the underlying master RPC fails
*/
public synchronized List<FileBlockInfo> getFileBlocks(long fid) throws IOException {
// TODO(hy) Should read from mClientFileInfos if possible.
// TODO(hy) Should add timeout to improve this.
try {
return mFSMasterClient.getFileBlockInfoList(fid);
} catch (FileDoesNotExistException e) {
throw new IOException(e);
}
}
@Override
public void stop() {
mSessionCleaner.stop();
if (mFilePersistenceService != null) {
mFilePersistenceService.cancel(true);
}
mFileSystemMasterWorkerClient.close();
// This needs to be shutdownNow because heartbeat threads will only stop when interrupted.
getExecutorService().shutdownNow();
}
/**
* Advanced API.
*
* <p>Gets the FileInfo object that represents the fileId, or the path if fileId is -1.
*
* @param fileId the file id of the file or folder.
* @param path the path of the file or folder. valid iff fileId is -1.
* @param useCachedMetadata if true use the local cached meta data
* @return the FileInfo of the file. null if the file does not exist.
* @throws IOException if the underlying master RPC fails
*/
public synchronized FileInfo getFileStatus(
long fileId, TachyonURI path, boolean useCachedMetadata) throws IOException {
if (fileId == -1) {
try {
fileId = mFSMasterClient.getFileId(path.getPath());
} catch (InvalidPathException e) {
throw new IOException(e);
}
}
return getFileStatus(
mIdToClientFileInfo, fileId, fileId, TachyonURI.EMPTY_URI.getPath(), useCachedMetadata);
}
/**
* Get the block id by the file id and block index. it will check whether the file and the block
* exist.
*
* @param fileId the file id
* @param blockIndex The index of the block in the file.
* @return the block id if exists
* @throws IOException if the file does not exist, or connection issue was encountered
*/
public synchronized long getBlockId(long fileId, int blockIndex) throws IOException {
FileInfo info = getFileStatus(fileId, true);
if (info == null) {
throw new IOException("File " + fileId + " does not exist.");
}
if (info.blockIds.size() > blockIndex) {
return info.blockIds.get(blockIndex);
}
try {
return mFSMasterClient.getFileBlockInfo(fileId, blockIndex).blockInfo.getBlockId();
} catch (TException e) {
throw new IOException(e);
}
}
/**
* Creates a dependency.
*
* @param parents the dependency's input files
* @param children the dependency's output files
* @param commandPrefix identifies a command prefix
* @param data stores dependency data
* @param comment records a dependency comment
* @param framework identifies the framework
* @param frameworkVersion identifies the framework version
* @param dependencyType the dependency's type, Wide or Narrow
* @param childrenBlockSizeByte the block size of the dependency's output files
* @return the dependency's id
* @throws IOException if the underlying master RPC fails
*/
public synchronized int createDependency(
List<String> parents,
List<String> children,
String commandPrefix,
List<ByteBuffer> data,
String comment,
String framework,
String frameworkVersion,
int dependencyType,
long childrenBlockSizeByte)
throws IOException {
return mFSMasterClient.user_createDependency(
parents,
children,
commandPrefix,
data,
comment,
framework,
frameworkVersion,
dependencyType,
childrenBlockSizeByte);
}
/**
* Checks if this client is connected to master.
*
* @return true if this client is connected to master, false otherwise
*/
public synchronized boolean isConnected() {
return mFSMasterClient.isConnected();
}
/**
* @param depId the dependency id
* @return the DependencyInfo of the specified dependency
* @throws IOException if the underlying master RPC fails
*/
public synchronized DependencyInfo getClientDependencyInfo(int depId) throws IOException {
return mFSMasterClient.getDependencyInfo(depId);
}
@Override
public void close() throws IOException {
if (mClosed) {
return;
}
if (mCurrentBlockOutStream != null) {
mPreviousBlockOutStreams.add(mCurrentBlockOutStream);
}
CompleteFileOptions options = CompleteFileOptions.defaults();
if (mUnderStorageType.isSyncPersist()) {
String tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
UnderFileSystem ufs = UnderFileSystem.get(tmpPath, ClientContext.getConf());
if (mCanceled) {
// TODO(yupeng): Handle this special case in under storage integrations.
mUnderStorageOutputStream.close();
if (!ufs.exists(tmpPath)) {
// Location of the temporary file has changed, recompute it.
updateUfsPath();
tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
}
ufs.delete(tmpPath, false);
} else {
mUnderStorageOutputStream.flush();
mUnderStorageOutputStream.close();
if (!ufs.exists(tmpPath)) {
// Location of the temporary file has changed, recompute it.
updateUfsPath();
tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
}
if (!ufs.rename(tmpPath, mUfsPath)) {
throw new IOException("Failed to rename " + tmpPath + " to " + mUfsPath);
}
options.setUfsLength(ufs.getFileSize(mUfsPath));
}
}
if (mAlluxioStorageType.isStore()) {
try {
if (mCanceled) {
for (BufferedBlockOutStream bos : mPreviousBlockOutStreams) {
bos.cancel();
}
} else {
for (BufferedBlockOutStream bos : mPreviousBlockOutStreams) {
bos.close();
}
}
} catch (IOException e) {
handleCacheWriteException(e);
}
}
// Complete the file if it's ready to be completed.
if (!mCanceled && (mUnderStorageType.isSyncPersist() || mAlluxioStorageType.isStore())) {
FileSystemMasterClient masterClient = mContext.acquireMasterClient();
try {
masterClient.completeFile(mUri, options);
} catch (AlluxioException e) {
throw new IOException(e);
} finally {
mContext.releaseMasterClient(masterClient);
}
}
if (mUnderStorageType.isAsyncPersist()) {
scheduleAsyncPersist();
}
mClosed = true;
}
/**
* @return the address of the UnderFileSystem
* @throws IOException if the underlying master RPC fails
*/
public synchronized String getUfsAddress() throws IOException {
return mFSMasterClient.getUfsAddress();
}
