/** * Gets a file status. * * @param cache FileInfo cache. * @param key the key in the cache. * @param fileId the id of the queried file. If it is -1, uses path. * @param path the path of the queried file. If fielId is not -1, this parameter is ignored. * @param useCachedMetaData whether to use the cached data or not. * @return the clientFileInfo. * @throws IOException if the underlying master RPC fails */ private synchronized <K> FileInfo getFileStatus( Map<K, FileInfo> cache, K key, long fileId, String path, boolean useCachedMetaData) throws IOException { FileInfo info = null; if (useCachedMetaData) { info = cache.get(key); if (info != null) { return info; } } if (fileId == -1) { try { fileId = mFSMasterClient.getFileId(path); } catch (InvalidPathException e) { throw new IOException(e); } } if (fileId == -1) { cache.remove(key); return null; } try { info = mFSMasterClient.getFileInfo(fileId); } catch (FileDoesNotExistException e) { throw new IOException(e); } path = info.getPath(); // TODO(hy): LRU mIdToClientFileInfo.put(fileId, info); mPathToClientFileInfo.put(path, info); return info; }
private long getNextBlockId() throws IOException { FileSystemMasterClient masterClient = mContext.acquireMasterClient(); try { return masterClient.getNewBlockIdForFile(mUri); } catch (AlluxioException e) { throw new IOException(e); } finally { mContext.releaseMasterClient(masterClient); } }
/** * Schedules the async persistence of the current file. * * @throws IOException an I/O error occurs */ protected void scheduleAsyncPersist() throws IOException { FileSystemMasterClient masterClient = mContext.acquireMasterClient(); try { masterClient.scheduleAsyncPersist(mUri); } catch (AlluxioException e) { throw new IOException(e); } finally { mContext.releaseMasterClient(masterClient); } }
private void updateUfsPath() throws IOException { FileSystemMasterClient client = mContext.acquireMasterClient(); try { URIStatus status = client.getStatus(mUri); mUfsPath = status.getUfsPath(); } catch (AlluxioException e) { throw new IOException(e); } finally { mContext.releaseMasterClient(client); } }
/** * Creates a new file in the file system. * * @param path The path of the file * @param ufsPath The path of the file in the under file system. If this is empty, the file does * not exist in the under file system yet. * @param blockSizeByte The size of the block in bytes. It is -1 iff ufsPath is non-empty. * @param recursive Creates necessary parent folders if true, not otherwise. * @return The file id, which is globally unique. * @throws IOException if the underlying master RPC fails */ @Override public synchronized long createFile( TachyonURI path, TachyonURI ufsPath, long blockSizeByte, boolean recursive) throws IOException { validateUri(path); try { if (blockSizeByte > 0) { return mFSMasterClient.createFile(path.getPath(), blockSizeByte, recursive); } else { return mFSMasterClient.loadFileInfoFromUfs(path.getPath(), ufsPath.toString(), recursive); } } catch (TException e) { throw new IOException(e); } }
/** * Marks the file as complete. * * @param fid the file id * @throws IOException if the underlying master RPC fails */ synchronized void completeFile(long fid) throws IOException { try { mFSMasterClient.completeFile(fid); } catch (TException e) { throw new IOException(e); } }
/** Stops the lineage worker service. */ public void stop() { if (mFilePersistenceService != null) { mFilePersistenceService.cancel(true); } mFileSystemMasterWorkerClient.close(); getExecutorService().shutdown(); }
/** * Requests the dependency's needed files. * * @param depId the dependency id * @throws IOException if the underlying master RPC fails */ public synchronized void requestFilesInDependency(int depId) throws IOException { try { mFSMasterClient.requestFilesInDependency(depId); } catch (DependencyDoesNotExistException e) { throw new IOException(e); } }
/** * Sets the "pinned" flag for the given file. Pinned files are never evicted by Tachyon until they * are unpinned. * * <p>Calling setPinned() on a folder will recursively set the "pinned" flag on all of that * folder's children. This may be an expensive operation for folders with many files/subfolders. * * @param fid the file id * @param pinned the target "pinned" flag value * @throws IOException if the underlying master RPC fails */ public synchronized void setPinned(long fid, boolean pinned) throws IOException { try { mFSMasterClient.setPinned(fid, pinned); } catch (FileDoesNotExistException e) { throw new IOException(e); } }
/** * Reports the lost file to master. * * @param fileId the lost file id * @throws IOException if the underlying master RPC fails */ public synchronized void reportLostFile(long fileId) throws IOException { try { mFSMasterClient.reportLostFile(fileId); } catch (FileDoesNotExistException e) { throw new IOException(e); } }
/** * Frees an in-memory file or folder. * * @param fileId The id of the file / folder. If it is not -1, path parameter is ignored. * Otherwise, the method uses the path parameter. * @param path The path of the file / folder. It could be empty iff id is not -1. * @param recursive If fileId or path represents a non-empty folder, free the folder recursively * or not * @return true if in-memory free successfully, false otherwise. * @throws IOException if the underlying master RPC fails */ @Override public synchronized boolean freepath(long fileId, TachyonURI path, boolean recursive) throws IOException { validateUri(path); if (fileId == -1) { try { fileId = mFSMasterClient.getFileId(path.getPath()); } catch (InvalidPathException e) { throw new IOException(e); } } try { return mFSMasterClient.free(fileId, recursive); } catch (FileDoesNotExistException e) { throw new IOException(e); } }
/** * Renames a file or folder to the indicated new path. * * @param fileId The id of the source file / folder. If it is not -1, path parameter is ignored. * Otherwise, the method uses the srcPath parameter. * @param srcPath The path of the source file / folder. It could be empty iff id is not -1. * @param dstPath The path of the destination file / folder. It could be empty iff id is not -1. * @return true if renames successfully, false otherwise. * @throws IOException if the underlying master RPC fails */ @Override public synchronized boolean rename(long fileId, TachyonURI srcPath, TachyonURI dstPath) throws IOException { validateUri(srcPath); validateUri(dstPath); if (fileId == -1) { try { fileId = mFSMasterClient.getFileId(srcPath.getPath()); } catch (InvalidPathException e) { throw new IOException(e); } } try { return mFSMasterClient.renameFile(fileId, dstPath.getPath()); } catch (FileDoesNotExistException e) { throw new IOException(e); } }
/** * Creates a folder. * * @param path the path of the folder to be created * @param recursive Creates necessary parent folders if true, not otherwise. * @return true if the folder is created successfully or already existing. false otherwise. * @throws IOException if the underlying master RPC fails */ @Override public synchronized boolean mkdirs(TachyonURI path, boolean recursive) throws IOException { validateUri(path); try { return mFSMasterClient.createDirectory(path.getPath(), recursive); } catch (TException e) { throw new IOException(e); } }
/** * If the <code>path</code> is a directory, returns all the direct entries in it. If the <code> * path</code> is a file, returns its ClientFileInfo. * * @param path the target directory/file path * @return A list of FileInfo, null if the file or folder does not exist. * @throws IOException when the underlying master RPC fails */ @Override public synchronized List<FileInfo> listStatus(TachyonURI path) throws IOException { validateUri(path); try { return mFSMasterClient.getFileInfoList(getFileStatus(-1, path).getFileId()); } catch (FileDoesNotExistException e) { throw new IOException(e); } }
/** * Gets all the blocks' info of the file. * * @param fid the file id * @return the list of the blocks' info * @throws IOException if the underlying master RPC fails */ public synchronized List<FileBlockInfo> getFileBlocks(long fid) throws IOException { // TODO(hy) Should read from mClientFileInfos if possible. // TODO(hy) Should add timeout to improve this. try { return mFSMasterClient.getFileBlockInfoList(fid); } catch (FileDoesNotExistException e) { throw new IOException(e); } }
@Override public void stop() { mSessionCleaner.stop(); if (mFilePersistenceService != null) { mFilePersistenceService.cancel(true); } mFileSystemMasterWorkerClient.close(); // This needs to be shutdownNow because heartbeat threads will only stop when interrupted. getExecutorService().shutdownNow(); }
/** * Advanced API. * * <p>Gets the FileInfo object that represents the fileId, or the path if fileId is -1. * * @param fileId the file id of the file or folder. * @param path the path of the file or folder. valid iff fileId is -1. * @param useCachedMetadata if true use the local cached meta data * @return the FileInfo of the file. null if the file does not exist. * @throws IOException if the underlying master RPC fails */ public synchronized FileInfo getFileStatus( long fileId, TachyonURI path, boolean useCachedMetadata) throws IOException { if (fileId == -1) { try { fileId = mFSMasterClient.getFileId(path.getPath()); } catch (InvalidPathException e) { throw new IOException(e); } } return getFileStatus( mIdToClientFileInfo, fileId, fileId, TachyonURI.EMPTY_URI.getPath(), useCachedMetadata); }
/** * Get the block id by the file id and block index. it will check whether the file and the block * exist. * * @param fileId the file id * @param blockIndex The index of the block in the file. * @return the block id if exists * @throws IOException if the file does not exist, or connection issue was encountered */ public synchronized long getBlockId(long fileId, int blockIndex) throws IOException { FileInfo info = getFileStatus(fileId, true); if (info == null) { throw new IOException("File " + fileId + " does not exist."); } if (info.blockIds.size() > blockIndex) { return info.blockIds.get(blockIndex); } try { return mFSMasterClient.getFileBlockInfo(fileId, blockIndex).blockInfo.getBlockId(); } catch (TException e) { throw new IOException(e); } }
/** * Creates a dependency. * * @param parents the dependency's input files * @param children the dependency's output files * @param commandPrefix identifies a command prefix * @param data stores dependency data * @param comment records a dependency comment * @param framework identifies the framework * @param frameworkVersion identifies the framework version * @param dependencyType the dependency's type, Wide or Narrow * @param childrenBlockSizeByte the block size of the dependency's output files * @return the dependency's id * @throws IOException if the underlying master RPC fails */ public synchronized int createDependency( List<String> parents, List<String> children, String commandPrefix, List<ByteBuffer> data, String comment, String framework, String frameworkVersion, int dependencyType, long childrenBlockSizeByte) throws IOException { return mFSMasterClient.user_createDependency( parents, children, commandPrefix, data, comment, framework, frameworkVersion, dependencyType, childrenBlockSizeByte); }
/** * Checks if this client is connected to master. * * @return true if this client is connected to master, false otherwise */ public synchronized boolean isConnected() { return mFSMasterClient.isConnected(); }
/** * @param depId the dependency id * @return the DependencyInfo of the specified dependency * @throws IOException if the underlying master RPC fails */ public synchronized DependencyInfo getClientDependencyInfo(int depId) throws IOException { return mFSMasterClient.getDependencyInfo(depId); }
@Override public void close() throws IOException { if (mClosed) { return; } if (mCurrentBlockOutStream != null) { mPreviousBlockOutStreams.add(mCurrentBlockOutStream); } CompleteFileOptions options = CompleteFileOptions.defaults(); if (mUnderStorageType.isSyncPersist()) { String tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath); UnderFileSystem ufs = UnderFileSystem.get(tmpPath, ClientContext.getConf()); if (mCanceled) { // TODO(yupeng): Handle this special case in under storage integrations. mUnderStorageOutputStream.close(); if (!ufs.exists(tmpPath)) { // Location of the temporary file has changed, recompute it. updateUfsPath(); tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath); } ufs.delete(tmpPath, false); } else { mUnderStorageOutputStream.flush(); mUnderStorageOutputStream.close(); if (!ufs.exists(tmpPath)) { // Location of the temporary file has changed, recompute it. updateUfsPath(); tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath); } if (!ufs.rename(tmpPath, mUfsPath)) { throw new IOException("Failed to rename " + tmpPath + " to " + mUfsPath); } options.setUfsLength(ufs.getFileSize(mUfsPath)); } } if (mAlluxioStorageType.isStore()) { try { if (mCanceled) { for (BufferedBlockOutStream bos : mPreviousBlockOutStreams) { bos.cancel(); } } else { for (BufferedBlockOutStream bos : mPreviousBlockOutStreams) { bos.close(); } } } catch (IOException e) { handleCacheWriteException(e); } } // Complete the file if it's ready to be completed. if (!mCanceled && (mUnderStorageType.isSyncPersist() || mAlluxioStorageType.isStore())) { FileSystemMasterClient masterClient = mContext.acquireMasterClient(); try { masterClient.completeFile(mUri, options); } catch (AlluxioException e) { throw new IOException(e); } finally { mContext.releaseMasterClient(masterClient); } } if (mUnderStorageType.isAsyncPersist()) { scheduleAsyncPersist(); } mClosed = true; }
/** * @return the address of the UnderFileSystem * @throws IOException if the underlying master RPC fails */ public synchronized String getUfsAddress() throws IOException { return mFSMasterClient.getUfsAddress(); }