/** * Notify the worker the block is cached. * * <p>This call is called remotely from {@link tachyon.client.TachyonFS#cacheBlock(long)} which is * only ever called from {@link tachyon.client.BlockOutStream#close()} (though its a public api so * anyone could call it). There are a few interesting preconditions for this to work. * * <p>1) Client process writes to files locally under a tachyon defined temp directory. 2) Worker * process is on the same node as the client 3) Client is talking to the local worker directly * * <p>If all conditions are true, then and only then can this method ever be called; all * operations work on local files. * * @param userId The user id of the client who send the notification * @param blockId The id of the block * @throws FileDoesNotExistException * @throws SuspectedFileSizeException * @throws BlockInfoException * @throws TException */ public void cacheBlock(long userId, long blockId) throws FileDoesNotExistException, SuspectedFileSizeException, BlockInfoException, TException { File srcFile = new File(CommonUtils.concat(getUserTempFolder(userId), blockId)); File dstFile = new File(CommonUtils.concat(mLocalDataFolder, blockId)); long fileSizeBytes = srcFile.length(); if (!srcFile.exists()) { throw new FileDoesNotExistException("File " + srcFile + " does not exist."); } if (!srcFile.renameTo(dstFile)) { throw new FileDoesNotExistException( "Failed to rename file from " + srcFile.getPath() + " to " + dstFile.getPath()); } addBlockId(blockId, fileSizeBytes); mUsers.addOwnBytes(userId, -fileSizeBytes); mMasterClient.worker_cacheBlock( mWorkerId, mWorkerSpaceCounter.getUsedBytes(), blockId, fileSizeBytes); LOG.info(userId + " " + dstFile); }
/** * Add the checkpoint information of a file. The information is from the user <code>userId</code>. * * <p>This method is normally triggered from {@link tachyon.client.FileOutStream#close()} if and * only if {@link tachyon.client.WriteType#isThrough()} is true. The current implementation of * checkpointing is that through {@link tachyon.client.WriteType} operations write to {@link * tachyon.UnderFileSystem} on the client's write path, but under a user temp directory (temp * directory is defined in the worker as {@link #getUserUnderfsTempFolder(long)}). * * @param userId The user id of the client who send the notification * @param fileId The id of the checkpointed file * @throws FileDoesNotExistException * @throws SuspectedFileSizeException * @throws FailedToCheckpointException * @throws BlockInfoException * @throws TException */ public void addCheckpoint(long userId, int fileId) throws FileDoesNotExistException, SuspectedFileSizeException, FailedToCheckpointException, BlockInfoException, TException { // TODO This part need to be changed. String srcPath = CommonUtils.concat(getUserUnderfsTempFolder(userId), fileId); String dstPath = CommonUtils.concat(COMMON_CONF.UNDERFS_DATA_FOLDER, fileId); try { if (!mUnderFs.rename(srcPath, dstPath)) { throw new FailedToCheckpointException("Failed to rename " + srcPath + " to " + dstPath); } } catch (IOException e) { throw new FailedToCheckpointException("Failed to rename " + srcPath + " to " + dstPath); } long fileSize; try { fileSize = mUnderFs.getFileSize(dstPath); } catch (IOException e) { throw new FailedToCheckpointException("Failed to getFileSize " + dstPath); } mMasterClient.addCheckpoint(mWorkerId, fileId, fileSize, dstPath); }
private ByteBuffer readRemoteByteBuffer(ClientBlockInfo blockInfo, long offset, long len) { ByteBuffer buf = null; try { List<NetAddress> blockLocations = blockInfo.getLocations(); LOG.info("Block locations:" + blockLocations); for (int k = 0; k < blockLocations.size(); k++) { String host = blockLocations.get(k).mHost; int port = blockLocations.get(k).mSecondaryPort; // The data is not in remote machine's memory if port == -1. if (port == -1) { continue; } if (host.equals(InetAddress.getLocalHost().getHostName()) || host.equals(InetAddress.getLocalHost().getHostAddress())) { String localFileName = CommonUtils.concat(TFS.getRootFolder(), blockInfo.blockId); LOG.warn("Master thinks the local machine has data " + localFileName + "! But not!"); } LOG.info( host + ":" + port + " current host is " + InetAddress.getLocalHost().getHostName() + " " + InetAddress.getLocalHost().getHostAddress()); try { buf = retrieveByteBufferFromRemoteMachine( new InetSocketAddress(host, port), blockInfo.blockId, offset, len); if (buf != null) { break; } } catch (IOException e) { LOG.error(e.getMessage()); buf = null; } } } catch (IOException e) { LOG.error("Failed to get read data from remote " + e.getMessage()); buf = null; } return buf; }
/** * Swap out those blocks missing INode information onto underFS which can be retrieved by user * later. Its cleanup only happens while formating the TFS. */ private void swapoutOrphanBlocks(long blockId, File file) throws IOException { RandomAccessFile localFile = new RandomAccessFile(file, "r"); ByteBuffer buf = localFile.getChannel().map(MapMode.READ_ONLY, 0, file.length()); String ufsOrphanBlock = CommonUtils.concat(mUnderfsOrphansFolder, blockId); OutputStream os = mUnderFs.create(ufsOrphanBlock); int BULKSIZE = Constants.KB * 64; byte[] bulk = new byte[BULKSIZE]; for (int k = 0; k < (buf.limit() + BULKSIZE - 1) / BULKSIZE; k++) { int len = BULKSIZE < buf.remaining() ? BULKSIZE : buf.remaining(); buf.get(bulk, 0, len); os.write(bulk, 0, len); } os.close(); localFile.close(); }
/** * This function sets the fileinfos for folders that are in the path to the current directory. * * @param path The path of the current directory. * @param request The HttpServletRequest object * @throws FileDoesNotExistException * @throws InvalidPathException */ private void setPathDirectories(String path, HttpServletRequest request) throws FileDoesNotExistException, InvalidPathException { if (path.equals(Constants.PATH_SEPARATOR)) { request.setAttribute("pathInfos", new UiFileInfo[0]); return; } String[] splitPath = path.split(Constants.PATH_SEPARATOR); UiFileInfo[] pathInfos = new UiFileInfo[splitPath.length - 1]; String currentPath = Constants.PATH_SEPARATOR; pathInfos[0] = new UiFileInfo(mMasterInfo.getClientFileInfo(currentPath)); for (int i = 1; i < splitPath.length - 1; i++) { currentPath = CommonUtils.concat(currentPath, splitPath[i]); pathInfos[i] = new UiFileInfo(mMasterInfo.getClientFileInfo(currentPath)); } request.setAttribute("pathInfos", pathInfos); }
/** * Remove a block from the memory. * * @param blockId The block to be removed. * @return Removed file size in bytes. */ private long freeBlock(long blockId) { long freedFileBytes = 0; synchronized (mLatestBlockAccessTimeMs) { if (mBlockSizes.containsKey(blockId)) { mWorkerSpaceCounter.returnUsedBytes(mBlockSizes.get(blockId)); File srcFile = new File(CommonUtils.concat(mLocalDataFolder, blockId)); srcFile.delete(); mLatestBlockAccessTimeMs.remove(blockId); freedFileBytes = mBlockSizes.remove(blockId); mRemovedBlockList.add(blockId); mMemoryData.remove(blockId); LOG.info("Removed Data " + blockId); } else { LOG.warn("File " + blockId + " does not exist in memory."); } } return freedFileBytes; }
BlockOutStream(TachyonFile file, WriteType opType, int blockIndex) throws IOException { super(file, opType); if (!opType.isCache()) { throw new IOException("BlockOutStream only support WriteType.CACHE"); } BLOCK_INDEX = blockIndex; BLOCK_CAPACITY_BYTE = FILE.getBlockSizeByte(); BLOCK_ID = FILE.getBlockId(BLOCK_INDEX); BLOCK_OFFSET = BLOCK_CAPACITY_BYTE * blockIndex; PIN = FILE.needPin(); mCanWrite = true; if (!TFS.hasLocalWorker()) { mCanWrite = false; String msg = "The machine does not have any local worker."; throw new IOException(msg); } File localFolder = TFS.createAndGetUserTempFolder(); if (localFolder == null) { mCanWrite = false; String msg = "Failed to create temp user folder for tachyon client."; throw new IOException(msg); } mLocalFilePath = CommonUtils.concat(localFolder.getPath(), BLOCK_ID); mLocalFile = new RandomAccessFile(mLocalFilePath, "rw"); mLocalFileChannel = mLocalFile.getChannel(); // change the permission of the temporary file in order that the worker can move it. CommonUtils.changeLocalFileToFullPermission(mLocalFilePath); // use the sticky bit, only the client and the worker can write to the block CommonUtils.setLocalFileStickyBit(mLocalFilePath); LOG.info(mLocalFilePath + " was created!"); mBuffer = ByteBuffer.allocate(USER_CONF.FILE_BUFFER_BYTES + 4); }
public void initialize(final NetAddress address) { mWorkerAddress = address; register(); mUnderfsWorkerFolder = CommonUtils.concat(COMMON_CONF.UNDERFS_WORKERS_FOLDER, mWorkerId); mUnderfsWorkerDataFolder = mUnderfsWorkerFolder + "/data"; mUnderFs = UnderFileSystem.get(COMMON_CONF.UNDERFS_ADDRESS); mUsers = new Users(mLocalUserFolder.toString(), mUnderfsWorkerFolder); for (int k = 0; k < WorkerConf.get().WORKER_CHECKPOINT_THREADS; k++) { Thread thread = new Thread(new CheckpointThread(k)); mCheckpointThreads.add(thread); thread.start(); } try { initializeWorkerStorage(); } catch (IOException e) { throw Throwables.propagate(e); } catch (FileDoesNotExistException e) { throw Throwables.propagate(e); } catch (SuspectedFileSizeException e) { throw Throwables.propagate(e); } catch (BlockInfoException e) { throw Throwables.propagate(e); } catch (TException e) { throw Throwables.propagate(e); } LOG.info( "Current Worker Info: ID " + mWorkerId + ", ADDRESS: " + mWorkerAddress + ", MemoryCapacityBytes: " + mWorkerSpaceCounter.getCapacityBytes()); }
@Override public void run() { while (true) { try { int fileId = -1; synchronized (mDependencyLock) { fileId = getFileIdBasedOnPriorityDependency(); if (fileId == -1) { if (mPriorityDependencies.size() == 0) { mPriorityDependencies = getSortedPriorityDependencyList(); if (!mPriorityDependencies.isEmpty()) { LOG.info( "Get new mPriorityDependencies " + CommonUtils.listToString(mPriorityDependencies)); } } else { List<Integer> tList = getSortedPriorityDependencyList(); boolean equal = true; if (mPriorityDependencies.size() != tList.size()) { equal = false; } if (equal) { for (int k = 0; k < tList.size(); k++) { if (tList.get(k) != mPriorityDependencies.get(k)) { equal = false; break; } } } if (!equal) { mPriorityDependencies = tList; } } fileId = getFileIdBasedOnPriorityDependency(); } if (fileId == -1) { fileId = getRandomUncheckpointedFile(); } } if (fileId == -1) { LOG.debug("Thread " + ID + " has nothing to checkpoint. Sleep for 1 sec."); CommonUtils.sleepMs(LOG, Constants.SECOND_MS); continue; } // TODO checkpoint process. In future, move from midPath to dstPath should be done by // master String midPath = CommonUtils.concat(mUnderfsWorkerDataFolder, fileId); String dstPath = CommonUtils.concat(CommonConf.get().UNDERFS_DATA_FOLDER, fileId); LOG.info( "Thread " + ID + " is checkpointing file " + fileId + " from " + mLocalDataFolder.toString() + " to " + midPath + " to " + dstPath); if (mCheckpointUnderFs == null) { mCheckpointUnderFs = UnderFileSystem.get(midPath); } long startCopyTimeMs = System.currentTimeMillis(); ClientFileInfo fileInfo = mMasterClient.getClientFileInfoById(fileId); if (!fileInfo.isComplete) { LOG.error("File " + fileInfo + " is not complete!"); continue; } for (int k = 0; k < fileInfo.blockIds.size(); k++) { lockBlock(fileInfo.blockIds.get(k), Users.sCHECKPOINT_USER_ID); } OutputStream os = mCheckpointUnderFs.create(midPath, (int) fileInfo.getBlockSizeByte()); long fileSizeByte = 0; for (int k = 0; k < fileInfo.blockIds.size(); k++) { File tempFile = new File(CommonUtils.concat(mLocalDataFolder.toString(), fileInfo.blockIds.get(k))); fileSizeByte += tempFile.length(); InputStream is = new FileInputStream(tempFile); byte[] buf = new byte[16 * Constants.KB]; int got = is.read(buf); while (got != -1) { os.write(buf, 0, got); got = is.read(buf); } is.close(); } os.close(); if (!mCheckpointUnderFs.rename(midPath, dstPath)) { LOG.error("Failed to rename from " + midPath + " to " + dstPath); } mMasterClient.addCheckpoint(mWorkerId, fileId, fileSizeByte, dstPath); for (int k = 0; k < fileInfo.blockIds.size(); k++) { unlockBlock(fileInfo.blockIds.get(k), Users.sCHECKPOINT_USER_ID); } long shouldTakeMs = (long) (1000.0 * fileSizeByte / Constants.MB / WorkerConf.get().WORKER_PER_THREAD_CHECKPOINT_CAP_MB_SEC); long currentTimeMs = System.currentTimeMillis(); if (startCopyTimeMs + shouldTakeMs > currentTimeMs) { long shouldSleepMs = startCopyTimeMs + shouldTakeMs - currentTimeMs; LOG.info( "Checkpointed last file " + fileId + " took " + (currentTimeMs - startCopyTimeMs) + " ms. Need to sleep " + shouldSleepMs + " ms."); CommonUtils.sleepMs(LOG, shouldSleepMs); } } catch (FileDoesNotExistException e) { LOG.warn(e); } catch (SuspectedFileSizeException e) { LOG.error(e); } catch (BlockInfoException e) { LOG.error(e); } catch (IOException e) { LOG.error(e); } catch (TException e) { LOG.warn(e); } } }