/**
* Notify the worker the block is cached.
*
* <p>This call is called remotely from {@link tachyon.client.TachyonFS#cacheBlock(long)} which is
* only ever called from {@link tachyon.client.BlockOutStream#close()} (though its a public api so
* anyone could call it). There are a few interesting preconditions for this to work.
*
* <p>1) Client process writes to files locally under a tachyon defined temp directory. 2) Worker
* process is on the same node as the client 3) Client is talking to the local worker directly
*
* <p>If all conditions are true, then and only then can this method ever be called; all
* operations work on local files.
*
* @param userId The user id of the client who send the notification
* @param blockId The id of the block
* @throws FileDoesNotExistException
* @throws SuspectedFileSizeException
* @throws BlockInfoException
* @throws TException
*/
public void cacheBlock(long userId, long blockId)
throws FileDoesNotExistException, SuspectedFileSizeException, BlockInfoException, TException {
File srcFile = new File(CommonUtils.concat(getUserTempFolder(userId), blockId));
File dstFile = new File(CommonUtils.concat(mLocalDataFolder, blockId));
long fileSizeBytes = srcFile.length();
if (!srcFile.exists()) {
throw new FileDoesNotExistException("File " + srcFile + " does not exist.");
}
if (!srcFile.renameTo(dstFile)) {
throw new FileDoesNotExistException(
"Failed to rename file from " + srcFile.getPath() + " to " + dstFile.getPath());
}
addBlockId(blockId, fileSizeBytes);
mUsers.addOwnBytes(userId, -fileSizeBytes);
mMasterClient.worker_cacheBlock(
mWorkerId, mWorkerSpaceCounter.getUsedBytes(), blockId, fileSizeBytes);
LOG.info(userId + " " + dstFile);
}
/**
* Add the checkpoint information of a file. The information is from the user <code>userId</code>.
*
* <p>This method is normally triggered from {@link tachyon.client.FileOutStream#close()} if and
* only if {@link tachyon.client.WriteType#isThrough()} is true. The current implementation of
* checkpointing is that through {@link tachyon.client.WriteType} operations write to {@link
* tachyon.UnderFileSystem} on the client's write path, but under a user temp directory (temp
* directory is defined in the worker as {@link #getUserUnderfsTempFolder(long)}).
*
* @param userId The user id of the client who send the notification
* @param fileId The id of the checkpointed file
* @throws FileDoesNotExistException
* @throws SuspectedFileSizeException
* @throws FailedToCheckpointException
* @throws BlockInfoException
* @throws TException
*/
public void addCheckpoint(long userId, int fileId)
throws FileDoesNotExistException, SuspectedFileSizeException, FailedToCheckpointException,
BlockInfoException, TException {
// TODO This part need to be changed.
String srcPath = CommonUtils.concat(getUserUnderfsTempFolder(userId), fileId);
String dstPath = CommonUtils.concat(COMMON_CONF.UNDERFS_DATA_FOLDER, fileId);
try {
if (!mUnderFs.rename(srcPath, dstPath)) {
throw new FailedToCheckpointException("Failed to rename " + srcPath + " to " + dstPath);
}
} catch (IOException e) {
throw new FailedToCheckpointException("Failed to rename " + srcPath + " to " + dstPath);
}
long fileSize;
try {
fileSize = mUnderFs.getFileSize(dstPath);
} catch (IOException e) {
throw new FailedToCheckpointException("Failed to getFileSize " + dstPath);
}
mMasterClient.addCheckpoint(mWorkerId, fileId, fileSize, dstPath);
}
private ByteBuffer readRemoteByteBuffer(ClientBlockInfo blockInfo, long offset, long len) {
ByteBuffer buf = null;
try {
List<NetAddress> blockLocations = blockInfo.getLocations();
LOG.info("Block locations:" + blockLocations);
for (int k = 0; k < blockLocations.size(); k++) {
String host = blockLocations.get(k).mHost;
int port = blockLocations.get(k).mSecondaryPort;
// The data is not in remote machine's memory if port == -1.
if (port == -1) {
continue;
}
if (host.equals(InetAddress.getLocalHost().getHostName())
|| host.equals(InetAddress.getLocalHost().getHostAddress())) {
String localFileName = CommonUtils.concat(TFS.getRootFolder(), blockInfo.blockId);
LOG.warn("Master thinks the local machine has data " + localFileName + "! But not!");
}
LOG.info(
host
+ ":"
+ port
+ " current host is "
+ InetAddress.getLocalHost().getHostName()
+ " "
+ InetAddress.getLocalHost().getHostAddress());
try {
buf =
retrieveByteBufferFromRemoteMachine(
new InetSocketAddress(host, port), blockInfo.blockId, offset, len);
if (buf != null) {
break;
}
} catch (IOException e) {
LOG.error(e.getMessage());
buf = null;
}
}
} catch (IOException e) {
LOG.error("Failed to get read data from remote " + e.getMessage());
buf = null;
}
return buf;
}
/**
* Swap out those blocks missing INode information onto underFS which can be retrieved by user
* later. Its cleanup only happens while formating the TFS.
*/
private void swapoutOrphanBlocks(long blockId, File file) throws IOException {
RandomAccessFile localFile = new RandomAccessFile(file, "r");
ByteBuffer buf = localFile.getChannel().map(MapMode.READ_ONLY, 0, file.length());
String ufsOrphanBlock = CommonUtils.concat(mUnderfsOrphansFolder, blockId);
OutputStream os = mUnderFs.create(ufsOrphanBlock);
int BULKSIZE = Constants.KB * 64;
byte[] bulk = new byte[BULKSIZE];
for (int k = 0; k < (buf.limit() + BULKSIZE - 1) / BULKSIZE; k++) {
int len = BULKSIZE < buf.remaining() ? BULKSIZE : buf.remaining();
buf.get(bulk, 0, len);
os.write(bulk, 0, len);
}
os.close();
localFile.close();
}
/**
* This function sets the fileinfos for folders that are in the path to the current directory.
*
* @param path The path of the current directory.
* @param request The HttpServletRequest object
* @throws FileDoesNotExistException
* @throws InvalidPathException
*/
private void setPathDirectories(String path, HttpServletRequest request)
throws FileDoesNotExistException, InvalidPathException {
if (path.equals(Constants.PATH_SEPARATOR)) {
request.setAttribute("pathInfos", new UiFileInfo[0]);
return;
}
String[] splitPath = path.split(Constants.PATH_SEPARATOR);
UiFileInfo[] pathInfos = new UiFileInfo[splitPath.length - 1];
String currentPath = Constants.PATH_SEPARATOR;
pathInfos[0] = new UiFileInfo(mMasterInfo.getClientFileInfo(currentPath));
for (int i = 1; i < splitPath.length - 1; i++) {
currentPath = CommonUtils.concat(currentPath, splitPath[i]);
pathInfos[i] = new UiFileInfo(mMasterInfo.getClientFileInfo(currentPath));
}
request.setAttribute("pathInfos", pathInfos);
}
/**
* Remove a block from the memory.
*
* @param blockId The block to be removed.
* @return Removed file size in bytes.
*/
private long freeBlock(long blockId) {
long freedFileBytes = 0;
synchronized (mLatestBlockAccessTimeMs) {
if (mBlockSizes.containsKey(blockId)) {
mWorkerSpaceCounter.returnUsedBytes(mBlockSizes.get(blockId));
File srcFile = new File(CommonUtils.concat(mLocalDataFolder, blockId));
srcFile.delete();
mLatestBlockAccessTimeMs.remove(blockId);
freedFileBytes = mBlockSizes.remove(blockId);
mRemovedBlockList.add(blockId);
mMemoryData.remove(blockId);
LOG.info("Removed Data " + blockId);
} else {
LOG.warn("File " + blockId + " does not exist in memory.");
}
}
return freedFileBytes;
}
BlockOutStream(TachyonFile file, WriteType opType, int blockIndex) throws IOException {
super(file, opType);
if (!opType.isCache()) {
throw new IOException("BlockOutStream only support WriteType.CACHE");
}
BLOCK_INDEX = blockIndex;
BLOCK_CAPACITY_BYTE = FILE.getBlockSizeByte();
BLOCK_ID = FILE.getBlockId(BLOCK_INDEX);
BLOCK_OFFSET = BLOCK_CAPACITY_BYTE * blockIndex;
PIN = FILE.needPin();
mCanWrite = true;
if (!TFS.hasLocalWorker()) {
mCanWrite = false;
String msg = "The machine does not have any local worker.";
throw new IOException(msg);
}
File localFolder = TFS.createAndGetUserTempFolder();
if (localFolder == null) {
mCanWrite = false;
String msg = "Failed to create temp user folder for tachyon client.";
throw new IOException(msg);
}
mLocalFilePath = CommonUtils.concat(localFolder.getPath(), BLOCK_ID);
mLocalFile = new RandomAccessFile(mLocalFilePath, "rw");
mLocalFileChannel = mLocalFile.getChannel();
// change the permission of the temporary file in order that the worker can move it.
CommonUtils.changeLocalFileToFullPermission(mLocalFilePath);
// use the sticky bit, only the client and the worker can write to the block
CommonUtils.setLocalFileStickyBit(mLocalFilePath);
LOG.info(mLocalFilePath + " was created!");
mBuffer = ByteBuffer.allocate(USER_CONF.FILE_BUFFER_BYTES + 4);
}
public void initialize(final NetAddress address) {
mWorkerAddress = address;
register();
mUnderfsWorkerFolder = CommonUtils.concat(COMMON_CONF.UNDERFS_WORKERS_FOLDER, mWorkerId);
mUnderfsWorkerDataFolder = mUnderfsWorkerFolder + "/data";
mUnderFs = UnderFileSystem.get(COMMON_CONF.UNDERFS_ADDRESS);
mUsers = new Users(mLocalUserFolder.toString(), mUnderfsWorkerFolder);
for (int k = 0; k < WorkerConf.get().WORKER_CHECKPOINT_THREADS; k++) {
Thread thread = new Thread(new CheckpointThread(k));
mCheckpointThreads.add(thread);
thread.start();
}
try {
initializeWorkerStorage();
} catch (IOException e) {
throw Throwables.propagate(e);
} catch (FileDoesNotExistException e) {
throw Throwables.propagate(e);
} catch (SuspectedFileSizeException e) {
throw Throwables.propagate(e);
} catch (BlockInfoException e) {
throw Throwables.propagate(e);
} catch (TException e) {
throw Throwables.propagate(e);
}
LOG.info(
"Current Worker Info: ID "
+ mWorkerId
+ ", ADDRESS: "
+ mWorkerAddress
+ ", MemoryCapacityBytes: "
+ mWorkerSpaceCounter.getCapacityBytes());
}
@Override
public void run() {
while (true) {
try {
int fileId = -1;
synchronized (mDependencyLock) {
fileId = getFileIdBasedOnPriorityDependency();
if (fileId == -1) {
if (mPriorityDependencies.size() == 0) {
mPriorityDependencies = getSortedPriorityDependencyList();
if (!mPriorityDependencies.isEmpty()) {
LOG.info(
"Get new mPriorityDependencies "
+ CommonUtils.listToString(mPriorityDependencies));
}
} else {
List<Integer> tList = getSortedPriorityDependencyList();
boolean equal = true;
if (mPriorityDependencies.size() != tList.size()) {
equal = false;
}
if (equal) {
for (int k = 0; k < tList.size(); k++) {
if (tList.get(k) != mPriorityDependencies.get(k)) {
equal = false;
break;
}
}
}
if (!equal) {
mPriorityDependencies = tList;
}
}
fileId = getFileIdBasedOnPriorityDependency();
}
if (fileId == -1) {
fileId = getRandomUncheckpointedFile();
}
}
if (fileId == -1) {
LOG.debug("Thread " + ID + " has nothing to checkpoint. Sleep for 1 sec.");
CommonUtils.sleepMs(LOG, Constants.SECOND_MS);
continue;
}
// TODO checkpoint process. In future, move from midPath to dstPath should be done by
// master
String midPath = CommonUtils.concat(mUnderfsWorkerDataFolder, fileId);
String dstPath = CommonUtils.concat(CommonConf.get().UNDERFS_DATA_FOLDER, fileId);
LOG.info(
"Thread "
+ ID
+ " is checkpointing file "
+ fileId
+ " from "
+ mLocalDataFolder.toString()
+ " to "
+ midPath
+ " to "
+ dstPath);
if (mCheckpointUnderFs == null) {
mCheckpointUnderFs = UnderFileSystem.get(midPath);
}
long startCopyTimeMs = System.currentTimeMillis();
ClientFileInfo fileInfo = mMasterClient.getClientFileInfoById(fileId);
if (!fileInfo.isComplete) {
LOG.error("File " + fileInfo + " is not complete!");
continue;
}
for (int k = 0; k < fileInfo.blockIds.size(); k++) {
lockBlock(fileInfo.blockIds.get(k), Users.sCHECKPOINT_USER_ID);
}
OutputStream os = mCheckpointUnderFs.create(midPath, (int) fileInfo.getBlockSizeByte());
long fileSizeByte = 0;
for (int k = 0; k < fileInfo.blockIds.size(); k++) {
File tempFile =
new File(CommonUtils.concat(mLocalDataFolder.toString(), fileInfo.blockIds.get(k)));
fileSizeByte += tempFile.length();
InputStream is = new FileInputStream(tempFile);
byte[] buf = new byte[16 * Constants.KB];
int got = is.read(buf);
while (got != -1) {
os.write(buf, 0, got);
got = is.read(buf);
}
is.close();
}
os.close();
if (!mCheckpointUnderFs.rename(midPath, dstPath)) {
LOG.error("Failed to rename from " + midPath + " to " + dstPath);
}
mMasterClient.addCheckpoint(mWorkerId, fileId, fileSizeByte, dstPath);
for (int k = 0; k < fileInfo.blockIds.size(); k++) {
unlockBlock(fileInfo.blockIds.get(k), Users.sCHECKPOINT_USER_ID);
}
long shouldTakeMs =
(long)
(1000.0
* fileSizeByte
/ Constants.MB
/ WorkerConf.get().WORKER_PER_THREAD_CHECKPOINT_CAP_MB_SEC);
long currentTimeMs = System.currentTimeMillis();
if (startCopyTimeMs + shouldTakeMs > currentTimeMs) {
long shouldSleepMs = startCopyTimeMs + shouldTakeMs - currentTimeMs;
LOG.info(
"Checkpointed last file "
+ fileId
+ " took "
+ (currentTimeMs - startCopyTimeMs)
+ " ms. Need to sleep "
+ shouldSleepMs
+ " ms.");
CommonUtils.sleepMs(LOG, shouldSleepMs);
}
} catch (FileDoesNotExistException e) {
LOG.warn(e);
} catch (SuspectedFileSizeException e) {
LOG.error(e);
} catch (BlockInfoException e) {
LOG.error(e);
} catch (IOException e) {
LOG.error(e);
} catch (TException e) {
LOG.warn(e);
}
}
}
