/**
* Gets the UiFileInfo object that represents the fileId, or the filePath if fileId is -1.
*
* @param tachyonClient the TachyonFS client.
* @param fileId the file id of the file.
* @param filePath the path of the file. valid iff fileId is -1.
* @return the UiFileInfo object of the file.
* @throws FileDoesNotExistException
* @throws IOException
*/
private UiFileInfo getUiFileInfo(TachyonFS tachyonClient, int fileId, TachyonURI filePath)
throws FileDoesNotExistException, NotFoundException, IOException {
ClientFileInfo fileInfo = tachyonClient.getFileStatus(fileId, filePath, true);
if (fileInfo == null) {
throw new FileDoesNotExistException(
fileId != -1 ? Integer.toString(fileId) : filePath.toString());
}
UiFileInfo uiFileInfo = new UiFileInfo(fileInfo);
boolean blockExistOnWorker = false;
for (long blockId : fileInfo.getBlockIds()) {
if (mBlockDataManager.hasBlockMeta(blockId)) {
blockExistOnWorker = true;
BlockMeta blockMeta = mBlockDataManager.getVolatileBlockMeta(blockId);
long blockSize = blockMeta.getBlockSize();
StorageLevelAlias storageLevelAlias =
StorageDirId.getStorageLevelAlias(blockMeta.getParentDir().getStorageDirId());
// The block last access time is not available. Use -1 for now.
uiFileInfo.addBlock(storageLevelAlias, blockId, blockSize, -1);
}
}
if (!blockExistOnWorker) {
throw new FileDoesNotExistException(
fileId != -1 ? Integer.toString(fileId) : filePath.toString());
}
return uiFileInfo;
}
/**
* Return the {@code InStream} of this file based on the specified read type. If it has no block,
* return an {@code EmptyBlockInStream}; if it has only one block, return a {@code BlockInStream}
* of the block; otherwise return a {@code FileInStream}.
*
* @param readType the InStream's read type
* @return the InStream
* @throws IOException
*/
public InStream getInStream(ReadType readType) throws IOException {
if (readType == null) {
throw new IOException("ReadType can not be null.");
}
if (!isComplete()) {
throw new IOException("The file " + this + " is not complete.");
}
if (isDirectory()) {
throw new IOException("Cannot open a directory for reading.");
}
ClientFileInfo fileStatus = getUnCachedFileStatus();
List<Long> blocks = fileStatus.getBlockIds();
if (blocks.size() == 0) {
return new EmptyBlockInStream(this, readType, mTachyonConf);
}
if (blocks.size() == 1) {
return BlockInStream.get(this, readType, 0, mUFSConf, mTachyonConf);
}
return new FileInStream(this, readType, mUFSConf, mTachyonConf);
}
/**
* Return the under filesystem path in the under file system of this file
*
* @return the under filesystem path
* @throws IOException
*/
String getUfsPath() throws IOException {
ClientFileInfo info = getCachedFileStatus();
if (!info.getUfsPath().isEmpty()) {
return info.getUfsPath();
}
return getUnCachedFileStatus().getUfsPath();
}
@Test
public void addCheckpointTest()
throws FileDoesNotExistException, SuspectedFileSizeException, FileAlreadyExistException,
InvalidPathException, BlockInfoException, FileNotFoundException, TachyonException {
int fileId = mMasterInfo.createFile("/testFile", Constants.DEFAULT_BLOCK_SIZE_BYTE);
ClientFileInfo fileInfo = mMasterInfo.getClientFileInfo("/testFile");
Assert.assertEquals("", fileInfo.getUfsPath());
mMasterInfo.addCheckpoint(-1, fileId, 1, "/testPath");
fileInfo = mMasterInfo.getClientFileInfo("/testFile");
Assert.assertEquals("/testPath", fileInfo.getUfsPath());
mMasterInfo.addCheckpoint(-1, fileId, 1, "/testPath");
fileInfo = mMasterInfo.getClientFileInfo("/testFile");
Assert.assertEquals("/testPath", fileInfo.getUfsPath());
}
@Test
public void clientFileInfoEmptyFileTest()
throws InvalidPathException, FileDoesNotExistException, FileAlreadyExistException,
BlockInfoException, TachyonException {
int fileId = mMasterInfo.createFile("/testFile", Constants.DEFAULT_BLOCK_SIZE_BYTE);
ClientFileInfo fileInfo = mMasterInfo.getClientFileInfo("/testFile");
Assert.assertEquals("testFile", fileInfo.getName());
Assert.assertEquals(fileId, fileInfo.getId());
Assert.assertEquals(0, fileInfo.getLength());
Assert.assertEquals("", fileInfo.getUfsPath());
Assert.assertFalse(fileInfo.isFolder);
Assert.assertFalse(fileInfo.isPinned);
Assert.assertTrue(fileInfo.isCache);
Assert.assertFalse(fileInfo.isComplete);
}
@Test
public void clientFileInfoDirectoryTest()
throws InvalidPathException, FileDoesNotExistException, FileAlreadyExistException,
TachyonException {
Assert.assertTrue(mMasterInfo.mkdirs("/testFolder", true));
ClientFileInfo fileInfo = mMasterInfo.getClientFileInfo("/testFolder");
Assert.assertEquals("testFolder", fileInfo.getName());
Assert.assertEquals(2, fileInfo.getId());
Assert.assertEquals(0, fileInfo.getLength());
Assert.assertEquals("", fileInfo.getUfsPath());
Assert.assertTrue(fileInfo.isFolder);
Assert.assertFalse(fileInfo.isPinned);
Assert.assertFalse(fileInfo.isCache);
Assert.assertTrue(fileInfo.isComplete);
}
/**
* Notify the worker to checkpoint the file asynchronously.
*
* @param fileId The id of the file
* @return true if succeed, false otherwise
* @throws IOException
* @throws TException
*/
public boolean asyncCheckpoint(int fileId) throws IOException, TException {
ClientFileInfo fileInfo = mMasterClient.getClientFileInfoById(fileId);
if (fileInfo.getDependencyId() != -1) {
synchronized (mDependencyLock) {
mUncheckpointFiles.add(fileId);
if (!mDepIdToFiles.containsKey(fileInfo.getDependencyId())) {
mDepIdToFiles.put(fileInfo.getDependencyId(), new HashSet<Integer>());
}
mDepIdToFiles.get(fileInfo.getDependencyId()).add(fileId);
}
return true;
}
return false;
}
public UiFileInfo(ClientFileInfo fileInfo) {
mId = fileInfo.getId();
mDependencyId = fileInfo.getDependencyId();
mName = fileInfo.getName();
mAbsolutePath = fileInfo.getPath();
mCheckpointPath = fileInfo.getUfsPath();
mBlockSizeBytes = fileInfo.getBlockSizeByte();
mSize = fileInfo.getLength();
mCreationTimeMs = fileInfo.getCreationTimeMs();
mLastModificationTimeMs = fileInfo.getLastModificationTimeMs();
mInMemory = (100 == fileInfo.inMemoryPercentage);
mInMemoryPercent = fileInfo.getInMemoryPercentage();
mIsDirectory = fileInfo.isFolder;
mIsPinned = fileInfo.isPinned;
mFileLocations = new ArrayList<String>();
}
private UiFileInfo(ClientFileInfo fileInfo) {
ID = fileInfo.getId();
NAME = fileInfo.getName();
ABSOLUATE_PATH = fileInfo.getPath();
CHECKPOINT_PATH = fileInfo.getCheckpointPath();
SIZE = fileInfo.getSizeBytes();
CREATION_TIME_MS = fileInfo.getCreationTimeMs();
IN_MEMORY = fileInfo.isInMemory();
IS_DIRECTORY = fileInfo.isFolder();
mFileLocations = new ArrayList<String>();
}
@Override
public void run() {
while (true) {
try {
int fileId = -1;
synchronized (mDependencyLock) {
fileId = getFileIdBasedOnPriorityDependency();
if (fileId == -1) {
if (mPriorityDependencies.size() == 0) {
mPriorityDependencies = getSortedPriorityDependencyList();
if (!mPriorityDependencies.isEmpty()) {
LOG.info(
"Get new mPriorityDependencies "
+ CommonUtils.listToString(mPriorityDependencies));
}
} else {
List<Integer> tList = getSortedPriorityDependencyList();
boolean equal = true;
if (mPriorityDependencies.size() != tList.size()) {
equal = false;
}
if (equal) {
for (int k = 0; k < tList.size(); k++) {
if (tList.get(k) != mPriorityDependencies.get(k)) {
equal = false;
break;
}
}
}
if (!equal) {
mPriorityDependencies = tList;
}
}
fileId = getFileIdBasedOnPriorityDependency();
}
if (fileId == -1) {
fileId = getRandomUncheckpointedFile();
}
}
if (fileId == -1) {
LOG.debug("Thread " + ID + " has nothing to checkpoint. Sleep for 1 sec.");
CommonUtils.sleepMs(LOG, Constants.SECOND_MS);
continue;
}
// TODO checkpoint process. In future, move from midPath to dstPath should be done by
// master
String midPath = CommonUtils.concat(mUnderfsWorkerDataFolder, fileId);
String dstPath = CommonUtils.concat(CommonConf.get().UNDERFS_DATA_FOLDER, fileId);
LOG.info(
"Thread "
+ ID
+ " is checkpointing file "
+ fileId
+ " from "
+ mLocalDataFolder.toString()
+ " to "
+ midPath
+ " to "
+ dstPath);
if (mCheckpointUnderFs == null) {
mCheckpointUnderFs = UnderFileSystem.get(midPath);
}
long startCopyTimeMs = System.currentTimeMillis();
ClientFileInfo fileInfo = mMasterClient.getClientFileInfoById(fileId);
if (!fileInfo.isComplete) {
LOG.error("File " + fileInfo + " is not complete!");
continue;
}
for (int k = 0; k < fileInfo.blockIds.size(); k++) {
lockBlock(fileInfo.blockIds.get(k), Users.sCHECKPOINT_USER_ID);
}
OutputStream os = mCheckpointUnderFs.create(midPath, (int) fileInfo.getBlockSizeByte());
long fileSizeByte = 0;
for (int k = 0; k < fileInfo.blockIds.size(); k++) {
File tempFile =
new File(CommonUtils.concat(mLocalDataFolder.toString(), fileInfo.blockIds.get(k)));
fileSizeByte += tempFile.length();
InputStream is = new FileInputStream(tempFile);
byte[] buf = new byte[16 * Constants.KB];
int got = is.read(buf);
while (got != -1) {
os.write(buf, 0, got);
got = is.read(buf);
}
is.close();
}
os.close();
if (!mCheckpointUnderFs.rename(midPath, dstPath)) {
LOG.error("Failed to rename from " + midPath + " to " + dstPath);
}
mMasterClient.addCheckpoint(mWorkerId, fileId, fileSizeByte, dstPath);
for (int k = 0; k < fileInfo.blockIds.size(); k++) {
unlockBlock(fileInfo.blockIds.get(k), Users.sCHECKPOINT_USER_ID);
}
long shouldTakeMs =
(long)
(1000.0
* fileSizeByte
/ Constants.MB
/ WorkerConf.get().WORKER_PER_THREAD_CHECKPOINT_CAP_MB_SEC);
long currentTimeMs = System.currentTimeMillis();
if (startCopyTimeMs + shouldTakeMs > currentTimeMs) {
long shouldSleepMs = startCopyTimeMs + shouldTakeMs - currentTimeMs;
LOG.info(
"Checkpointed last file "
+ fileId
+ " took "
+ (currentTimeMs - startCopyTimeMs)
+ " ms. Need to sleep "
+ shouldSleepMs
+ " ms.");
CommonUtils.sleepMs(LOG, shouldSleepMs);
}
} catch (FileDoesNotExistException e) {
LOG.warn(e);
} catch (SuspectedFileSizeException e) {
LOG.error(e);
} catch (BlockInfoException e) {
LOG.error(e);
} catch (IOException e) {
LOG.error(e);
} catch (TException e) {
LOG.warn(e);
}
}
}
/**
* Populates attribute fields with data from the MasterInfo associated with this servlet. Errors
* will be displayed in an error field. Debugging can be enabled to display additional data. Will
* eventually redirect the request to a jsp.
*
* @param request The HttpServletRequest object
* @param response The HttpServletResponse object
* @throws ServletException
* @throws IOException
* @throws UnknownHostException
*/
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException, UnknownHostException {
request.setAttribute("debug", Constants.DEBUG);
request.setAttribute("masterNodeAddress", mMasterInfo.getMasterAddress().toString());
request.setAttribute("invalidPathError", "");
List<ClientFileInfo> filesInfo = null;
String currentPath = request.getParameter("path");
if (currentPath == null || currentPath.isEmpty()) {
currentPath = Constants.PATH_SEPARATOR;
}
request.setAttribute("currentPath", currentPath);
request.setAttribute("viewingOffset", 0);
try {
ClientFileInfo clientFileInfo = mMasterInfo.getClientFileInfo(currentPath);
UiFileInfo currentFileInfo = new UiFileInfo(clientFileInfo);
request.setAttribute("currentDirectory", currentFileInfo);
request.setAttribute("blockSizeByte", currentFileInfo.getBlockSizeBytes());
if (!currentFileInfo.getIsDirectory()) {
String tmpParam = request.getParameter("offset");
long offset = 0;
try {
if (tmpParam != null) {
offset = Long.valueOf(tmpParam);
}
} catch (NumberFormatException nfe) {
offset = 0;
}
if (offset < 0) {
offset = 0;
} else if (offset > clientFileInfo.getLength()) {
offset = clientFileInfo.getLength();
}
displayFile(currentFileInfo.getAbsolutePath(), request, offset);
request.setAttribute("viewingOffset", offset);
getServletContext().getRequestDispatcher("/viewFile.jsp").forward(request, response);
return;
}
CommonUtils.validatePath(currentPath);
setPathDirectories(currentPath, request);
filesInfo = mMasterInfo.getFilesInfo(currentPath);
} catch (FileDoesNotExistException fdne) {
request.setAttribute("invalidPathError", "Error: Invalid Path " + fdne.getMessage());
getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
return;
} catch (InvalidPathException ipe) {
request.setAttribute("invalidPathError", "Error: Invalid Path " + ipe.getLocalizedMessage());
getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
return;
} catch (IOException ie) {
request.setAttribute(
"invalidPathError",
"Error: File " + currentPath + " is not available " + ie.getMessage());
getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
return;
}
List<UiFileInfo> fileInfos = new ArrayList<UiFileInfo>(filesInfo.size());
for (ClientFileInfo fileInfo : filesInfo) {
UiFileInfo toAdd = new UiFileInfo(fileInfo);
try {
if (!toAdd.getIsDirectory() && fileInfo.getLength() > 0) {
toAdd.setFileLocations(mMasterInfo.getFileBlocks(toAdd.getId()).get(0).getLocations());
}
} catch (FileDoesNotExistException fdne) {
request.setAttribute("invalidPathError", "Error: Invalid Path " + fdne.getMessage());
getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
return;
}
fileInfos.add(toAdd);
}
Collections.sort(fileInfos);
request.setAttribute("fileInfos", fileInfos);
getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
}
@Override
public ClientFileInfo generateClientFileInfo(String path) {
ClientFileInfo ret = new ClientFileInfo();
ret.id = getId();
ret.name = getName();
ret.path = path;
ret.ufsPath = mUfsPath;
ret.length = mLength;
ret.blockSizeByte = mBlockSizeByte;
ret.creationTimeMs = getCreationTimeMs();
ret.isComplete = isComplete();
ret.isFolder = false;
ret.isPinned = isPinned();
ret.isCache = mCache;
ret.blockIds = getBlockIds();
ret.dependencyId = mDependencyId;
ret.inMemoryPercentage = getInMemoryPercentage();
ret.lastModificationTimeMs = getLastModificationTimeMs();
return ret;
}