/**
* Populates attribute fields with data from the MasterInfo associated with this servlet. Errors
* will be displayed in an error field. Debugging can be enabled to display additional data. Will
* eventually redirect the request to a jsp.
*
* @param request The HttpServletRequest object
* @param response The HttpServletResponse object
* @throws ServletException
* @throws IOException
* @throws UnknownHostException
*/
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException, UnknownHostException {
request.setAttribute("debug", Constants.DEBUG);
request.setAttribute("masterNodeAddress", mMasterInfo.getMasterAddress().toString());
request.setAttribute("invalidPathError", "");
List<ClientFileInfo> filesInfo = null;
String currentPath = request.getParameter("path");
if (currentPath.isEmpty()) {
currentPath = "/";
}
request.setAttribute("currentPath", currentPath);
try {
UiFileInfo currentFileInfo = new UiFileInfo(mMasterInfo.getFileInfo(currentPath));
request.setAttribute("currentDirectory", currentFileInfo);
if (!currentFileInfo.getIsDirectory()) {
displayFile(currentFileInfo.getAbsolutePath(), request);
getServletContext().getRequestDispatcher("/viewFile.jsp").forward(request, response);
return;
}
CommonUtils.validatePath(currentPath);
setPathDirectories(currentPath, request);
filesInfo = mMasterInfo.getFilesInfo(currentPath);
} catch (FileDoesNotExistException fdne) {
request.setAttribute("invalidPathError", "Error: Invalid Path " + fdne.getMessage());
getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
return;
} catch (InvalidPathException ipe) {
request.setAttribute("invalidPathError", "Error: Invalid Path " + ipe.getLocalizedMessage());
getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
return;
}
List<UiFileInfo> fileInfos = new ArrayList<UiFileInfo>(filesInfo.size());
for (ClientFileInfo fileInfo : filesInfo) {
UiFileInfo toAdd = new UiFileInfo(fileInfo);
try {
if (!toAdd.getIsDirectory()) {
toAdd.setFileLocations(mMasterInfo.getFileLocations(toAdd.getId()));
}
} catch (FileDoesNotExistException fdne) {
request.setAttribute("invalidPathError", "Error: Invalid Path " + fdne.getMessage());
getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
return;
}
fileInfos.add(toAdd);
}
Collections.sort(fileInfos);
request.setAttribute("fileInfos", fileInfos);
getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
}
/**
* Populates attributes before redirecting to a jsp.
*
* @param request The HttpServletRequest object
* @param response The HttpServletReponse object
* @throws ServletException
* @throws IOException
*/
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
request.setAttribute("fatalError", "");
TachyonFS tachyonClient = TachyonFS.get(mTachyonConf);
String filePath = request.getParameter("path");
if (!(filePath == null || filePath.isEmpty())) {
// Display file block info
try {
UiFileInfo uiFileInfo = getUiFileInfo(tachyonClient, new TachyonURI(filePath));
request.setAttribute("fileBlocksOnTier", uiFileInfo.getBlocksOnTier());
request.setAttribute("blockSizeByte", uiFileInfo.getBlockSizeBytes());
request.setAttribute("path", filePath);
getServletContext()
.getRequestDispatcher("/worker/viewFileBlocks.jsp")
.forward(request, response);
return;
} catch (FileDoesNotExistException fdne) {
request.setAttribute("fatalError", "Error: Invalid Path " + fdne.getMessage());
getServletContext()
.getRequestDispatcher("/worker/blockInfo.jsp")
.forward(request, response);
return;
} catch (IOException ie) {
request.setAttribute(
"invalidPathError", "Error: File " + filePath + " is not available " + ie.getMessage());
getServletContext()
.getRequestDispatcher("/worker/blockInfo.jsp")
.forward(request, response);
return;
} catch (NotFoundException nfe) {
request.setAttribute("fatalError", "Error: block not found. " + nfe.getMessage());
getServletContext()
.getRequestDispatcher("/worker/blockInfo.jsp")
.forward(request, response);
return;
}
}
List<Integer> fileIds = getSortedFileIds();
request.setAttribute("nTotalFile", fileIds.size());
// URL can not determine offset and limit, let javascript in jsp determine and redirect
if (request.getParameter("offset") == null && request.getParameter("limit") == null) {
getServletContext().getRequestDispatcher("/worker/blockInfo.jsp").forward(request, response);
return;
}
try {
int offset = Integer.parseInt(request.getParameter("offset"));
int limit = Integer.parseInt(request.getParameter("limit"));
List<Integer> subFileIds = fileIds.subList(offset, offset + limit);
List<UiFileInfo> uiFileInfos = new ArrayList<UiFileInfo>(subFileIds.size());
for (int fileId : subFileIds) {
uiFileInfos.add(getUiFileInfo(tachyonClient, fileId));
}
request.setAttribute("fileInfos", uiFileInfos);
} catch (FileDoesNotExistException fdne) {
request.setAttribute("fatalError", "Error: Invalid FileId " + fdne.getMessage());
getServletContext().getRequestDispatcher("/worker/blockInfo.jsp").forward(request, response);
return;
} catch (NumberFormatException nfe) {
request.setAttribute(
"fatalError", "Error: offset or limit parse error, " + nfe.getLocalizedMessage());
getServletContext().getRequestDispatcher("/worker/blockInfo.jsp").forward(request, response);
return;
} catch (IndexOutOfBoundsException iobe) {
request.setAttribute(
"fatalError",
"Error: offset or offset + limit is out of bound, " + iobe.getLocalizedMessage());
getServletContext().getRequestDispatcher("/worker/blockInfo.jsp").forward(request, response);
return;
} catch (IllegalArgumentException iae) {
request.setAttribute("fatalError", iae.getLocalizedMessage());
getServletContext().getRequestDispatcher("/worker/blockInfo.jsp").forward(request, response);
return;
} catch (NotFoundException nfe) {
request.setAttribute("fatalError", nfe.getLocalizedMessage());
getServletContext().getRequestDispatcher("/worker/blockInfo.jsp").forward(request, response);
return;
}
getServletContext().getRequestDispatcher("/worker/blockInfo.jsp").forward(request, response);
}
/**
* Populates attribute fields with data from the MasterInfo associated with this servlet. Errors
* will be displayed in an error field. Debugging can be enabled to display additional data. Will
* eventually redirect the request to a jsp.
*
* @param request The HttpServletRequest object
* @param response The HttpServletResponse object
* @throws ServletException
* @throws IOException
* @throws UnknownHostException
*/
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException, UnknownHostException {
request.setAttribute("debug", Constants.DEBUG);
request.setAttribute("masterNodeAddress", mMasterInfo.getMasterAddress().toString());
request.setAttribute("invalidPathError", "");
List<ClientFileInfo> filesInfo = null;
String currentPath = request.getParameter("path");
if (currentPath == null || currentPath.isEmpty()) {
currentPath = Constants.PATH_SEPARATOR;
}
request.setAttribute("currentPath", currentPath);
request.setAttribute("viewingOffset", 0);
try {
ClientFileInfo clientFileInfo = mMasterInfo.getClientFileInfo(currentPath);
UiFileInfo currentFileInfo = new UiFileInfo(clientFileInfo);
request.setAttribute("currentDirectory", currentFileInfo);
request.setAttribute("blockSizeByte", currentFileInfo.getBlockSizeBytes());
if (!currentFileInfo.getIsDirectory()) {
String tmpParam = request.getParameter("offset");
long offset = 0;
try {
if (tmpParam != null) {
offset = Long.valueOf(tmpParam);
}
} catch (NumberFormatException nfe) {
offset = 0;
}
if (offset < 0) {
offset = 0;
} else if (offset > clientFileInfo.getLength()) {
offset = clientFileInfo.getLength();
}
displayFile(currentFileInfo.getAbsolutePath(), request, offset);
request.setAttribute("viewingOffset", offset);
getServletContext().getRequestDispatcher("/viewFile.jsp").forward(request, response);
return;
}
CommonUtils.validatePath(currentPath);
setPathDirectories(currentPath, request);
filesInfo = mMasterInfo.getFilesInfo(currentPath);
} catch (FileDoesNotExistException fdne) {
request.setAttribute("invalidPathError", "Error: Invalid Path " + fdne.getMessage());
getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
return;
} catch (InvalidPathException ipe) {
request.setAttribute("invalidPathError", "Error: Invalid Path " + ipe.getLocalizedMessage());
getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
return;
} catch (IOException ie) {
request.setAttribute(
"invalidPathError",
"Error: File " + currentPath + " is not available " + ie.getMessage());
getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
return;
}
List<UiFileInfo> fileInfos = new ArrayList<UiFileInfo>(filesInfo.size());
for (ClientFileInfo fileInfo : filesInfo) {
UiFileInfo toAdd = new UiFileInfo(fileInfo);
try {
if (!toAdd.getIsDirectory() && fileInfo.getLength() > 0) {
toAdd.setFileLocations(mMasterInfo.getFileBlocks(toAdd.getId()).get(0).getLocations());
}
} catch (FileDoesNotExistException fdne) {
request.setAttribute("invalidPathError", "Error: Invalid Path " + fdne.getMessage());
getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
return;
}
fileInfos.add(toAdd);
}
Collections.sort(fileInfos);
request.setAttribute("fileInfos", fileInfos);
getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
}
@Override
public void run() {
while (true) {
try {
int fileId = -1;
synchronized (mDependencyLock) {
fileId = getFileIdBasedOnPriorityDependency();
if (fileId == -1) {
if (mPriorityDependencies.size() == 0) {
mPriorityDependencies = getSortedPriorityDependencyList();
if (!mPriorityDependencies.isEmpty()) {
LOG.info(
"Get new mPriorityDependencies "
+ CommonUtils.listToString(mPriorityDependencies));
}
} else {
List<Integer> tList = getSortedPriorityDependencyList();
boolean equal = true;
if (mPriorityDependencies.size() != tList.size()) {
equal = false;
}
if (equal) {
for (int k = 0; k < tList.size(); k++) {
if (tList.get(k) != mPriorityDependencies.get(k)) {
equal = false;
break;
}
}
}
if (!equal) {
mPriorityDependencies = tList;
}
}
fileId = getFileIdBasedOnPriorityDependency();
}
if (fileId == -1) {
fileId = getRandomUncheckpointedFile();
}
}
if (fileId == -1) {
LOG.debug("Thread {} has nothing to checkpoint. Sleep for 1 sec.", mId);
CommonUtils.sleepMs(LOG, Constants.SECOND_MS);
continue;
}
// TODO checkpoint process. In future, move from midPath to dstPath should be done by
// master
String midPath = CommonUtils.concat(mUfsWorkerDataFolder, fileId);
String dstPath = CommonUtils.concat(CommonConf.get().UNDERFS_DATA_FOLDER, fileId);
LOG.info(
"Thread "
+ mId
+ " is checkpointing file "
+ fileId
+ " from "
+ mLocalDataFolder.toString()
+ " to "
+ midPath
+ " to "
+ dstPath);
if (mCheckpointUfs == null) {
mCheckpointUfs = UnderFileSystem.get(midPath);
}
final long startCopyTimeMs = System.currentTimeMillis();
ClientFileInfo fileInfo = mMasterClient.getFileStatus(fileId, "");
if (!fileInfo.isComplete) {
LOG.error("File " + fileInfo + " is not complete!");
continue;
}
for (int k = 0; k < fileInfo.blockIds.size(); k++) {
lockBlock(fileInfo.blockIds.get(k), Users.CHECKPOINT_USER_ID);
}
OutputStream os = mCheckpointUfs.create(midPath, (int) fileInfo.getBlockSizeByte());
long fileSizeByte = 0;
for (int k = 0; k < fileInfo.blockIds.size(); k++) {
File tempFile =
new File(CommonUtils.concat(mLocalDataFolder.toString(), fileInfo.blockIds.get(k)));
fileSizeByte += tempFile.length();
InputStream is = new FileInputStream(tempFile);
byte[] buf = new byte[16 * Constants.KB];
int got = is.read(buf);
while (got != -1) {
os.write(buf, 0, got);
got = is.read(buf);
}
is.close();
}
os.close();
if (!mCheckpointUfs.rename(midPath, dstPath)) {
LOG.error("Failed to rename from " + midPath + " to " + dstPath);
}
mMasterClient.addCheckpoint(mWorkerId, fileId, fileSizeByte, dstPath);
for (int k = 0; k < fileInfo.blockIds.size(); k++) {
unlockBlock(fileInfo.blockIds.get(k), Users.CHECKPOINT_USER_ID);
}
long shouldTakeMs =
(long)
(1000.0
* fileSizeByte
/ Constants.MB
/ WorkerConf.get().WORKER_PER_THREAD_CHECKPOINT_CAP_MB_SEC);
long currentTimeMs = System.currentTimeMillis();
if (startCopyTimeMs + shouldTakeMs > currentTimeMs) {
long shouldSleepMs = startCopyTimeMs + shouldTakeMs - currentTimeMs;
LOG.info(
"Checkpointed last file "
+ fileId
+ " took "
+ (currentTimeMs - startCopyTimeMs)
+ " ms. Need to sleep "
+ shouldSleepMs
+ " ms.");
CommonUtils.sleepMs(LOG, shouldSleepMs);
}
} catch (FileDoesNotExistException e) {
LOG.warn(e.getMessage(), e);
} catch (SuspectedFileSizeException e) {
LOG.error(e.getMessage(), e);
} catch (BlockInfoException e) {
LOG.error(e.getMessage(), e);
} catch (IOException e) {
LOG.error(e.getMessage(), e);
} catch (TException e) {
LOG.warn(e.getMessage(), e);
}
}
}