Пример #1
0
  /**
   * Populates attribute fields with data from the MasterInfo associated with this servlet. Errors
   * will be displayed in an error field. Debugging can be enabled to display additional data. Will
   * eventually redirect the request to a jsp.
   *
   * @param request The HttpServletRequest object
   * @param response The HttpServletResponse object
   * @throws ServletException
   * @throws IOException
   * @throws UnknownHostException
   */
  @Override
  protected void doGet(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException, UnknownHostException {
    request.setAttribute("debug", Constants.DEBUG);

    request.setAttribute("masterNodeAddress", mMasterInfo.getMasterAddress().toString());
    request.setAttribute("invalidPathError", "");
    List<ClientFileInfo> filesInfo = null;
    String currentPath = request.getParameter("path");

    if (currentPath.isEmpty()) {
      currentPath = "/";
    }
    request.setAttribute("currentPath", currentPath);
    try {
      UiFileInfo currentFileInfo = new UiFileInfo(mMasterInfo.getFileInfo(currentPath));
      request.setAttribute("currentDirectory", currentFileInfo);
      if (!currentFileInfo.getIsDirectory()) {
        displayFile(currentFileInfo.getAbsolutePath(), request);
        getServletContext().getRequestDispatcher("/viewFile.jsp").forward(request, response);
        return;
      }
      CommonUtils.validatePath(currentPath);
      setPathDirectories(currentPath, request);
      filesInfo = mMasterInfo.getFilesInfo(currentPath);
    } catch (FileDoesNotExistException fdne) {
      request.setAttribute("invalidPathError", "Error: Invalid Path " + fdne.getMessage());
      getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
      return;
    } catch (InvalidPathException ipe) {
      request.setAttribute("invalidPathError", "Error: Invalid Path " + ipe.getLocalizedMessage());
      getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
      return;
    }

    List<UiFileInfo> fileInfos = new ArrayList<UiFileInfo>(filesInfo.size());
    for (ClientFileInfo fileInfo : filesInfo) {
      UiFileInfo toAdd = new UiFileInfo(fileInfo);
      try {
        if (!toAdd.getIsDirectory()) {
          toAdd.setFileLocations(mMasterInfo.getFileLocations(toAdd.getId()));
        }
      } catch (FileDoesNotExistException fdne) {
        request.setAttribute("invalidPathError", "Error: Invalid Path " + fdne.getMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
      }
      fileInfos.add(toAdd);
    }
    Collections.sort(fileInfos);
    request.setAttribute("fileInfos", fileInfos);

    getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
  }
  /**
   * Populates attributes before redirecting to a jsp.
   *
   * @param request The HttpServletRequest object
   * @param response The HttpServletReponse object
   * @throws ServletException
   * @throws IOException
   */
  @Override
  protected void doGet(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException {
    request.setAttribute("fatalError", "");
    TachyonFS tachyonClient = TachyonFS.get(mTachyonConf);

    String filePath = request.getParameter("path");
    if (!(filePath == null || filePath.isEmpty())) {
      // Display file block info
      try {
        UiFileInfo uiFileInfo = getUiFileInfo(tachyonClient, new TachyonURI(filePath));
        request.setAttribute("fileBlocksOnTier", uiFileInfo.getBlocksOnTier());
        request.setAttribute("blockSizeByte", uiFileInfo.getBlockSizeBytes());
        request.setAttribute("path", filePath);

        getServletContext()
            .getRequestDispatcher("/worker/viewFileBlocks.jsp")
            .forward(request, response);
        return;
      } catch (FileDoesNotExistException fdne) {
        request.setAttribute("fatalError", "Error: Invalid Path " + fdne.getMessage());
        getServletContext()
            .getRequestDispatcher("/worker/blockInfo.jsp")
            .forward(request, response);
        return;
      } catch (IOException ie) {
        request.setAttribute(
            "invalidPathError", "Error: File " + filePath + " is not available " + ie.getMessage());
        getServletContext()
            .getRequestDispatcher("/worker/blockInfo.jsp")
            .forward(request, response);
        return;
      } catch (NotFoundException nfe) {
        request.setAttribute("fatalError", "Error: block not found. " + nfe.getMessage());
        getServletContext()
            .getRequestDispatcher("/worker/blockInfo.jsp")
            .forward(request, response);
        return;
      }
    }

    List<Integer> fileIds = getSortedFileIds();
    request.setAttribute("nTotalFile", fileIds.size());

    // URL can not determine offset and limit, let javascript in jsp determine and redirect
    if (request.getParameter("offset") == null && request.getParameter("limit") == null) {
      getServletContext().getRequestDispatcher("/worker/blockInfo.jsp").forward(request, response);
      return;
    }

    try {
      int offset = Integer.parseInt(request.getParameter("offset"));
      int limit = Integer.parseInt(request.getParameter("limit"));
      List<Integer> subFileIds = fileIds.subList(offset, offset + limit);
      List<UiFileInfo> uiFileInfos = new ArrayList<UiFileInfo>(subFileIds.size());
      for (int fileId : subFileIds) {
        uiFileInfos.add(getUiFileInfo(tachyonClient, fileId));
      }
      request.setAttribute("fileInfos", uiFileInfos);
    } catch (FileDoesNotExistException fdne) {
      request.setAttribute("fatalError", "Error: Invalid FileId " + fdne.getMessage());
      getServletContext().getRequestDispatcher("/worker/blockInfo.jsp").forward(request, response);
      return;
    } catch (NumberFormatException nfe) {
      request.setAttribute(
          "fatalError", "Error: offset or limit parse error, " + nfe.getLocalizedMessage());
      getServletContext().getRequestDispatcher("/worker/blockInfo.jsp").forward(request, response);
      return;
    } catch (IndexOutOfBoundsException iobe) {
      request.setAttribute(
          "fatalError",
          "Error: offset or offset + limit is out of bound, " + iobe.getLocalizedMessage());
      getServletContext().getRequestDispatcher("/worker/blockInfo.jsp").forward(request, response);
      return;
    } catch (IllegalArgumentException iae) {
      request.setAttribute("fatalError", iae.getLocalizedMessage());
      getServletContext().getRequestDispatcher("/worker/blockInfo.jsp").forward(request, response);
      return;
    } catch (NotFoundException nfe) {
      request.setAttribute("fatalError", nfe.getLocalizedMessage());
      getServletContext().getRequestDispatcher("/worker/blockInfo.jsp").forward(request, response);
      return;
    }

    getServletContext().getRequestDispatcher("/worker/blockInfo.jsp").forward(request, response);
  }
  /**
   * Populates attribute fields with data from the MasterInfo associated with this servlet. Errors
   * will be displayed in an error field. Debugging can be enabled to display additional data. Will
   * eventually redirect the request to a jsp.
   *
   * @param request The HttpServletRequest object
   * @param response The HttpServletResponse object
   * @throws ServletException
   * @throws IOException
   * @throws UnknownHostException
   */
  @Override
  protected void doGet(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException, UnknownHostException {
    request.setAttribute("debug", Constants.DEBUG);

    request.setAttribute("masterNodeAddress", mMasterInfo.getMasterAddress().toString());
    request.setAttribute("invalidPathError", "");
    List<ClientFileInfo> filesInfo = null;
    String currentPath = request.getParameter("path");
    if (currentPath == null || currentPath.isEmpty()) {
      currentPath = Constants.PATH_SEPARATOR;
    }
    request.setAttribute("currentPath", currentPath);
    request.setAttribute("viewingOffset", 0);
    try {
      ClientFileInfo clientFileInfo = mMasterInfo.getClientFileInfo(currentPath);
      UiFileInfo currentFileInfo = new UiFileInfo(clientFileInfo);
      request.setAttribute("currentDirectory", currentFileInfo);
      request.setAttribute("blockSizeByte", currentFileInfo.getBlockSizeBytes());
      if (!currentFileInfo.getIsDirectory()) {
        String tmpParam = request.getParameter("offset");
        long offset = 0;
        try {
          if (tmpParam != null) {
            offset = Long.valueOf(tmpParam);
          }
        } catch (NumberFormatException nfe) {
          offset = 0;
        }
        if (offset < 0) {
          offset = 0;
        } else if (offset > clientFileInfo.getLength()) {
          offset = clientFileInfo.getLength();
        }
        displayFile(currentFileInfo.getAbsolutePath(), request, offset);
        request.setAttribute("viewingOffset", offset);
        getServletContext().getRequestDispatcher("/viewFile.jsp").forward(request, response);
        return;
      }
      CommonUtils.validatePath(currentPath);
      setPathDirectories(currentPath, request);
      filesInfo = mMasterInfo.getFilesInfo(currentPath);
    } catch (FileDoesNotExistException fdne) {
      request.setAttribute("invalidPathError", "Error: Invalid Path " + fdne.getMessage());
      getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
      return;
    } catch (InvalidPathException ipe) {
      request.setAttribute("invalidPathError", "Error: Invalid Path " + ipe.getLocalizedMessage());
      getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
      return;
    } catch (IOException ie) {
      request.setAttribute(
          "invalidPathError",
          "Error: File " + currentPath + " is not available " + ie.getMessage());
      getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
      return;
    }

    List<UiFileInfo> fileInfos = new ArrayList<UiFileInfo>(filesInfo.size());
    for (ClientFileInfo fileInfo : filesInfo) {
      UiFileInfo toAdd = new UiFileInfo(fileInfo);
      try {
        if (!toAdd.getIsDirectory() && fileInfo.getLength() > 0) {
          toAdd.setFileLocations(mMasterInfo.getFileBlocks(toAdd.getId()).get(0).getLocations());
        }
      } catch (FileDoesNotExistException fdne) {
        request.setAttribute("invalidPathError", "Error: Invalid Path " + fdne.getMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
      }
      fileInfos.add(toAdd);
    }
    Collections.sort(fileInfos);
    request.setAttribute("fileInfos", fileInfos);

    getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
  }
Пример #4
0
    @Override
    public void run() {
      while (true) {
        try {
          int fileId = -1;
          synchronized (mDependencyLock) {
            fileId = getFileIdBasedOnPriorityDependency();

            if (fileId == -1) {
              if (mPriorityDependencies.size() == 0) {
                mPriorityDependencies = getSortedPriorityDependencyList();
                if (!mPriorityDependencies.isEmpty()) {
                  LOG.info(
                      "Get new mPriorityDependencies "
                          + CommonUtils.listToString(mPriorityDependencies));
                }
              } else {
                List<Integer> tList = getSortedPriorityDependencyList();
                boolean equal = true;
                if (mPriorityDependencies.size() != tList.size()) {
                  equal = false;
                }
                if (equal) {
                  for (int k = 0; k < tList.size(); k++) {
                    if (tList.get(k) != mPriorityDependencies.get(k)) {
                      equal = false;
                      break;
                    }
                  }
                }

                if (!equal) {
                  mPriorityDependencies = tList;
                }
              }

              fileId = getFileIdBasedOnPriorityDependency();
            }

            if (fileId == -1) {
              fileId = getRandomUncheckpointedFile();
            }
          }

          if (fileId == -1) {
            LOG.debug("Thread {} has nothing to checkpoint. Sleep for 1 sec.", mId);
            CommonUtils.sleepMs(LOG, Constants.SECOND_MS);
            continue;
          }

          // TODO checkpoint process. In future, move from midPath to dstPath should be done by
          // master
          String midPath = CommonUtils.concat(mUfsWorkerDataFolder, fileId);
          String dstPath = CommonUtils.concat(CommonConf.get().UNDERFS_DATA_FOLDER, fileId);
          LOG.info(
              "Thread "
                  + mId
                  + " is checkpointing file "
                  + fileId
                  + " from "
                  + mLocalDataFolder.toString()
                  + " to "
                  + midPath
                  + " to "
                  + dstPath);

          if (mCheckpointUfs == null) {
            mCheckpointUfs = UnderFileSystem.get(midPath);
          }

          final long startCopyTimeMs = System.currentTimeMillis();
          ClientFileInfo fileInfo = mMasterClient.getFileStatus(fileId, "");
          if (!fileInfo.isComplete) {
            LOG.error("File " + fileInfo + " is not complete!");
            continue;
          }
          for (int k = 0; k < fileInfo.blockIds.size(); k++) {
            lockBlock(fileInfo.blockIds.get(k), Users.CHECKPOINT_USER_ID);
          }
          OutputStream os = mCheckpointUfs.create(midPath, (int) fileInfo.getBlockSizeByte());
          long fileSizeByte = 0;
          for (int k = 0; k < fileInfo.blockIds.size(); k++) {
            File tempFile =
                new File(CommonUtils.concat(mLocalDataFolder.toString(), fileInfo.blockIds.get(k)));
            fileSizeByte += tempFile.length();
            InputStream is = new FileInputStream(tempFile);
            byte[] buf = new byte[16 * Constants.KB];
            int got = is.read(buf);
            while (got != -1) {
              os.write(buf, 0, got);
              got = is.read(buf);
            }
            is.close();
          }
          os.close();
          if (!mCheckpointUfs.rename(midPath, dstPath)) {
            LOG.error("Failed to rename from " + midPath + " to " + dstPath);
          }
          mMasterClient.addCheckpoint(mWorkerId, fileId, fileSizeByte, dstPath);
          for (int k = 0; k < fileInfo.blockIds.size(); k++) {
            unlockBlock(fileInfo.blockIds.get(k), Users.CHECKPOINT_USER_ID);
          }

          long shouldTakeMs =
              (long)
                  (1000.0
                      * fileSizeByte
                      / Constants.MB
                      / WorkerConf.get().WORKER_PER_THREAD_CHECKPOINT_CAP_MB_SEC);
          long currentTimeMs = System.currentTimeMillis();
          if (startCopyTimeMs + shouldTakeMs > currentTimeMs) {
            long shouldSleepMs = startCopyTimeMs + shouldTakeMs - currentTimeMs;
            LOG.info(
                "Checkpointed last file "
                    + fileId
                    + " took "
                    + (currentTimeMs - startCopyTimeMs)
                    + " ms. Need to sleep "
                    + shouldSleepMs
                    + " ms.");
            CommonUtils.sleepMs(LOG, shouldSleepMs);
          }
        } catch (FileDoesNotExistException e) {
          LOG.warn(e.getMessage(), e);
        } catch (SuspectedFileSizeException e) {
          LOG.error(e.getMessage(), e);
        } catch (BlockInfoException e) {
          LOG.error(e.getMessage(), e);
        } catch (IOException e) {
          LOG.error(e.getMessage(), e);
        } catch (TException e) {
          LOG.warn(e.getMessage(), e);
        }
      }
    }