/**
   * Gets the UiFileInfo object that represents the fileId, or the filePath if fileId is -1.
   *
   * @param tachyonClient the TachyonFS client.
   * @param fileId the file id of the file.
   * @param filePath the path of the file. valid iff fileId is -1.
   * @return the UiFileInfo object of the file.
   * @throws FileDoesNotExistException
   * @throws IOException
   */
  private UiFileInfo getUiFileInfo(TachyonFS tachyonClient, int fileId, TachyonURI filePath)
      throws FileDoesNotExistException, NotFoundException, IOException {
    ClientFileInfo fileInfo = tachyonClient.getFileStatus(fileId, filePath, true);
    if (fileInfo == null) {
      throw new FileDoesNotExistException(
          fileId != -1 ? Integer.toString(fileId) : filePath.toString());
    }

    UiFileInfo uiFileInfo = new UiFileInfo(fileInfo);
    boolean blockExistOnWorker = false;
    for (long blockId : fileInfo.getBlockIds()) {
      if (mBlockDataManager.hasBlockMeta(blockId)) {
        blockExistOnWorker = true;
        BlockMeta blockMeta = mBlockDataManager.getVolatileBlockMeta(blockId);
        long blockSize = blockMeta.getBlockSize();
        StorageLevelAlias storageLevelAlias =
            StorageDirId.getStorageLevelAlias(blockMeta.getParentDir().getStorageDirId());
        // The block last access time is not available. Use -1 for now.
        uiFileInfo.addBlock(storageLevelAlias, blockId, blockSize, -1);
      }
    }
    if (!blockExistOnWorker) {
      throw new FileDoesNotExistException(
          fileId != -1 ? Integer.toString(fileId) : filePath.toString());
    }
    return uiFileInfo;
  }
Пример #2
0
  /**
   * Return the {@code InStream} of this file based on the specified read type. If it has no block,
   * return an {@code EmptyBlockInStream}; if it has only one block, return a {@code BlockInStream}
   * of the block; otherwise return a {@code FileInStream}.
   *
   * @param readType the InStream's read type
   * @return the InStream
   * @throws IOException
   */
  public InStream getInStream(ReadType readType) throws IOException {
    if (readType == null) {
      throw new IOException("ReadType can not be null.");
    }

    if (!isComplete()) {
      throw new IOException("The file " + this + " is not complete.");
    }

    if (isDirectory()) {
      throw new IOException("Cannot open a directory for reading.");
    }

    ClientFileInfo fileStatus = getUnCachedFileStatus();
    List<Long> blocks = fileStatus.getBlockIds();

    if (blocks.size() == 0) {
      return new EmptyBlockInStream(this, readType, mTachyonConf);
    }

    if (blocks.size() == 1) {
      return BlockInStream.get(this, readType, 0, mUFSConf, mTachyonConf);
    }

    return new FileInStream(this, readType, mUFSConf, mTachyonConf);
  }
Пример #3
0
  /**
   * Return the under filesystem path in the under file system of this file
   *
   * @return the under filesystem path
   * @throws IOException
   */
  String getUfsPath() throws IOException {
    ClientFileInfo info = getCachedFileStatus();

    if (!info.getUfsPath().isEmpty()) {
      return info.getUfsPath();
    }

    return getUnCachedFileStatus().getUfsPath();
  }
 @Test
 public void addCheckpointTest()
     throws FileDoesNotExistException, SuspectedFileSizeException, FileAlreadyExistException,
         InvalidPathException, BlockInfoException, FileNotFoundException, TachyonException {
   int fileId = mMasterInfo.createFile("/testFile", Constants.DEFAULT_BLOCK_SIZE_BYTE);
   ClientFileInfo fileInfo = mMasterInfo.getClientFileInfo("/testFile");
   Assert.assertEquals("", fileInfo.getUfsPath());
   mMasterInfo.addCheckpoint(-1, fileId, 1, "/testPath");
   fileInfo = mMasterInfo.getClientFileInfo("/testFile");
   Assert.assertEquals("/testPath", fileInfo.getUfsPath());
   mMasterInfo.addCheckpoint(-1, fileId, 1, "/testPath");
   fileInfo = mMasterInfo.getClientFileInfo("/testFile");
   Assert.assertEquals("/testPath", fileInfo.getUfsPath());
 }
 @Test
 public void clientFileInfoEmptyFileTest()
     throws InvalidPathException, FileDoesNotExistException, FileAlreadyExistException,
         BlockInfoException, TachyonException {
   int fileId = mMasterInfo.createFile("/testFile", Constants.DEFAULT_BLOCK_SIZE_BYTE);
   ClientFileInfo fileInfo = mMasterInfo.getClientFileInfo("/testFile");
   Assert.assertEquals("testFile", fileInfo.getName());
   Assert.assertEquals(fileId, fileInfo.getId());
   Assert.assertEquals(0, fileInfo.getLength());
   Assert.assertEquals("", fileInfo.getUfsPath());
   Assert.assertFalse(fileInfo.isFolder);
   Assert.assertFalse(fileInfo.isPinned);
   Assert.assertTrue(fileInfo.isCache);
   Assert.assertFalse(fileInfo.isComplete);
 }
 @Test
 public void clientFileInfoDirectoryTest()
     throws InvalidPathException, FileDoesNotExistException, FileAlreadyExistException,
         TachyonException {
   Assert.assertTrue(mMasterInfo.mkdirs("/testFolder", true));
   ClientFileInfo fileInfo = mMasterInfo.getClientFileInfo("/testFolder");
   Assert.assertEquals("testFolder", fileInfo.getName());
   Assert.assertEquals(2, fileInfo.getId());
   Assert.assertEquals(0, fileInfo.getLength());
   Assert.assertEquals("", fileInfo.getUfsPath());
   Assert.assertTrue(fileInfo.isFolder);
   Assert.assertFalse(fileInfo.isPinned);
   Assert.assertFalse(fileInfo.isCache);
   Assert.assertTrue(fileInfo.isComplete);
 }
Пример #7
0
  /**
   * Notify the worker to checkpoint the file asynchronously.
   *
   * @param fileId The id of the file
   * @return true if succeed, false otherwise
   * @throws IOException
   * @throws TException
   */
  public boolean asyncCheckpoint(int fileId) throws IOException, TException {
    ClientFileInfo fileInfo = mMasterClient.getClientFileInfoById(fileId);

    if (fileInfo.getDependencyId() != -1) {
      synchronized (mDependencyLock) {
        mUncheckpointFiles.add(fileId);
        if (!mDepIdToFiles.containsKey(fileInfo.getDependencyId())) {
          mDepIdToFiles.put(fileInfo.getDependencyId(), new HashSet<Integer>());
        }
        mDepIdToFiles.get(fileInfo.getDependencyId()).add(fileId);
      }
      return true;
    }

    return false;
  }
Пример #8
0
 public UiFileInfo(ClientFileInfo fileInfo) {
   mId = fileInfo.getId();
   mDependencyId = fileInfo.getDependencyId();
   mName = fileInfo.getName();
   mAbsolutePath = fileInfo.getPath();
   mCheckpointPath = fileInfo.getUfsPath();
   mBlockSizeBytes = fileInfo.getBlockSizeByte();
   mSize = fileInfo.getLength();
   mCreationTimeMs = fileInfo.getCreationTimeMs();
   mLastModificationTimeMs = fileInfo.getLastModificationTimeMs();
   mInMemory = (100 == fileInfo.inMemoryPercentage);
   mInMemoryPercent = fileInfo.getInMemoryPercentage();
   mIsDirectory = fileInfo.isFolder;
   mIsPinned = fileInfo.isPinned;
   mFileLocations = new ArrayList<String>();
 }
Пример #9
0
 private UiFileInfo(ClientFileInfo fileInfo) {
   ID = fileInfo.getId();
   NAME = fileInfo.getName();
   ABSOLUATE_PATH = fileInfo.getPath();
   CHECKPOINT_PATH = fileInfo.getCheckpointPath();
   SIZE = fileInfo.getSizeBytes();
   CREATION_TIME_MS = fileInfo.getCreationTimeMs();
   IN_MEMORY = fileInfo.isInMemory();
   IS_DIRECTORY = fileInfo.isFolder();
   mFileLocations = new ArrayList<String>();
 }
Пример #10
0
    @Override
    public void run() {
      while (true) {
        try {
          int fileId = -1;
          synchronized (mDependencyLock) {
            fileId = getFileIdBasedOnPriorityDependency();

            if (fileId == -1) {
              if (mPriorityDependencies.size() == 0) {
                mPriorityDependencies = getSortedPriorityDependencyList();
                if (!mPriorityDependencies.isEmpty()) {
                  LOG.info(
                      "Get new mPriorityDependencies "
                          + CommonUtils.listToString(mPriorityDependencies));
                }
              } else {
                List<Integer> tList = getSortedPriorityDependencyList();
                boolean equal = true;
                if (mPriorityDependencies.size() != tList.size()) {
                  equal = false;
                }
                if (equal) {
                  for (int k = 0; k < tList.size(); k++) {
                    if (tList.get(k) != mPriorityDependencies.get(k)) {
                      equal = false;
                      break;
                    }
                  }
                }

                if (!equal) {
                  mPriorityDependencies = tList;
                }
              }

              fileId = getFileIdBasedOnPriorityDependency();
            }

            if (fileId == -1) {
              fileId = getRandomUncheckpointedFile();
            }
          }

          if (fileId == -1) {
            LOG.debug("Thread " + ID + " has nothing to checkpoint. Sleep for 1 sec.");
            CommonUtils.sleepMs(LOG, Constants.SECOND_MS);
            continue;
          }

          // TODO checkpoint process. In future, move from midPath to dstPath should be done by
          // master
          String midPath = CommonUtils.concat(mUnderfsWorkerDataFolder, fileId);
          String dstPath = CommonUtils.concat(CommonConf.get().UNDERFS_DATA_FOLDER, fileId);
          LOG.info(
              "Thread "
                  + ID
                  + " is checkpointing file "
                  + fileId
                  + " from "
                  + mLocalDataFolder.toString()
                  + " to "
                  + midPath
                  + " to "
                  + dstPath);

          if (mCheckpointUnderFs == null) {
            mCheckpointUnderFs = UnderFileSystem.get(midPath);
          }

          long startCopyTimeMs = System.currentTimeMillis();
          ClientFileInfo fileInfo = mMasterClient.getClientFileInfoById(fileId);
          if (!fileInfo.isComplete) {
            LOG.error("File " + fileInfo + " is not complete!");
            continue;
          }
          for (int k = 0; k < fileInfo.blockIds.size(); k++) {
            lockBlock(fileInfo.blockIds.get(k), Users.sCHECKPOINT_USER_ID);
          }
          OutputStream os = mCheckpointUnderFs.create(midPath, (int) fileInfo.getBlockSizeByte());
          long fileSizeByte = 0;
          for (int k = 0; k < fileInfo.blockIds.size(); k++) {
            File tempFile =
                new File(CommonUtils.concat(mLocalDataFolder.toString(), fileInfo.blockIds.get(k)));
            fileSizeByte += tempFile.length();
            InputStream is = new FileInputStream(tempFile);
            byte[] buf = new byte[16 * Constants.KB];
            int got = is.read(buf);
            while (got != -1) {
              os.write(buf, 0, got);
              got = is.read(buf);
            }
            is.close();
          }
          os.close();
          if (!mCheckpointUnderFs.rename(midPath, dstPath)) {
            LOG.error("Failed to rename from " + midPath + " to " + dstPath);
          }
          mMasterClient.addCheckpoint(mWorkerId, fileId, fileSizeByte, dstPath);
          for (int k = 0; k < fileInfo.blockIds.size(); k++) {
            unlockBlock(fileInfo.blockIds.get(k), Users.sCHECKPOINT_USER_ID);
          }

          long shouldTakeMs =
              (long)
                  (1000.0
                      * fileSizeByte
                      / Constants.MB
                      / WorkerConf.get().WORKER_PER_THREAD_CHECKPOINT_CAP_MB_SEC);
          long currentTimeMs = System.currentTimeMillis();
          if (startCopyTimeMs + shouldTakeMs > currentTimeMs) {
            long shouldSleepMs = startCopyTimeMs + shouldTakeMs - currentTimeMs;
            LOG.info(
                "Checkpointed last file "
                    + fileId
                    + " took "
                    + (currentTimeMs - startCopyTimeMs)
                    + " ms. Need to sleep "
                    + shouldSleepMs
                    + " ms.");
            CommonUtils.sleepMs(LOG, shouldSleepMs);
          }
        } catch (FileDoesNotExistException e) {
          LOG.warn(e);
        } catch (SuspectedFileSizeException e) {
          LOG.error(e);
        } catch (BlockInfoException e) {
          LOG.error(e);
        } catch (IOException e) {
          LOG.error(e);
        } catch (TException e) {
          LOG.warn(e);
        }
      }
    }
  /**
   * Populates attribute fields with data from the MasterInfo associated with this servlet. Errors
   * will be displayed in an error field. Debugging can be enabled to display additional data. Will
   * eventually redirect the request to a jsp.
   *
   * @param request The HttpServletRequest object
   * @param response The HttpServletResponse object
   * @throws ServletException
   * @throws IOException
   * @throws UnknownHostException
   */
  @Override
  protected void doGet(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException, UnknownHostException {
    request.setAttribute("debug", Constants.DEBUG);

    request.setAttribute("masterNodeAddress", mMasterInfo.getMasterAddress().toString());
    request.setAttribute("invalidPathError", "");
    List<ClientFileInfo> filesInfo = null;
    String currentPath = request.getParameter("path");
    if (currentPath == null || currentPath.isEmpty()) {
      currentPath = Constants.PATH_SEPARATOR;
    }
    request.setAttribute("currentPath", currentPath);
    request.setAttribute("viewingOffset", 0);
    try {
      ClientFileInfo clientFileInfo = mMasterInfo.getClientFileInfo(currentPath);
      UiFileInfo currentFileInfo = new UiFileInfo(clientFileInfo);
      request.setAttribute("currentDirectory", currentFileInfo);
      request.setAttribute("blockSizeByte", currentFileInfo.getBlockSizeBytes());
      if (!currentFileInfo.getIsDirectory()) {
        String tmpParam = request.getParameter("offset");
        long offset = 0;
        try {
          if (tmpParam != null) {
            offset = Long.valueOf(tmpParam);
          }
        } catch (NumberFormatException nfe) {
          offset = 0;
        }
        if (offset < 0) {
          offset = 0;
        } else if (offset > clientFileInfo.getLength()) {
          offset = clientFileInfo.getLength();
        }
        displayFile(currentFileInfo.getAbsolutePath(), request, offset);
        request.setAttribute("viewingOffset", offset);
        getServletContext().getRequestDispatcher("/viewFile.jsp").forward(request, response);
        return;
      }
      CommonUtils.validatePath(currentPath);
      setPathDirectories(currentPath, request);
      filesInfo = mMasterInfo.getFilesInfo(currentPath);
    } catch (FileDoesNotExistException fdne) {
      request.setAttribute("invalidPathError", "Error: Invalid Path " + fdne.getMessage());
      getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
      return;
    } catch (InvalidPathException ipe) {
      request.setAttribute("invalidPathError", "Error: Invalid Path " + ipe.getLocalizedMessage());
      getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
      return;
    } catch (IOException ie) {
      request.setAttribute(
          "invalidPathError",
          "Error: File " + currentPath + " is not available " + ie.getMessage());
      getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
      return;
    }

    List<UiFileInfo> fileInfos = new ArrayList<UiFileInfo>(filesInfo.size());
    for (ClientFileInfo fileInfo : filesInfo) {
      UiFileInfo toAdd = new UiFileInfo(fileInfo);
      try {
        if (!toAdd.getIsDirectory() && fileInfo.getLength() > 0) {
          toAdd.setFileLocations(mMasterInfo.getFileBlocks(toAdd.getId()).get(0).getLocations());
        }
      } catch (FileDoesNotExistException fdne) {
        request.setAttribute("invalidPathError", "Error: Invalid Path " + fdne.getMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
      }
      fileInfos.add(toAdd);
    }
    Collections.sort(fileInfos);
    request.setAttribute("fileInfos", fileInfos);

    getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
  }
Пример #12
0
  @Override
  public ClientFileInfo generateClientFileInfo(String path) {
    ClientFileInfo ret = new ClientFileInfo();

    ret.id = getId();
    ret.name = getName();
    ret.path = path;
    ret.ufsPath = mUfsPath;
    ret.length = mLength;
    ret.blockSizeByte = mBlockSizeByte;
    ret.creationTimeMs = getCreationTimeMs();
    ret.isComplete = isComplete();
    ret.isFolder = false;
    ret.isPinned = isPinned();
    ret.isCache = mCache;
    ret.blockIds = getBlockIds();
    ret.dependencyId = mDependencyId;
    ret.inMemoryPercentage = getInMemoryPercentage();
    ret.lastModificationTimeMs = getLastModificationTimeMs();

    return ret;
  }