Esempio n. 1
0
  /*
   * @Test public void locationTest() throws IOException { TachyonFile file =
   * TachyonFSTestUtils.createByteFile(mTfs, "/testFile", CacheType.STORE,
   * UnderStorageType.NO_PERSIST, 10); mFsShell.run(new String[] {"location", "/testFile"});
   *
   * FileInfo fileInfo = mTfs.getInfo(file); Assert.assertNotNull(fileInfo); List<String>
   * locationsList = tFile.getLocationHosts(); String[] commandParameters = new String[3 +
   * locationsList.size()]; commandParameters[0] = "location"; commandParameters[1] = "/testFile";
   * commandParameters[2] = String.valueOf(file.getFileId()); Iterator<String> iter =
   * locationsList.iterator(); int i = 3; while (iter.hasNext()) { commandParameters[i ++] =
   * iter.next(); } Assert.assertEquals(getCommandOutput(commandParameters), mOutput.toString()); }
   */
  @Test
  public void lsrTest() throws IOException, TException {
    FileInfo[] files = new FileInfo[4];

    TachyonFile fileA =
        TachyonFSTestUtils.createByteFile(
            mTfs, "/testRoot/testFileA", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
    files[0] = mTfs.getInfo(fileA);
    TachyonFSTestUtils.createByteFile(
        mTfs,
        "/testRoot/testDir/testFileB",
        TachyonStorageType.STORE,
        UnderStorageType.NO_PERSIST,
        20);
    files[1] = mTfs.getInfo(mTfs.open(new TachyonURI("/testRoot/testDir")));
    files[2] = mTfs.getInfo(mTfs.open(new TachyonURI("/testRoot/testDir/testFileB")));
    TachyonFile fileC =
        TachyonFSTestUtils.createByteFile(
            mTfs, "/testRoot/testFileC", TachyonStorageType.NO_STORE, UnderStorageType.PERSIST, 30);
    files[3] = mTfs.getInfo(fileC);
    mFsShell.run(new String[] {"lsr", "/testRoot"});
    String expected = "";
    String format = "%-10s%-25s%-15s%-5s\n";
    expected +=
        String.format(
            format,
            FormatUtils.getSizeFromBytes(10),
            TFsShell.convertMsToDate(files[0].getCreationTimeMs()),
            "In Memory",
            "/testRoot/testFileA");
    expected +=
        String.format(
            format,
            FormatUtils.getSizeFromBytes(0),
            TFsShell.convertMsToDate(files[1].getCreationTimeMs()),
            "",
            "/testRoot/testDir");
    expected +=
        String.format(
            format,
            FormatUtils.getSizeFromBytes(20),
            TFsShell.convertMsToDate(files[2].getCreationTimeMs()),
            "In Memory",
            "/testRoot/testDir/testFileB");
    expected +=
        String.format(
            format,
            FormatUtils.getSizeFromBytes(30),
            TFsShell.convertMsToDate(files[3].getCreationTimeMs()),
            "Not In Memory",
            "/testRoot/testFileC");
    Assert.assertEquals(expected, mOutput.toString());
  }
Esempio n. 2
0
  private void initStorageTier() throws AlreadyExistsException, IOException, OutOfSpaceException {
    String workerDataFolder =
        WorkerContext.getConf().get(Constants.WORKER_DATA_FOLDER, Constants.DEFAULT_DATA_FOLDER);
    String tierDirPathConf =
        String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, mTierLevel);
    String[] dirPaths = WorkerContext.getConf().get(tierDirPathConf, "/mnt/ramdisk").split(",");

    // Add the worker data folder path after each storage directory, the final path will be like
    // /mnt/ramdisk/tachyonworker
    for (int i = 0; i < dirPaths.length; i++) {
      dirPaths[i] = PathUtils.concatPath(dirPaths[i].trim(), workerDataFolder);
    }

    String tierDirCapacityConf =
        String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_QUOTA_FORMAT, mTierLevel);
    String[] dirQuotas = WorkerContext.getConf().get(tierDirCapacityConf, "0").split(",");

    mDirs = new ArrayList<StorageDir>(dirPaths.length);

    long totalCapacity = 0;
    for (int i = 0; i < dirPaths.length; i++) {
      int index = i >= dirQuotas.length ? dirQuotas.length - 1 : i;
      long capacity = FormatUtils.parseSpaceSize(dirQuotas[index]);
      totalCapacity += capacity;
      mDirs.add(StorageDir.newStorageDir(this, i, capacity, dirPaths[i]));
    }
    mCapacityBytes = totalCapacity;
  }
Esempio n. 3
0
  /**
   * Marks a block as committed on a specific worker.
   *
   * @param workerId the worker id committing the block
   * @param usedBytesOnTier the updated used bytes on the tier of the worker
   * @param tierAlias the alias of the storage tier where the worker is committing the block to
   * @param blockId the committing block id
   * @param length the length of the block
   */
  public void commitBlock(
      long workerId, long usedBytesOnTier, String tierAlias, long blockId, long length) {
    LOG.debug(
        "Commit block from worker: {}",
        FormatUtils.parametersToString(workerId, usedBytesOnTier, blockId, length));
    synchronized (mBlocks) {
      synchronized (mWorkers) {
        MasterWorkerInfo workerInfo = mWorkers.getFirstByField(mIdIndex, workerId);
        workerInfo.addBlock(blockId);
        workerInfo.updateUsedBytes(tierAlias, usedBytesOnTier);
        workerInfo.updateLastUpdatedTimeMs();

        MasterBlockInfo masterBlockInfo = mBlocks.get(blockId);
        if (masterBlockInfo == null) {
          masterBlockInfo = new MasterBlockInfo(blockId, length);
          mBlocks.put(blockId, masterBlockInfo);
          BlockInfoEntry blockInfo =
              BlockInfoEntry.newBuilder()
                  .setBlockId(masterBlockInfo.getBlockId())
                  .setLength(masterBlockInfo.getLength())
                  .build();
          writeJournalEntry(JournalEntry.newBuilder().setBlockInfo(blockInfo).build());
          flushJournal();
        }
        masterBlockInfo.addWorker(workerId, tierAlias);
        mLostBlocks.remove(blockId);
      }
    }
  }
Esempio n. 4
0
 public long getBytes(String key) {
   if (mProperties.containsKey(key)) {
     String rawValue = get(key);
     try {
       return FormatUtils.parseSpaceSize(rawValue);
     } catch (Exception ex) {
       throw new RuntimeException("Configuration cannot evaluate key " + key + " as bytes.");
     }
   }
   throw new RuntimeException("Invalid configuration key " + key + ".");
 }
Esempio n. 5
0
 @Test
 public void lsrTest() throws IOException {
   int fileIdA =
       TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileA", WriteType.MUST_CACHE, 10);
   TachyonFile[] files = new TachyonFile[4];
   files[0] = mTfs.getFile(fileIdA);
   TachyonFSTestUtils.createByteFile(
       mTfs, "/testRoot/testDir/testFileB", WriteType.MUST_CACHE, 20);
   files[1] = mTfs.getFile(new TachyonURI("/testRoot/testDir"));
   files[2] = mTfs.getFile(new TachyonURI("/testRoot/testDir/testFileB"));
   int fileIdC =
       TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileC", WriteType.THROUGH, 30);
   files[3] = mTfs.getFile(fileIdC);
   mFsShell.ls(new String[] {"count", "/testRoot"});
   String expected = "";
   String format = "%-10s%-25s%-15s%-5s\n";
   expected +=
       String.format(
           format,
           FormatUtils.getSizeFromBytes(10),
           TFsShell.convertMsToDate(files[0].getCreationTimeMs()),
           "In Memory",
           "/testRoot/testFileA");
   expected +=
       String.format(
           format,
           FormatUtils.getSizeFromBytes(0),
           TFsShell.convertMsToDate(files[1].getCreationTimeMs()),
           "",
           "/testRoot/testDir");
   expected +=
       String.format(
           format,
           FormatUtils.getSizeFromBytes(30),
           TFsShell.convertMsToDate(files[3].getCreationTimeMs()),
           "Not In Memory",
           "/testRoot/testFileC");
   Assert.assertEquals(expected, mOutput.toString());
 }
Esempio n. 6
0
  /**
   * Marks a block as committed, but without a worker location. This means the block is only in ufs.
   *
   * @param blockId the id of the block to commit
   * @param length the length of the block
   */
  public void commitBlockInUFS(long blockId, long length) {
    LOG.debug("Commit block to ufs: {}", FormatUtils.parametersToString(blockId, length));
    synchronized (mBlocks) {
      MasterBlockInfo masterBlockInfo = mBlocks.get(blockId);
      if (masterBlockInfo == null) {
        // The block has not been committed previously, so add the metadata to commit the block.
        masterBlockInfo = new MasterBlockInfo(blockId, length);
        mBlocks.put(blockId, masterBlockInfo);

        BlockInfoEntry blockInfo =
            BlockInfoEntry.newBuilder()
                .setBlockId(masterBlockInfo.getBlockId())
                .setLength(masterBlockInfo.getLength())
                .build();
        writeJournalEntry(JournalEntry.newBuilder().setBlockInfo(blockInfo).build());
        flushJournal();
      }
    }
  }
Esempio n. 7
0
 /**
  * Displays information for all directories and files directly under the path specified in argv.
  *
  * @param path The TachyonURI path as the input of the command
  * @return 0 if command is successful, -1 if an error occurred.
  * @throws IOException
  */
 public int ls(TachyonURI path) throws IOException {
   List<FileInfo> files = listStatusSortedByIncreasingCreationTime(path);
   String format = "%-10s%-25s%-15s%-5s%n";
   for (FileInfo file : files) {
     String inMemory = "";
     if (!file.isFolder) {
       if (100 == file.inMemoryPercentage) {
         inMemory = "In Memory";
       } else {
         inMemory = "Not In Memory";
       }
     }
     System.out.format(
         format,
         FormatUtils.getSizeFromBytes(file.getLength()),
         convertMsToDate(file.getCreationTimeMs()),
         inMemory,
         file.getPath());
   }
   return 0;
 }