Пример #1
0
 @Override
 public boolean rename(String src, String dst) throws IOException {
   if (!exists(src)) {
     LOG.error("Unable to rename " + src + " to " + dst + " because source does not exist.");
     return false;
   }
   if (exists(dst)) {
     LOG.error("Unable to rename " + src + " to " + dst + " because destination already exists.");
     return false;
   }
   // Source exists and destination does not exist
   if (isFolder(src)) {
     // Rename the source folder first
     if (!copy(convertToFolderName(src), convertToFolderName(dst))) {
       return false;
     }
     // Rename each child in the src folder to destination/child
     String[] children = list(src);
     for (String child : children) {
       if (!rename(PathUtils.concatPath(src, child), PathUtils.concatPath(dst, child))) {
         return false;
       }
     }
     // Delete src and everything under src
     return delete(src, true);
   }
   // Source is a file and Destination does not exist
   return copy(src, dst) && deleteInternal(src);
 }
  /** Test <code>long skip(long len)</code>. */
  @Test
  public void skipTest() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);

      FileInStream is = mTfs.getInStream(f, mReadCache);
      Assert.assertEquals(k / 2, is.skip(k / 2));
      Assert.assertEquals(k / 2, is.read());
      is.close();
      Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);

      if (k >= 3) {
        is = mTfs.getInStream(f, mReadCache);
        int t = k / 3;
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals(t, is.read());
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals(2 * t + 1, is.read());
        is.close();
        Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      }
    }
  }
Пример #3
0
  /**
   * Copies a list of files or directories specified by srcFiles from the local filesystem to
   * dstPath in the Tachyon filesystem space. This method is used when the input path contains
   * wildcards.
   *
   * @param srcFiles The list of files in the local filesystem
   * @param dstPath The {@link TachyonURI} of the destination
   * @throws IOException if a non-Tachyon related exception occurs
   */
  private void copyFromLocalWildcard(List<File> srcFiles, TachyonURI dstPath) throws IOException {
    try {
      mTfs.mkdir(dstPath);
    } catch (FileAlreadyExistsException e) {
      // it's fine if the directory already exists
    } catch (TachyonException e) {
      throw new IOException(e.getMessage());
    }

    FileInfo dstFileInfo;
    try {
      TachyonFile dstFd = mTfs.open(dstPath);
      dstFileInfo = mTfs.getInfo(dstFd);
    } catch (TachyonException e) {
      throw new IOException(e.getMessage());
    }
    if (!dstFileInfo.isFolder) {
      throw new IOException(
          ExceptionMessage.DESTINATION_FILE_CANNOT_EXIST_WITH_WILDCARD_SOURCE.getMessage());
    }

    List<String> errorMessages = Lists.newArrayList();
    for (File srcFile : srcFiles) {
      try {
        copyFromLocal(
            srcFile, new TachyonURI(PathUtils.concatPath(dstPath.getPath(), srcFile.getName())));
      } catch (IOException e) {
        errorMessages.add(e.getMessage());
      }
    }
    if (errorMessages.size() != 0) {
      throw new IOException(Joiner.on('\n').join(errorMessages));
    }
  }
Пример #4
0
 /**
  * Creates a new file output stream.
  *
  * @param path the file path
  * @param options the client options
  * @throws IOException if an I/O error occurs
  */
 public FileOutStream(TachyonURI path, OutStreamOptions options) throws IOException {
   mUri = Preconditions.checkNotNull(path);
   mNonce = ClientUtils.getRandomNonNegativeLong();
   mBlockSize = options.getBlockSizeBytes();
   mTachyonStorageType = options.getTachyonStorageType();
   mUnderStorageType = options.getUnderStorageType();
   mContext = FileSystemContext.INSTANCE;
   mPreviousBlockOutStreams = new LinkedList<BufferedBlockOutStream>();
   if (mUnderStorageType.isSyncPersist()) {
     updateUfsPath();
     String tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
     UnderFileSystem ufs = UnderFileSystem.get(tmpPath, ClientContext.getConf());
     // TODO(jiri): Implement collection of temporary files left behind by dead clients.
     mUnderStorageOutputStream = ufs.create(tmpPath, (int) mBlockSize);
   } else {
     mUfsPath = null;
     mUnderStorageOutputStream = null;
   }
   mClosed = false;
   mCanceled = false;
   mShouldCacheCurrentBlock = mTachyonStorageType.isStore();
   mBytesWritten = 0;
   mLocationPolicy =
       Preconditions.checkNotNull(
           options.getLocationPolicy(),
           PreconditionMessage.FILE_WRITE_LOCATION_POLICY_UNSPECIFIED);
 }
Пример #5
0
  private void initStorageTier() throws AlreadyExistsException, IOException, OutOfSpaceException {
    String workerDataFolder =
        WorkerContext.getConf().get(Constants.WORKER_DATA_FOLDER, Constants.DEFAULT_DATA_FOLDER);
    String tierDirPathConf =
        String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, mTierLevel);
    String[] dirPaths = WorkerContext.getConf().get(tierDirPathConf, "/mnt/ramdisk").split(",");

    // Add the worker data folder path after each storage directory, the final path will be like
    // /mnt/ramdisk/tachyonworker
    for (int i = 0; i < dirPaths.length; i++) {
      dirPaths[i] = PathUtils.concatPath(dirPaths[i].trim(), workerDataFolder);
    }

    String tierDirCapacityConf =
        String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_QUOTA_FORMAT, mTierLevel);
    String[] dirQuotas = WorkerContext.getConf().get(tierDirCapacityConf, "0").split(",");

    mDirs = new ArrayList<StorageDir>(dirPaths.length);

    long totalCapacity = 0;
    for (int i = 0; i < dirPaths.length; i++) {
      int index = i >= dirQuotas.length ? dirQuotas.length - 1 : i;
      long capacity = FormatUtils.parseSpaceSize(dirQuotas[index]);
      totalCapacity += capacity;
      mDirs.add(StorageDir.newStorageDir(this, i, capacity, dirPaths[i]));
    }
    mCapacityBytes = totalCapacity;
  }
  /**
   * Creates a store with the specified number of partitions.
   *
   * <p>NOTE: calling this method will set {@link Constants#KEY_VALUE_PARTITION_SIZE_BYTES_MAX} to
   * {@link Constants#MB}.
   *
   * @param partitionNumber the number of partitions
   * @param keyValuePairs the key-value pairs in the store, null if you don't want to know them
   * @return the URI to the created store
   */
  private TachyonURI createStoreOfMultiplePartitions(
      int partitionNumber, List<KeyValuePair> keyValuePairs) throws Exception {
    // These sizes are carefully selected, one partition holds only one key-value pair.
    final long maxPartitionSize = Constants.MB; // Each partition is at most 1 MB
    ClientContext.getConf()
        .set(Constants.KEY_VALUE_PARTITION_SIZE_BYTES_MAX, String.valueOf(maxPartitionSize));
    final int keyLength = 4; // 4Byte key
    final int valueLength = 500 * Constants.KB; // 500KB value

    TachyonURI storeUri = new TachyonURI(PathUtils.uniqPath());
    mWriter = sKeyValueSystem.createStore(storeUri);
    for (int i = 0; i < partitionNumber; i++) {
      byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
      byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
      mWriter.put(key, value);
      if (keyValuePairs != null) {
        keyValuePairs.add(new KeyValuePair(key, value));
      }
    }
    mWriter.close();

    Assert.assertEquals(partitionNumber, getPartitionNumber(storeUri));

    return storeUri;
  }
  /** Test <code>void read()</code>. Read from remote data server. */
  @Test
  public void readTest4() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);

      long blockId = mTfs.getInfo(f).getBlockIds().get(0);
      BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
      RemoteBlockInStream is =
          new RemoteBlockInStream(
              info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
      byte[] ret = new byte[k];
      int value = is.read();
      int cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }
  /** Test <code>void read(byte[] b, int off, int len)</code>. Read from underfs. */
  @Test
  public void readTest3() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);

      FileInStream is = mTfs.getInStream(f, mReadNoCache);
      byte[] ret = new byte[k / 2];
      Assert.assertEquals(k / 2, is.read(ret, 0, k / 2));
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k / 2, ret));
      is.close();
      if (k == 0) {
        Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      } else {
        Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      }

      is = mTfs.getInStream(f, mReadCache);
      ret = new byte[k];
      Assert.assertEquals(k, is.read(ret, 0, k));
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);

      is = mTfs.getInStream(f, mReadCache);
      ret = new byte[k];
      Assert.assertEquals(k, is.read(ret));
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }
  /** Test {@link tachyon.client.block.BufferedBlockInStream#read()}. */
  @Test
  public void readTest1() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      for (CreateFileOptions op : getOptionSet()) {
        TachyonURI path = new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode());
        FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);

        for (int i = 0; i < 2; i++) {
          FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
          byte[] ret = new byte[k];
          int value = is.read();
          int cnt = 0;
          while (value != -1) {
            Assert.assertTrue(value >= 0);
            Assert.assertTrue(value < 256);
            ret[cnt++] = (byte) value;
            value = is.read();
          }
          Assert.assertEquals(cnt, k);
          Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
          is.close();
        }
      }
    }
  }
  /** Test <code>void read()</code>. Read from underfs. */
  @Test
  public void readTest1() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);

      FileInStream is = mTfs.getInStream(f, mReadNoCache);
      byte[] ret = new byte[k];
      int value = is.read();
      int cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      if (k == 0) {
        Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      } else {
        Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
      }

      is = mTfs.getInStream(f, mReadCache);
      ret = new byte[k];
      value = is.read();
      cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);

      is = mTfs.getInStream(f, mReadCache);
      ret = new byte[k];
      value = is.read();
      cnt = 0;
      while (value != -1) {
        Assert.assertTrue(value >= 0);
        Assert.assertTrue(value < 256);
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertEquals(cnt, k);
      Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }
  /**
   * Tests that not reading a file the whole way through then closing it will cause it to not
   * recache
   */
  @Test
  public void incompleteFileReadCancelsRecache() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    TachyonFile f = TachyonFSTestUtils.createByteFile(mTfs, uniqPath, 2, mWriteUnderStore);

    FileInStream is = mTfs.getInStream(f, mReadNoCache);
    Assert.assertEquals(0, is.read());
    is.close();
    Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    is = mTfs.getInStream(f, mReadNoCache);
    is.close();
  }
 /** Tests that seeking around a file cached locally works. */
 @Test
 public void seekAroundLocalBlock() throws IOException, TachyonException {
   String uniqPath = PathUtils.uniqPath();
   // The number of bytes per remote block read should be set to 100 in the before function
   TachyonFile f = TachyonFSTestUtils.createByteFile(mTfs, uniqPath, 200, mWriteTachyon);
   FileInStream is = mTfs.getInStream(f, mReadNoCache);
   Assert.assertEquals(0, is.read());
   is.seek(199);
   Assert.assertEquals(199, is.read());
   is.seek(99);
   Assert.assertEquals(99, is.read());
   is.close();
 }
  /** Tests that reading a file the whole way through with the STORE ReadType will recache it */
  @Test
  public void completeFileReadTriggersRecache() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    int len = 2;
    TachyonFile f = TachyonFSTestUtils.createByteFile(mTfs, uniqPath, len, mWriteUnderStore);

    FileInStream is = mTfs.getInStream(f, mReadCache);
    for (int i = 0; i < len; ++i) {
      Assert.assertEquals(i, is.read());
    }
    is.close();
    Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
  }
Пример #14
0
  @Override
  public boolean delete(String path, boolean recursive) throws IOException {
    File file = new File(path);
    boolean success = true;
    if (recursive && file.isDirectory()) {
      String[] files = file.list();
      for (String child : files) {
        success = success && delete(PathUtils.concatPath(path, child), true);
      }
    }

    return success && file.delete();
  }
Пример #15
0
 /**
  * Validates the path, verifying that it contains the {@link Constants#HEADER} or {@link
  * Constants#HEADER_FT} and a hostname:port specified.
  *
  * @param path the path to be verified
  * @param tachyonConf the instance of {@link tachyon.conf.TachyonConf} to be used
  * @return the verified path in a form like tachyon://host:port/dir. If only the "/dir" or "dir"
  *     part is provided, the host and port are retrieved from property, tachyon.master.hostname
  *     and tachyon.master.port, respectively.
  * @throws IOException if the given path is not valid
  */
 public static String validatePath(String path, TachyonConf tachyonConf) throws IOException {
   if (path.startsWith(Constants.HEADER) || path.startsWith(Constants.HEADER_FT)) {
     if (!path.contains(":")) {
       throw new IOException(
           "Invalid Path: "
               + path
               + ". Use "
               + Constants.HEADER
               + "host:port/ ,"
               + Constants.HEADER_FT
               + "host:port/"
               + " , or /file");
     } else {
       return path;
     }
   } else {
     String hostname = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, tachyonConf);
     int port = tachyonConf.getInt(Constants.MASTER_PORT);
     if (tachyonConf.getBoolean(Constants.ZOOKEEPER_ENABLED)) {
       return PathUtils.concatPath(Constants.HEADER_FT + hostname + ":" + port, path);
     }
     return PathUtils.concatPath(Constants.HEADER + hostname + ":" + port, path);
   }
 }
Пример #16
0
  /**
   * Copies a list of files or directories specified by srcFiles from the local filesystem to
   * dstPath in the Tachyon filesystem space. This method is used when the input path contains
   * wildcards.
   *
   * @param srcFiles The list of files in the local filesystem
   * @param dstPath The TachyonURI of the destination
   * @return 0 if command is successful, -1 if an error occurred.
   * @throws IOException
   */
  public int copyFromLocalWildcard(List<File> srcFiles, TachyonURI dstPath) throws IOException {
    try {
      mTfs.mkdir(dstPath);
    } catch (TachyonException e) {
      switch (e.getType()) {
        case INVALID_PATH:
          System.out.print("Fail to create directory (Invalid path): " + dstPath);
          return -1;
        case FILE_DOES_NOT_EXIST:
          // this is fine
          break;
        default:
          throw new IOException(e.getMessage());
      }
    }

    FileInfo dstFileInfo;
    try {
      TachyonFile dstFd = mTfs.open(dstPath);
      if (dstFd == null) {
        System.out.print(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(dstPath));
        return -1;
      }
      dstFileInfo = mTfs.getInfo(dstFd);
    } catch (TachyonException e) {
      throw new IOException(e);
    }
    if (!dstFileInfo.isFolder) {
      System.out.println(
          "The destination cannot be an existent file when the src contains " + "wildcards.");
      return -1;
    }

    int exitCode = 0;
    for (File srcFile : srcFiles) {
      try {
        exitCode |=
            copyFromLocal(
                srcFile,
                new TachyonURI(PathUtils.concatPath(dstPath.getPath(), srcFile.getName())));
      } catch (IOException ioe) {
        System.out.println(ioe.getMessage());
        exitCode |= -1;
      }
    }
    return exitCode;
  }
  /**
   * Test <code>void seek(long pos)</code>. Validate the expected exception for seeking a position
   * that is past block size.
   *
   * @throws IOException
   * @throws TachyonException
   */
  @Test
  public void seekExceptionTest2() throws IOException, TachyonException {
    mThrown.expect(IllegalArgumentException.class);
    mThrown.expectMessage("Seek position is past EOF: 1, fileSize = 0");
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);

      FileInStream is = mTfs.getInStream(f, mReadNoCache);
      try {
        is.seek(k + 1);
      } finally {
        is.close();
      }
    }
  }
  @BeforeClass
  public static final void beforeClass() throws Exception {
    sFileSystem = sLocalTachyonClusterResource.get().getClient();
    sTachyonConf = sLocalTachyonClusterResource.get().getMasterTachyonConf();
    sWriteBoth = StreamOptionUtils.getCreateFileOptionsCacheThrough(sTachyonConf);
    sWriteTachyon = StreamOptionUtils.getCreateFileOptionsMustCache(sTachyonConf);
    sWriteUnderStore = StreamOptionUtils.getCreateFileOptionsThrough(sTachyonConf);
    sTestPath = PathUtils.uniqPath();

    // Create files of varying size and write type to later read from
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      for (CreateFileOptions op : getOptionSet()) {
        TachyonURI path = new TachyonURI(sTestPath + "/file_" + k + "_" + op.hashCode());
        FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);
      }
    }
  }
  /**
   * Creates a store with the specified number of key-value pairs. The key-value pairs are in the
   * format specified in {@link #genBaseKey(int)} and {@link #genBaseValue(int)} with id starts from
   * 0.
   *
   * <p>The created store's size is {@link Assert}ed before return.
   *
   * @param size the number of key-value pairs
   * @param pairs the key-value pairs in the store, null if you don't want to know them
   * @return the URI to the store
   * @throws Exception if any error happens
   */
  private TachyonURI createStoreOfSize(int size, List<KeyValuePair> pairs) throws Exception {
    TachyonURI path = new TachyonURI(PathUtils.uniqPath());
    KeyValueStoreWriter writer = sKeyValueSystem.createStore(path);
    for (int i = 0; i < size; i++) {
      byte[] key = genBaseKey(i).getBytes();
      byte[] value = genBaseValue(i).getBytes();
      writer.put(key, value);
      if (pairs != null) {
        pairs.add(new KeyValuePair(key, value));
      }
    }
    writer.close();

    Assert.assertEquals(size, sKeyValueSystem.openStore(path).size());

    return path;
  }
  /**
   * Test <code>void seek(long pos)</code>.
   *
   * @throws IOException
   * @throws TachyonException
   */
  @Test
  public void seekTest() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);

      FileInStream is = mTfs.getInStream(f, mReadNoCache);

      Assert.assertEquals(0, is.read());
      is.seek(k / 3);
      Assert.assertEquals(k / 3, is.read());
      is.seek(k / 2);
      Assert.assertEquals(k / 2, is.read());
      is.seek(k / 4);
      Assert.assertEquals(k / 4, is.read());
      is.close();
    }
  }
  /** Tests that reading a file consisting of more than one block from the underfs works */
  @Test
  public void readMultiBlockFile() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    int blockSizeByte = 10;
    int numBlocks = 10;
    FileOutStream os = mTfs.getOutStream(new TachyonURI(uniqPath), mWriteUnderStore);
    for (int i = 0; i < numBlocks; i++) {
      for (int j = 0; j < blockSizeByte; j++) {
        os.write((byte) (i * blockSizeByte + j));
      }
    }
    os.close();

    TachyonFile f = mTfs.open(new TachyonURI(uniqPath));
    FileInStream is = mTfs.getInStream(f, mReadCache);
    for (int i = 0; i < blockSizeByte * numBlocks; i++) {
      Assert.assertEquals((byte) i, is.read());
    }
    is.close();
    Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
  }
  /** Test {@link tachyon.client.block.BufferedBlockInStream#read(byte[])}. */
  @Test
  public void readTest2() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      for (CreateFileOptions op : getOptionSet()) {
        TachyonURI path = new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode());
        FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);

        FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
        byte[] ret = new byte[k];
        Assert.assertEquals(k, is.read(ret));
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
        is.close();

        is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
        ret = new byte[k];
        Assert.assertEquals(k, is.read(ret));
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
        is.close();
      }
    }
  }
  /** Test <code>void read(byte[] b, int off, int len)</code>. Read from remote data server. */
  @Test
  public void readTest6() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      TachyonFile f =
          TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);

      long blockId = mTfs.getInfo(f).getBlockIds().get(0);
      BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
      RemoteBlockInStream is =
          new RemoteBlockInStream(
              info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
      byte[] ret = new byte[k / 2];
      int start = 0;
      while (start < k / 2) {
        int read = is.read(ret, 0, (k / 2) - start);
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(start, read, ret));
        start += read;
      }
      is.close();
      Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
    }
  }
Пример #24
0
 @Override
 public boolean delete(String path, boolean recursive) throws IOException {
   if (!recursive) {
     if (isFolder(path) && listInternal(path, false).length != 0) {
       LOG.error(
           "Unable to delete "
               + path
               + " because it is a non empty directory. Specify "
               + "recursive as true in order to delete non empty directories.");
       return false;
     }
     return deleteInternal(path);
   }
   // Get all relevant files
   String[] pathsToDelete = listInternal(path, true);
   for (String pathToDelete : pathsToDelete) {
     // If we fail to deleteInternal one file, stop
     if (!deleteInternal(PathUtils.concatPath(path, pathToDelete))) {
       LOG.error("Failed to delete path " + pathToDelete + ", aborting delete.");
       return false;
     }
   }
   return deleteInternal(path);
 }
  /** Test {@link tachyon.client.block.BufferedBlockInStream#skip(long)}. */
  @Test
  public void skipTest() throws IOException, TachyonException {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      for (CreateFileOptions op : getOptionSet()) {
        TachyonURI path = new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode());
        FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);

        FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));

        Assert.assertEquals(k / 2, is.skip(k / 2));
        Assert.assertEquals(k / 2, is.read());
        is.close();

        is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
        int t = k / 3;
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals(t, is.read());
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals(2 * t + 1, is.read());
        is.close();
      }
    }
  }
Пример #26
0
  private LocalTachyonMaster(final String tachyonHome) throws IOException {
    mTachyonHome = tachyonHome;

    TachyonConf tachyonConf = MasterContext.getConf();
    mHostname = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, tachyonConf);

    // To start the UFS either for integration or unit test. If it targets the unit test, UFS is
    // setup over the local file system (see also {@link LocalFilesystemCluster} - under folder of
    // "mTachyonHome/tachyon*". Otherwise, it starts some distributed file system cluster e.g.,
    // miniDFSCluster (see also {@link tachyon.LocalMiniDFScluster} and setup the folder like
    // "hdfs://xxx:xxx/tachyon*".
    mUnderFSCluster = UnderFileSystemCluster.get(mTachyonHome + "/dfs", tachyonConf);
    mUnderFSFolder = mUnderFSCluster.getUnderFilesystemAddress() + "/tachyon_underfs_folder";
    // To setup the journalFolder under either local file system or distributed ufs like
    // miniDFSCluster
    mJournalFolder = mUnderFSCluster.getUnderFilesystemAddress() + "/journal";

    UnderFileSystemUtils.mkdirIfNotExists(mJournalFolder, tachyonConf);
    String[] masterServiceNames =
        new String[] {
          Constants.BLOCK_MASTER_SERVICE_NAME,
          Constants.FILE_SYSTEM_MASTER_SERVICE_NAME,
          Constants.RAW_TABLE_MASTER_SERVICE_NAME,
        };
    for (String masterServiceName : masterServiceNames) {
      UnderFileSystemUtils.mkdirIfNotExists(
          PathUtils.concatPath(mJournalFolder, masterServiceName), tachyonConf);
    }
    UnderFileSystemUtils.touch(
        mJournalFolder + "/_format_" + System.currentTimeMillis(), tachyonConf);

    tachyonConf.set(Constants.MASTER_JOURNAL_FOLDER, mJournalFolder);
    tachyonConf.set(Constants.UNDERFS_ADDRESS, mUnderFSFolder);

    tachyonConf.set(Constants.MASTER_MIN_WORKER_THREADS, "1");
    tachyonConf.set(Constants.MASTER_MAX_WORKER_THREADS, "100");

    // If tests fail to connect they should fail early rather than using the default ridiculously
    // high retries
    tachyonConf.set(Constants.MASTER_RETRY_COUNT, "3");

    // Since tests are always running on a single host keep the resolution timeout low as otherwise
    // people running with strange network configurations will see very slow tests
    tachyonConf.set(Constants.HOST_RESOLUTION_TIMEOUT_MS, "250");

    tachyonConf.set(Constants.WEB_THREAD_COUNT, "1");
    tachyonConf.set(
        Constants.WEB_RESOURCES,
        PathUtils.concatPath(System.getProperty("user.dir"), "../servers/src/main/webapp"));

    mTachyonMaster = TachyonMaster.Factory.createMaster();

    // Reset the master port
    tachyonConf.set(Constants.MASTER_PORT, Integer.toString(getRPCLocalPort()));

    Runnable runMaster =
        new Runnable() {
          @Override
          public void run() {
            try {
              mTachyonMaster.start();
            } catch (Exception e) {
              throw new RuntimeException(e + " \n Start Master Error \n" + e.getMessage(), e);
            }
          }
        };

    mMasterThread = new Thread(runMaster);
  }
 @Before
 public void before() throws Exception {
   mStoreUri = new TachyonURI(PathUtils.uniqPath());
 }
Пример #28
0
 /**
  * @param baseDirectory the base journal directory
  * @return the journal directory for this master
  */
 public static String getJournalDirectory(String baseDirectory) {
   return PathUtils.concatPath(baseDirectory, Constants.BLOCK_MASTER_NAME);
 }
Пример #29
0
  @Override
  public void close() throws IOException {
    if (mClosed) {
      return;
    }
    if (mCurrentBlockOutStream != null) {
      mPreviousBlockOutStreams.add(mCurrentBlockOutStream);
    }

    Boolean canComplete = false;
    CompleteFileOptions options = CompleteFileOptions.defaults();
    if (mUnderStorageType.isSyncPersist()) {
      String tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
      UnderFileSystem ufs = UnderFileSystem.get(tmpPath, ClientContext.getConf());
      if (mCanceled) {
        // TODO(yupeng): Handle this special case in under storage integrations.
        mUnderStorageOutputStream.close();
        if (!ufs.exists(tmpPath)) {
          // Location of the temporary file has changed, recompute it.
          updateUfsPath();
          tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
        }
        ufs.delete(tmpPath, false);
      } else {
        mUnderStorageOutputStream.flush();
        mUnderStorageOutputStream.close();
        if (!ufs.exists(tmpPath)) {
          // Location of the temporary file has changed, recompute it.
          updateUfsPath();
          tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
        }
        if (!ufs.rename(tmpPath, mUfsPath)) {
          throw new IOException("Failed to rename " + tmpPath + " to " + mUfsPath);
        }
        options.setUfsLength(ufs.getFileSize(mUfsPath));
        canComplete = true;
      }
    }

    if (mTachyonStorageType.isStore()) {
      try {
        if (mCanceled) {
          for (BufferedBlockOutStream bos : mPreviousBlockOutStreams) {
            bos.cancel();
          }
        } else {
          for (BufferedBlockOutStream bos : mPreviousBlockOutStreams) {
            bos.close();
          }
          canComplete = true;
        }
      } catch (IOException ioe) {
        handleCacheWriteException(ioe);
      }
    }

    if (canComplete) {
      FileSystemMasterClient masterClient = mContext.acquireMasterClient();
      try {
        masterClient.completeFile(mUri, options);
      } catch (TachyonException e) {
        throw new IOException(e);
      } finally {
        mContext.releaseMasterClient(masterClient);
      }
    }

    if (mUnderStorageType.isAsyncPersist()) {
      scheduleAsyncPersist();
    }
    mClosed = true;
  }