// Tests that lock block returns the correct path
  @Test
  public void lockBlockTest() throws Exception {
    final int blockSize = (int) WORKER_CAPACITY_BYTES / 2;

    CreateFileOptions options =
        CreateFileOptions.defaults()
            .setBlockSizeBytes(blockSize)
            .setWriteType(WriteType.MUST_CACHE);
    FileOutStream out = mFileSystem.createFile(new AlluxioURI("/testFile"), options);
    URIStatus file = mFileSystem.getStatus(new AlluxioURI("/testFile"));

    final long blockId = BlockId.createBlockId(BlockId.getContainerId(file.getFileId()), 0);

    out.write(BufferUtils.getIncreasingByteArray(blockSize));
    out.close();

    String localPath = mBlockWorkerServiceHandler.lockBlock(blockId, SESSION_ID).getBlockPath();

    // The local path should exist
    Assert.assertNotNull(localPath);

    UnderFileSystem ufs = UnderFileSystem.get(localPath, mMasterConfiguration);
    byte[] data = new byte[blockSize];
    int bytesRead = ufs.open(localPath).read(data);

    // The data in the local file should equal the data we wrote earlier
    Assert.assertEquals(blockSize, bytesRead);
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(bytesRead, data));

    mBlockWorkerServiceHandler.unlockBlock(blockId, SESSION_ID);
  }
 private void writeTest3Util(AlluxioURI filePath, int len, CreateFileOptions op) throws Exception {
   FileOutStream os = mFileSystem.createFile(filePath, op);
   os.write(BufferUtils.getIncreasingByteArray(0, len / 2), 0, len / 2);
   os.write(BufferUtils.getIncreasingByteArray(len / 2, len / 2), 0, len / 2);
   os.close();
   checkWrite(filePath, op.getUnderStorageType(), len, len / 2 * 2);
 }
 private void writeTest1Util(AlluxioURI filePath, int len, CreateFileOptions op) throws Exception {
   FileOutStream os = mFileSystem.createFile(filePath, op);
   for (int k = 0; k < len; k++) {
     os.write((byte) k);
   }
   os.close();
   checkWrite(filePath, op.getUnderStorageType(), len, len);
 }
 /**
  * Tests writing to a file for longer than HEARTBEAT_INTERVAL_MS to make sure the sessionId
  * doesn't change. Tracks [ALLUXIO-171].
  */
 @Test
 public void longWrite() throws Exception {
   AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
   final int length = 2;
   FileOutStream os =
       mFileSystem.createFile(
           filePath, CreateFileOptions.defaults().setWriteType(WriteType.THROUGH));
   os.write((byte) 0);
   Thread.sleep(Configuration.getInt(PropertyKey.USER_HEARTBEAT_INTERVAL_MS) * 2);
   os.write((byte) 1);
   os.close();
   checkWrite(filePath, UnderStorageType.SYNC_PERSIST, length, length);
 }
 /** Tests writing to a file and specify the location to be localhost. */
 @Test
 public void writeSpecifyLocal() throws Exception {
   AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
   final int length = 2;
   FileOutStream os =
       mFileSystem.createFile(
           filePath,
           CreateFileOptions.defaults()
               .setWriteType(WriteType.CACHE_THROUGH)
               .setLocationPolicy(new LocalFirstPolicy()));
   os.write((byte) 0);
   os.write((byte) 1);
   os.close();
   checkWrite(filePath, UnderStorageType.SYNC_PERSIST, length, length);
 }
  /**
   * Tests if out-of-order writes are possible. Writes could be out-of-order when the following are
   * both true: - a "large" write (over half the internal buffer size) follows a smaller write. -
   * the "large" write does not cause the internal buffer to overflow.
   */
  @Test
  public void outOfOrderWrite() throws Exception {
    AlluxioURI filePath = new AlluxioURI(PathUtils.uniqPath());
    FileOutStream os =
        mFileSystem.createFile(
            filePath, CreateFileOptions.defaults().setWriteType(WriteType.MUST_CACHE));

    // Write something small, so it is written into the buffer, and not directly to the file.
    os.write((byte) 0);

    // A length greater than 0.5 * BUFFER_BYTES and less than BUFFER_BYTES.
    int length = (BUFFER_BYTES * 3) / 4;

    // Write a large amount of data (larger than BUFFER_BYTES/2, but will not overflow the buffer.
    os.write(BufferUtils.getIncreasingByteArray(1, length));
    os.close();

    checkWrite(filePath, UnderStorageType.NO_PERSIST, length + 1, length + 1);
  }
예제 #7
0
 @Override
 public void cancel() throws IOException {
   mCanceled = true;
   close();
 }
예제 #8
0
  /**
   * Entry point for the {@link MultiMount} program.
   *
   * @param args command-line arguments
   */
  public static void main(String[] args) {
    if (args.length != 1) {
      System.err.println("Usage: ./bin/alluxio runClass alluxio.examples.MultiMount <HDFS_URL>");
      System.exit(-1);
    }

    AlluxioURI mntPath = new AlluxioURI("/mnt");
    AlluxioURI s3Mount = new AlluxioURI("/mnt/s3");
    AlluxioURI inputPath = new AlluxioURI("/mnt/s3/hello.txt");
    AlluxioURI s3Path = new AlluxioURI("s3n://alluxio-demo/");
    AlluxioURI hdfsMount = new AlluxioURI("/mnt/hdfs");
    AlluxioURI outputPath = new AlluxioURI("/mnt/hdfs/hello.txt");
    AlluxioURI hdfsPath = new AlluxioURI(args[0]);
    FileSystem fileSystem = FileSystem.Factory.get();

    try {
      // Make sure mount directory exists.
      if (!fileSystem.exists(mntPath)) {
        System.out.print("creating " + mntPath + " ... ");
        fileSystem.createDirectory(mntPath);
        System.out.println("done");
      }

      // Make sure the S3 mount point does not exist.
      if (fileSystem.exists(s3Mount)) {
        System.out.print("unmounting " + s3Mount + " ... ");
        fileSystem.unmount(s3Mount);
        System.out.println("done");
      }

      // Make sure the HDFS mount point does not exist.
      if (fileSystem.exists(hdfsMount)) {
        System.out.print("unmounting " + hdfsMount + " ... ");
        fileSystem.unmount(hdfsMount);
        System.out.println("done");
      }

      // Mount S3.
      System.out.print("mounting " + s3Path + " to " + s3Mount + " ... ");
      fileSystem.mount(s3Mount, s3Path);
      System.out.println("done");

      // Mount HDFS.
      System.out.print("mounting " + hdfsPath + " to " + hdfsMount + " ... ");
      fileSystem.mount(hdfsMount, hdfsPath);
      System.out.println("done");

      // Make sure output file does not exist.
      if (fileSystem.exists(outputPath)) {
        System.out.print("deleting " + outputPath + " ... ");
        fileSystem.delete(outputPath);
        System.out.println("done");
      }

      // Open the input stream.
      System.out.print("opening " + inputPath + " ... ");
      FileInStream is = fileSystem.openFile(inputPath);
      System.out.println("done");

      // Open the output stream, setting the write type to make sure result is persisted.
      System.out.print("opening " + outputPath + " ... ");
      CreateFileOptions options =
          CreateFileOptions.defaults().setWriteType(WriteType.CACHE_THROUGH);
      FileOutStream os = fileSystem.createFile(outputPath, options);
      System.out.println("done");

      // Copy the data
      System.out.print("transferring data from " + inputPath + " to " + outputPath + " ... ");
      IOUtils.copy(is, os);
      System.out.println("done");

      // Close the input stream.
      System.out.print("closing " + inputPath + " ... ");
      is.close();
      System.out.println("done");

      // Close the output stream.
      System.out.print("closing " + outputPath + " ... ");
      os.close();
      System.out.println("done");
    } catch (Exception e) {
      System.out.println("fail");
      e.printStackTrace();
    } finally {
      // Make sure the S3 mount point is removed.
      try {
        if (fileSystem.exists(s3Mount)) {
          System.out.print("unmounting " + s3Mount + " ... ");
          fileSystem.unmount(s3Mount);
          System.out.println("done");
        }
      } catch (Exception e) {
        System.out.println("fail");
        e.printStackTrace();
      }

      // Make sure the HDFS mount point is removed.
      try {
        if (fileSystem.exists(hdfsMount)) {
          System.out.print("unmounting " + hdfsMount + " ... ");
          fileSystem.unmount(hdfsMount);
          System.out.println("done");
        }
      } catch (Exception e) {
        System.out.println("fail");
        e.printStackTrace();
      }
    }
  }