예제 #1
0
  @Override
  public int read(byte[] buffer, int offset, int length) throws IOException {
    if (mClosed) {
      throw new IOException("Cannot read from a closed stream.");
    }
    if (mAlluxioFileInputStream != null) {
      try {
        int ret = mAlluxioFileInputStream.read(buffer, offset, length);
        if (mStatistics != null && ret != -1) {
          mStatistics.incrementBytesRead(ret);
        }
        mCurrentPosition += ret;
        return ret;
      } catch (IOException e) {
        LOG.error(e.getMessage(), e);
        mAlluxioFileInputStream.close();
        mAlluxioFileInputStream = null;
      }
    }

    getHdfsInputStream();
    int byteRead = readFromHdfsBuffer();
    // byteRead is an unsigned byte, if its -1 then we have hit EOF
    if (byteRead == -1) {
      return -1;
    }
    // Convert byteRead back to a signed byte
    buffer[offset] = (byte) byteRead;
    return 1;
  }
예제 #2
0
 @Override
 public int available() throws IOException {
   if (mClosed) {
     throw new IOException("Cannot query available bytes from a closed stream.");
   }
   return (int) mAlluxioFileInputStream.remaining();
 }
예제 #3
0
 @Override
 public void close() throws IOException {
   if (mAlluxioFileInputStream != null) {
     mAlluxioFileInputStream.close();
   }
   if (mHdfsInputStream != null) {
     mHdfsInputStream.close();
   }
   mClosed = true;
 }
예제 #4
0
 @Override
 public int read() throws IOException {
   if (mClosed) {
     throw new IOException("Cannot read from a closed stream.");
   }
   if (mAlluxioFileInputStream != null) {
     try {
       int ret = mAlluxioFileInputStream.read();
       if (mStatistics != null && ret != -1) {
         mStatistics.incrementBytesRead(1);
       }
       mCurrentPosition++;
       return ret;
     } catch (IOException e) {
       LOG.error(e.getMessage(), e);
       mAlluxioFileInputStream.close();
       mAlluxioFileInputStream = null;
     }
   }
   getHdfsInputStream();
   return readFromHdfsBuffer();
 }
예제 #5
0
  @Override
  public synchronized int read(long position, byte[] buffer, int offset, int length)
      throws IOException {
    if (mClosed) {
      throw new IOException(ExceptionMessage.READ_CLOSED_STREAM.getMessage());
    }
    int ret = -1;
    long oldPos = getPos();
    if ((position < 0) || (position >= mFileInfo.getLength())) {
      return ret;
    }

    if (mAlluxioFileInputStream != null) {
      try {
        mAlluxioFileInputStream.seek(position);
        ret = mAlluxioFileInputStream.read(buffer, offset, length);
        if (mStatistics != null && ret != -1) {
          mStatistics.incrementBytesRead(ret);
        }
        return ret;
      } finally {
        mAlluxioFileInputStream.seek(oldPos);
      }
    }

    try {
      getHdfsInputStream(position);
      ret = mHdfsInputStream.read(buffer, offset, length);
      if (mStatistics != null && ret != -1) {
        mStatistics.incrementBytesRead(ret);
      }
      return ret;
    } finally {
      if (mHdfsInputStream != null) {
        mHdfsInputStream.seek(oldPos);
      }
    }
  }
예제 #6
0
  /**
   * Seek to the given offset from the start of the file. The next {@link #read()} will be from that
   * location. Can't seek past the end of the file.
   *
   * @param pos the position to seek to
   * @throws IOException if the position is negative or exceeds the end of the file
   */
  @Override
  public void seek(long pos) throws IOException {
    if (pos == mCurrentPosition) {
      return;
    }

    if (pos < 0) {
      throw new IOException(ExceptionMessage.SEEK_NEGATIVE.getMessage(pos));
    }
    if (pos > mFileInfo.getLength()) {
      throw new IOException(ExceptionMessage.SEEK_PAST_EOF.getMessage(pos, mFileInfo.getLength()));
    }

    if (mAlluxioFileInputStream != null) {
      mAlluxioFileInputStream.seek(pos);
    } else {
      getHdfsInputStream(pos);
      // TODO(calvin): Optimize for the case when the data is still valid in the buffer
      // Invalidate buffer
      mBufferLimit = -1;
    }

    mCurrentPosition = pos;
  }
예제 #7
0
  /**
   * Entry point for the {@link MultiMount} program.
   *
   * @param args command-line arguments
   */
  public static void main(String[] args) {
    if (args.length != 1) {
      System.err.println("Usage: ./bin/alluxio runClass alluxio.examples.MultiMount <HDFS_URL>");
      System.exit(-1);
    }

    AlluxioURI mntPath = new AlluxioURI("/mnt");
    AlluxioURI s3Mount = new AlluxioURI("/mnt/s3");
    AlluxioURI inputPath = new AlluxioURI("/mnt/s3/hello.txt");
    AlluxioURI s3Path = new AlluxioURI("s3n://alluxio-demo/");
    AlluxioURI hdfsMount = new AlluxioURI("/mnt/hdfs");
    AlluxioURI outputPath = new AlluxioURI("/mnt/hdfs/hello.txt");
    AlluxioURI hdfsPath = new AlluxioURI(args[0]);
    FileSystem fileSystem = FileSystem.Factory.get();

    try {
      // Make sure mount directory exists.
      if (!fileSystem.exists(mntPath)) {
        System.out.print("creating " + mntPath + " ... ");
        fileSystem.createDirectory(mntPath);
        System.out.println("done");
      }

      // Make sure the S3 mount point does not exist.
      if (fileSystem.exists(s3Mount)) {
        System.out.print("unmounting " + s3Mount + " ... ");
        fileSystem.unmount(s3Mount);
        System.out.println("done");
      }

      // Make sure the HDFS mount point does not exist.
      if (fileSystem.exists(hdfsMount)) {
        System.out.print("unmounting " + hdfsMount + " ... ");
        fileSystem.unmount(hdfsMount);
        System.out.println("done");
      }

      // Mount S3.
      System.out.print("mounting " + s3Path + " to " + s3Mount + " ... ");
      fileSystem.mount(s3Mount, s3Path);
      System.out.println("done");

      // Mount HDFS.
      System.out.print("mounting " + hdfsPath + " to " + hdfsMount + " ... ");
      fileSystem.mount(hdfsMount, hdfsPath);
      System.out.println("done");

      // Make sure output file does not exist.
      if (fileSystem.exists(outputPath)) {
        System.out.print("deleting " + outputPath + " ... ");
        fileSystem.delete(outputPath);
        System.out.println("done");
      }

      // Open the input stream.
      System.out.print("opening " + inputPath + " ... ");
      FileInStream is = fileSystem.openFile(inputPath);
      System.out.println("done");

      // Open the output stream, setting the write type to make sure result is persisted.
      System.out.print("opening " + outputPath + " ... ");
      CreateFileOptions options =
          CreateFileOptions.defaults().setWriteType(WriteType.CACHE_THROUGH);
      FileOutStream os = fileSystem.createFile(outputPath, options);
      System.out.println("done");

      // Copy the data
      System.out.print("transferring data from " + inputPath + " to " + outputPath + " ... ");
      IOUtils.copy(is, os);
      System.out.println("done");

      // Close the input stream.
      System.out.print("closing " + inputPath + " ... ");
      is.close();
      System.out.println("done");

      // Close the output stream.
      System.out.print("closing " + outputPath + " ... ");
      os.close();
      System.out.println("done");
    } catch (Exception e) {
      System.out.println("fail");
      e.printStackTrace();
    } finally {
      // Make sure the S3 mount point is removed.
      try {
        if (fileSystem.exists(s3Mount)) {
          System.out.print("unmounting " + s3Mount + " ... ");
          fileSystem.unmount(s3Mount);
          System.out.println("done");
        }
      } catch (Exception e) {
        System.out.println("fail");
        e.printStackTrace();
      }

      // Make sure the HDFS mount point is removed.
      try {
        if (fileSystem.exists(hdfsMount)) {
          System.out.print("unmounting " + hdfsMount + " ... ");
          fileSystem.unmount(hdfsMount);
          System.out.println("done");
        }
      } catch (Exception e) {
        System.out.println("fail");
        e.printStackTrace();
      }
    }
  }