@Override
public int read(byte[] buffer, int offset, int length) throws IOException {
if (mClosed) {
throw new IOException("Cannot read from a closed stream.");
}
if (mAlluxioFileInputStream != null) {
try {
int ret = mAlluxioFileInputStream.read(buffer, offset, length);
if (mStatistics != null && ret != -1) {
mStatistics.incrementBytesRead(ret);
}
mCurrentPosition += ret;
return ret;
} catch (IOException e) {
LOG.error(e.getMessage(), e);
mAlluxioFileInputStream.close();
mAlluxioFileInputStream = null;
}
}
getHdfsInputStream();
int byteRead = readFromHdfsBuffer();
// byteRead is an unsigned byte, if its -1 then we have hit EOF
if (byteRead == -1) {
return -1;
}
// Convert byteRead back to a signed byte
buffer[offset] = (byte) byteRead;
return 1;
}
@Override
public int available() throws IOException {
if (mClosed) {
throw new IOException("Cannot query available bytes from a closed stream.");
}
return (int) mAlluxioFileInputStream.remaining();
}
@Override
public void close() throws IOException {
if (mAlluxioFileInputStream != null) {
mAlluxioFileInputStream.close();
}
if (mHdfsInputStream != null) {
mHdfsInputStream.close();
}
mClosed = true;
}
@Override
public int read() throws IOException {
if (mClosed) {
throw new IOException("Cannot read from a closed stream.");
}
if (mAlluxioFileInputStream != null) {
try {
int ret = mAlluxioFileInputStream.read();
if (mStatistics != null && ret != -1) {
mStatistics.incrementBytesRead(1);
}
mCurrentPosition++;
return ret;
} catch (IOException e) {
LOG.error(e.getMessage(), e);
mAlluxioFileInputStream.close();
mAlluxioFileInputStream = null;
}
}
getHdfsInputStream();
return readFromHdfsBuffer();
}
@Override
public synchronized int read(long position, byte[] buffer, int offset, int length)
throws IOException {
if (mClosed) {
throw new IOException(ExceptionMessage.READ_CLOSED_STREAM.getMessage());
}
int ret = -1;
long oldPos = getPos();
if ((position < 0) || (position >= mFileInfo.getLength())) {
return ret;
}
if (mAlluxioFileInputStream != null) {
try {
mAlluxioFileInputStream.seek(position);
ret = mAlluxioFileInputStream.read(buffer, offset, length);
if (mStatistics != null && ret != -1) {
mStatistics.incrementBytesRead(ret);
}
return ret;
} finally {
mAlluxioFileInputStream.seek(oldPos);
}
}
try {
getHdfsInputStream(position);
ret = mHdfsInputStream.read(buffer, offset, length);
if (mStatistics != null && ret != -1) {
mStatistics.incrementBytesRead(ret);
}
return ret;
} finally {
if (mHdfsInputStream != null) {
mHdfsInputStream.seek(oldPos);
}
}
}
/**
* Seek to the given offset from the start of the file. The next {@link #read()} will be from that
* location. Can't seek past the end of the file.
*
* @param pos the position to seek to
* @throws IOException if the position is negative or exceeds the end of the file
*/
@Override
public void seek(long pos) throws IOException {
if (pos == mCurrentPosition) {
return;
}
if (pos < 0) {
throw new IOException(ExceptionMessage.SEEK_NEGATIVE.getMessage(pos));
}
if (pos > mFileInfo.getLength()) {
throw new IOException(ExceptionMessage.SEEK_PAST_EOF.getMessage(pos, mFileInfo.getLength()));
}
if (mAlluxioFileInputStream != null) {
mAlluxioFileInputStream.seek(pos);
} else {
getHdfsInputStream(pos);
// TODO(calvin): Optimize for the case when the data is still valid in the buffer
// Invalidate buffer
mBufferLimit = -1;
}
mCurrentPosition = pos;
}
/**
* Entry point for the {@link MultiMount} program.
*
* @param args command-line arguments
*/
public static void main(String[] args) {
if (args.length != 1) {
System.err.println("Usage: ./bin/alluxio runClass alluxio.examples.MultiMount <HDFS_URL>");
System.exit(-1);
}
AlluxioURI mntPath = new AlluxioURI("/mnt");
AlluxioURI s3Mount = new AlluxioURI("/mnt/s3");
AlluxioURI inputPath = new AlluxioURI("/mnt/s3/hello.txt");
AlluxioURI s3Path = new AlluxioURI("s3n://alluxio-demo/");
AlluxioURI hdfsMount = new AlluxioURI("/mnt/hdfs");
AlluxioURI outputPath = new AlluxioURI("/mnt/hdfs/hello.txt");
AlluxioURI hdfsPath = new AlluxioURI(args[0]);
FileSystem fileSystem = FileSystem.Factory.get();
try {
// Make sure mount directory exists.
if (!fileSystem.exists(mntPath)) {
System.out.print("creating " + mntPath + " ... ");
fileSystem.createDirectory(mntPath);
System.out.println("done");
}
// Make sure the S3 mount point does not exist.
if (fileSystem.exists(s3Mount)) {
System.out.print("unmounting " + s3Mount + " ... ");
fileSystem.unmount(s3Mount);
System.out.println("done");
}
// Make sure the HDFS mount point does not exist.
if (fileSystem.exists(hdfsMount)) {
System.out.print("unmounting " + hdfsMount + " ... ");
fileSystem.unmount(hdfsMount);
System.out.println("done");
}
// Mount S3.
System.out.print("mounting " + s3Path + " to " + s3Mount + " ... ");
fileSystem.mount(s3Mount, s3Path);
System.out.println("done");
// Mount HDFS.
System.out.print("mounting " + hdfsPath + " to " + hdfsMount + " ... ");
fileSystem.mount(hdfsMount, hdfsPath);
System.out.println("done");
// Make sure output file does not exist.
if (fileSystem.exists(outputPath)) {
System.out.print("deleting " + outputPath + " ... ");
fileSystem.delete(outputPath);
System.out.println("done");
}
// Open the input stream.
System.out.print("opening " + inputPath + " ... ");
FileInStream is = fileSystem.openFile(inputPath);
System.out.println("done");
// Open the output stream, setting the write type to make sure result is persisted.
System.out.print("opening " + outputPath + " ... ");
CreateFileOptions options =
CreateFileOptions.defaults().setWriteType(WriteType.CACHE_THROUGH);
FileOutStream os = fileSystem.createFile(outputPath, options);
System.out.println("done");
// Copy the data
System.out.print("transferring data from " + inputPath + " to " + outputPath + " ... ");
IOUtils.copy(is, os);
System.out.println("done");
// Close the input stream.
System.out.print("closing " + inputPath + " ... ");
is.close();
System.out.println("done");
// Close the output stream.
System.out.print("closing " + outputPath + " ... ");
os.close();
System.out.println("done");
} catch (Exception e) {
System.out.println("fail");
e.printStackTrace();
} finally {
// Make sure the S3 mount point is removed.
try {
if (fileSystem.exists(s3Mount)) {
System.out.print("unmounting " + s3Mount + " ... ");
fileSystem.unmount(s3Mount);
System.out.println("done");
}
} catch (Exception e) {
System.out.println("fail");
e.printStackTrace();
}
// Make sure the HDFS mount point is removed.
try {
if (fileSystem.exists(hdfsMount)) {
System.out.print("unmounting " + hdfsMount + " ... ");
fileSystem.unmount(hdfsMount);
System.out.println("done");
}
} catch (Exception e) {
System.out.println("fail");
e.printStackTrace();
}
}
}