/**
* Constructs a new stream for reading a file from HDFS.
*
* @param uri the Alluxio file URI
* @param conf Hadoop configuration
* @param bufferSize the buffer size
* @param stats filesystem statistics
* @throws IOException if the underlying file does not exist or its stream cannot be created
*/
public HdfsFileInputStream(
AlluxioURI uri,
org.apache.hadoop.conf.Configuration conf,
int bufferSize,
org.apache.hadoop.fs.FileSystem.Statistics stats)
throws IOException {
LOG.debug("HdfsFileInputStream({}, {}, {}, {}, {})", uri, conf, bufferSize, stats);
long bufferBytes = Configuration.getBytes(Constants.USER_FILE_BUFFER_BYTES);
mBuffer = new byte[Ints.checkedCast(bufferBytes) * 4];
mCurrentPosition = 0;
FileSystem fs = FileSystem.Factory.get();
mHadoopConf = conf;
mHadoopBufferSize = bufferSize;
mStatistics = stats;
try {
mFileInfo = fs.getStatus(uri);
mHdfsPath = new Path(mFileInfo.getUfsPath());
mAlluxioFileInputStream = fs.openFile(uri, OpenFileOptions.defaults());
} catch (FileDoesNotExistException e) {
throw new FileNotFoundException(
ExceptionMessage.HDFS_FILE_NOT_FOUND.getMessage(mHdfsPath, uri));
} catch (AlluxioException e) {
throw new IOException(e);
}
}
/**
* Entry point for the {@link MultiMount} program.
*
* @param args command-line arguments
*/
public static void main(String[] args) {
if (args.length != 1) {
System.err.println("Usage: ./bin/alluxio runClass alluxio.examples.MultiMount <HDFS_URL>");
System.exit(-1);
}
AlluxioURI mntPath = new AlluxioURI("/mnt");
AlluxioURI s3Mount = new AlluxioURI("/mnt/s3");
AlluxioURI inputPath = new AlluxioURI("/mnt/s3/hello.txt");
AlluxioURI s3Path = new AlluxioURI("s3n://alluxio-demo/");
AlluxioURI hdfsMount = new AlluxioURI("/mnt/hdfs");
AlluxioURI outputPath = new AlluxioURI("/mnt/hdfs/hello.txt");
AlluxioURI hdfsPath = new AlluxioURI(args[0]);
FileSystem fileSystem = FileSystem.Factory.get();
try {
// Make sure mount directory exists.
if (!fileSystem.exists(mntPath)) {
System.out.print("creating " + mntPath + " ... ");
fileSystem.createDirectory(mntPath);
System.out.println("done");
}
// Make sure the S3 mount point does not exist.
if (fileSystem.exists(s3Mount)) {
System.out.print("unmounting " + s3Mount + " ... ");
fileSystem.unmount(s3Mount);
System.out.println("done");
}
// Make sure the HDFS mount point does not exist.
if (fileSystem.exists(hdfsMount)) {
System.out.print("unmounting " + hdfsMount + " ... ");
fileSystem.unmount(hdfsMount);
System.out.println("done");
}
// Mount S3.
System.out.print("mounting " + s3Path + " to " + s3Mount + " ... ");
fileSystem.mount(s3Mount, s3Path);
System.out.println("done");
// Mount HDFS.
System.out.print("mounting " + hdfsPath + " to " + hdfsMount + " ... ");
fileSystem.mount(hdfsMount, hdfsPath);
System.out.println("done");
// Make sure output file does not exist.
if (fileSystem.exists(outputPath)) {
System.out.print("deleting " + outputPath + " ... ");
fileSystem.delete(outputPath);
System.out.println("done");
}
// Open the input stream.
System.out.print("opening " + inputPath + " ... ");
FileInStream is = fileSystem.openFile(inputPath);
System.out.println("done");
// Open the output stream, setting the write type to make sure result is persisted.
System.out.print("opening " + outputPath + " ... ");
CreateFileOptions options =
CreateFileOptions.defaults().setWriteType(WriteType.CACHE_THROUGH);
FileOutStream os = fileSystem.createFile(outputPath, options);
System.out.println("done");
// Copy the data
System.out.print("transferring data from " + inputPath + " to " + outputPath + " ... ");
IOUtils.copy(is, os);
System.out.println("done");
// Close the input stream.
System.out.print("closing " + inputPath + " ... ");
is.close();
System.out.println("done");
// Close the output stream.
System.out.print("closing " + outputPath + " ... ");
os.close();
System.out.println("done");
} catch (Exception e) {
System.out.println("fail");
e.printStackTrace();
} finally {
// Make sure the S3 mount point is removed.
try {
if (fileSystem.exists(s3Mount)) {
System.out.print("unmounting " + s3Mount + " ... ");
fileSystem.unmount(s3Mount);
System.out.println("done");
}
} catch (Exception e) {
System.out.println("fail");
e.printStackTrace();
}
// Make sure the HDFS mount point is removed.
try {
if (fileSystem.exists(hdfsMount)) {
System.out.print("unmounting " + hdfsMount + " ... ");
fileSystem.unmount(hdfsMount);
System.out.println("done");
}
} catch (Exception e) {
System.out.println("fail");
e.printStackTrace();
}
}
}