@Override public int read(byte[] buffer, int offset, int length) throws IOException { if (mClosed) { throw new IOException("Cannot read from a closed stream."); } if (mAlluxioFileInputStream != null) { try { int ret = mAlluxioFileInputStream.read(buffer, offset, length); if (mStatistics != null && ret != -1) { mStatistics.incrementBytesRead(ret); } mCurrentPosition += ret; return ret; } catch (IOException e) { LOG.error(e.getMessage(), e); mAlluxioFileInputStream.close(); mAlluxioFileInputStream = null; } } getHdfsInputStream(); int byteRead = readFromHdfsBuffer(); // byteRead is an unsigned byte, if its -1 then we have hit EOF if (byteRead == -1) { return -1; } // Convert byteRead back to a signed byte buffer[offset] = (byte) byteRead; return 1; }
@Override public void close() throws IOException { if (mAlluxioFileInputStream != null) { mAlluxioFileInputStream.close(); } if (mHdfsInputStream != null) { mHdfsInputStream.close(); } mClosed = true; }
@Override public int read() throws IOException { if (mClosed) { throw new IOException("Cannot read from a closed stream."); } if (mAlluxioFileInputStream != null) { try { int ret = mAlluxioFileInputStream.read(); if (mStatistics != null && ret != -1) { mStatistics.incrementBytesRead(1); } mCurrentPosition++; return ret; } catch (IOException e) { LOG.error(e.getMessage(), e); mAlluxioFileInputStream.close(); mAlluxioFileInputStream = null; } } getHdfsInputStream(); return readFromHdfsBuffer(); }
/** * Entry point for the {@link MultiMount} program. * * @param args command-line arguments */ public static void main(String[] args) { if (args.length != 1) { System.err.println("Usage: ./bin/alluxio runClass alluxio.examples.MultiMount <HDFS_URL>"); System.exit(-1); } AlluxioURI mntPath = new AlluxioURI("/mnt"); AlluxioURI s3Mount = new AlluxioURI("/mnt/s3"); AlluxioURI inputPath = new AlluxioURI("/mnt/s3/hello.txt"); AlluxioURI s3Path = new AlluxioURI("s3n://alluxio-demo/"); AlluxioURI hdfsMount = new AlluxioURI("/mnt/hdfs"); AlluxioURI outputPath = new AlluxioURI("/mnt/hdfs/hello.txt"); AlluxioURI hdfsPath = new AlluxioURI(args[0]); FileSystem fileSystem = FileSystem.Factory.get(); try { // Make sure mount directory exists. if (!fileSystem.exists(mntPath)) { System.out.print("creating " + mntPath + " ... "); fileSystem.createDirectory(mntPath); System.out.println("done"); } // Make sure the S3 mount point does not exist. if (fileSystem.exists(s3Mount)) { System.out.print("unmounting " + s3Mount + " ... "); fileSystem.unmount(s3Mount); System.out.println("done"); } // Make sure the HDFS mount point does not exist. if (fileSystem.exists(hdfsMount)) { System.out.print("unmounting " + hdfsMount + " ... "); fileSystem.unmount(hdfsMount); System.out.println("done"); } // Mount S3. System.out.print("mounting " + s3Path + " to " + s3Mount + " ... "); fileSystem.mount(s3Mount, s3Path); System.out.println("done"); // Mount HDFS. System.out.print("mounting " + hdfsPath + " to " + hdfsMount + " ... "); fileSystem.mount(hdfsMount, hdfsPath); System.out.println("done"); // Make sure output file does not exist. if (fileSystem.exists(outputPath)) { System.out.print("deleting " + outputPath + " ... "); fileSystem.delete(outputPath); System.out.println("done"); } // Open the input stream. System.out.print("opening " + inputPath + " ... "); FileInStream is = fileSystem.openFile(inputPath); System.out.println("done"); // Open the output stream, setting the write type to make sure result is persisted. System.out.print("opening " + outputPath + " ... "); CreateFileOptions options = CreateFileOptions.defaults().setWriteType(WriteType.CACHE_THROUGH); FileOutStream os = fileSystem.createFile(outputPath, options); System.out.println("done"); // Copy the data System.out.print("transferring data from " + inputPath + " to " + outputPath + " ... "); IOUtils.copy(is, os); System.out.println("done"); // Close the input stream. System.out.print("closing " + inputPath + " ... "); is.close(); System.out.println("done"); // Close the output stream. System.out.print("closing " + outputPath + " ... "); os.close(); System.out.println("done"); } catch (Exception e) { System.out.println("fail"); e.printStackTrace(); } finally { // Make sure the S3 mount point is removed. try { if (fileSystem.exists(s3Mount)) { System.out.print("unmounting " + s3Mount + " ... "); fileSystem.unmount(s3Mount); System.out.println("done"); } } catch (Exception e) { System.out.println("fail"); e.printStackTrace(); } // Make sure the HDFS mount point is removed. try { if (fileSystem.exists(hdfsMount)) { System.out.print("unmounting " + hdfsMount + " ... "); fileSystem.unmount(hdfsMount); System.out.println("done"); } } catch (Exception e) { System.out.println("fail"); e.printStackTrace(); } } }