public void testInputFormat() { try { JobConf conf = new JobConf(); String TMP_DIR = System.getProperty("test.build.data", "/tmp"); Path filename = new Path("file:///" + TMP_DIR + "/tmpSeqFile"); SequenceFile.Writer sfw = SequenceFile.createWriter( FileSystem.getLocal(conf), conf, filename, ChukwaArchiveKey.class, ChunkImpl.class, SequenceFile.CompressionType.NONE, Reporter.NULL); StringBuilder buf = new StringBuilder(); int offsets[] = new int[lines.length]; for (int i = 0; i < lines.length; ++i) { buf.append(lines[i]); buf.append("\n"); offsets[i] = buf.length() - 1; } ChukwaArchiveKey key = new ChukwaArchiveKey(0, "datatype", "sname", 0); ChunkImpl val = new ChunkImpl("datatype", "sname", 0, buf.toString().getBytes(), null); val.setRecordOffsets(offsets); sfw.append(key, val); sfw.append(key, val); // write it twice sfw.close(); long len = FileSystem.getLocal(conf).getFileStatus(filename).getLen(); InputSplit split = new FileSplit(filename, 0, len, (String[]) null); ChukwaInputFormat in = new ChukwaInputFormat(); RecordReader<LongWritable, Text> r = in.getRecordReader(split, conf, Reporter.NULL); LongWritable l = r.createKey(); Text line = r.createValue(); for (int i = 0; i < lines.length * 2; ++i) { boolean succeeded = r.next(l, line); assertTrue(succeeded); assertEquals(i, l.get()); assertEquals(lines[i % lines.length], line.toString()); System.out.println("read line: " + l.get() + " " + line); } boolean succeeded = r.next(l, line); assertFalse(succeeded); } catch (IOException e) { e.printStackTrace(); fail("IO exception " + e); } }
/** * Distributed i/o benchmark. * * <p>This test writes into or reads from a specified number of files. File size is specified as a * parameter to the test. Each file is accessed in a separate map task. * * <p>The reducer collects the following statistics: * * <ul> * <li>number of tasks completed * <li>number of bytes written/read * <li>execution time * <li>io rate * <li>io rate squared * </ul> * * Finally, the following information is appended to a local file * * <ul> * <li>read or write test * <li>date and time the test finished * <li>number of files * <li>total number of bytes processed * <li>throughput in mb/sec (total number of bytes / sum of processing times) * <li>average i/o rate in mb/sec per file * <li>standard i/o rate deviation * </ul> */ @Ignore public class DFSCIOTest extends TestCase { // Constants private static final Log LOG = LogFactory.getLog(DFSCIOTest.class); private static final int TEST_TYPE_READ = 0; private static final int TEST_TYPE_WRITE = 1; private static final int TEST_TYPE_CLEANUP = 2; private static final int DEFAULT_BUFFER_SIZE = 1000000; private static final String BASE_FILE_NAME = "test_io_"; private static final String DEFAULT_RES_FILE_NAME = "DFSCIOTest_results.log"; private static Configuration fsConfig = new Configuration(); private static final long MEGA = 0x100000; private static String TEST_ROOT_DIR = System.getProperty("test.build.data", "/benchmarks/DFSCIOTest"); private static Path CONTROL_DIR = new Path(TEST_ROOT_DIR, "io_control"); private static Path WRITE_DIR = new Path(TEST_ROOT_DIR, "io_write"); private static Path READ_DIR = new Path(TEST_ROOT_DIR, "io_read"); private static Path DATA_DIR = new Path(TEST_ROOT_DIR, "io_data"); private static Path HDFS_TEST_DIR = new Path("/tmp/DFSCIOTest"); private static String HDFS_LIB_VERSION = System.getProperty("libhdfs.version", "1"); private static String CHMOD = new String("chmod"); private static Path HDFS_SHLIB = new Path(HDFS_TEST_DIR + "/libhdfs.so." + HDFS_LIB_VERSION); private static Path HDFS_READ = new Path(HDFS_TEST_DIR + "/hdfs_read"); private static Path HDFS_WRITE = new Path(HDFS_TEST_DIR + "/hdfs_write"); /** * Run the test with default parameters. * * @throws Exception */ public void testIOs() throws Exception { testIOs(10, 10); } /** * Run the test with the specified parameters. * * @param fileSize file size * @param nrFiles number of files * @throws IOException */ public static void testIOs(int fileSize, int nrFiles) throws IOException { FileSystem fs = FileSystem.get(fsConfig); createControlFile(fs, fileSize, nrFiles); writeTest(fs); readTest(fs); } private static void createControlFile( FileSystem fs, int fileSize, // in MB int nrFiles) throws IOException { LOG.info("creating control file: " + fileSize + " mega bytes, " + nrFiles + " files"); fs.delete(CONTROL_DIR, true); for (int i = 0; i < nrFiles; i++) { String name = getFileName(i); Path controlFile = new Path(CONTROL_DIR, "in_file_" + name); SequenceFile.Writer writer = null; try { writer = SequenceFile.createWriter( fs, fsConfig, controlFile, Text.class, LongWritable.class, CompressionType.NONE); writer.append(new Text(name), new LongWritable(fileSize)); } catch (Exception e) { throw new IOException(e.getLocalizedMessage()); } finally { if (writer != null) writer.close(); writer = null; } } LOG.info("created control files for: " + nrFiles + " files"); } private static String getFileName(int fIdx) { return BASE_FILE_NAME + Integer.toString(fIdx); } /** * Write/Read mapper base class. * * <p>Collects the following statistics per task: * * <ul> * <li>number of tasks completed * <li>number of bytes written/read * <li>execution time * <li>i/o rate * <li>i/o rate squared * </ul> */ private abstract static class IOStatMapper extends IOMapperBase<Long> { IOStatMapper() {} void collectStats(OutputCollector<Text, Text> output, String name, long execTime, Long objSize) throws IOException { long totalSize = objSize.longValue(); float ioRateMbSec = (float) totalSize * 1000 / (execTime * MEGA); LOG.info("Number of bytes processed = " + totalSize); LOG.info("Exec time = " + execTime); LOG.info("IO rate = " + ioRateMbSec); output.collect( new Text(AccumulatingReducer.VALUE_TYPE_LONG + "tasks"), new Text(String.valueOf(1))); output.collect( new Text(AccumulatingReducer.VALUE_TYPE_LONG + "size"), new Text(String.valueOf(totalSize))); output.collect( new Text(AccumulatingReducer.VALUE_TYPE_LONG + "time"), new Text(String.valueOf(execTime))); output.collect( new Text(AccumulatingReducer.VALUE_TYPE_FLOAT + "rate"), new Text(String.valueOf(ioRateMbSec * 1000))); output.collect( new Text(AccumulatingReducer.VALUE_TYPE_FLOAT + "sqrate"), new Text(String.valueOf(ioRateMbSec * ioRateMbSec * 1000))); } } /** Write mapper class. */ public static class WriteMapper extends IOStatMapper { public WriteMapper() { super(); for (int i = 0; i < bufferSize; i++) buffer[i] = (byte) ('0' + i % 50); } public Long doIO(Reporter reporter, String name, long totalSize) throws IOException { // create file totalSize *= MEGA; // create instance of local filesystem FileSystem localFS = FileSystem.getLocal(fsConfig); try { // native runtime Runtime runTime = Runtime.getRuntime(); // copy the dso and executable from dfs and chmod them synchronized (this) { localFS.delete(HDFS_TEST_DIR, true); if (!(localFS.mkdirs(HDFS_TEST_DIR))) { throw new IOException("Failed to create " + HDFS_TEST_DIR + " on local filesystem"); } } synchronized (this) { if (!localFS.exists(HDFS_SHLIB)) { FileUtil.copy(fs, HDFS_SHLIB, localFS, HDFS_SHLIB, false, fsConfig); String chmodCmd = new String(CHMOD + " a+x " + HDFS_SHLIB); Process process = runTime.exec(chmodCmd); int exitStatus = process.waitFor(); if (exitStatus != 0) { throw new IOException(chmodCmd + ": Failed with exitStatus: " + exitStatus); } } } synchronized (this) { if (!localFS.exists(HDFS_WRITE)) { FileUtil.copy(fs, HDFS_WRITE, localFS, HDFS_WRITE, false, fsConfig); String chmodCmd = new String(CHMOD + " a+x " + HDFS_WRITE); Process process = runTime.exec(chmodCmd); int exitStatus = process.waitFor(); if (exitStatus != 0) { throw new IOException(chmodCmd + ": Failed with exitStatus: " + exitStatus); } } } // exec the C program Path outFile = new Path(DATA_DIR, name); String writeCmd = new String(HDFS_WRITE + " " + outFile + " " + totalSize + " " + bufferSize); Process process = runTime.exec(writeCmd, null, new File(HDFS_TEST_DIR.toString())); int exitStatus = process.waitFor(); if (exitStatus != 0) { throw new IOException(writeCmd + ": Failed with exitStatus: " + exitStatus); } } catch (InterruptedException interruptedException) { reporter.setStatus(interruptedException.toString()); } finally { localFS.close(); } return new Long(totalSize); } } private static void writeTest(FileSystem fs) throws IOException { fs.delete(DATA_DIR, true); fs.delete(WRITE_DIR, true); runIOTest(WriteMapper.class, WRITE_DIR); } private static void runIOTest(Class<? extends Mapper> mapperClass, Path outputDir) throws IOException { JobConf job = new JobConf(fsConfig, DFSCIOTest.class); FileInputFormat.setInputPaths(job, CONTROL_DIR); job.setInputFormat(SequenceFileInputFormat.class); job.setMapperClass(mapperClass); job.setReducerClass(AccumulatingReducer.class); FileOutputFormat.setOutputPath(job, outputDir); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setNumReduceTasks(1); JobClient.runJob(job); } /** Read mapper class. */ public static class ReadMapper extends IOStatMapper { public ReadMapper() { super(); } public Long doIO(Reporter reporter, String name, long totalSize) throws IOException { totalSize *= MEGA; // create instance of local filesystem FileSystem localFS = FileSystem.getLocal(fsConfig); try { // native runtime Runtime runTime = Runtime.getRuntime(); // copy the dso and executable from dfs synchronized (this) { localFS.delete(HDFS_TEST_DIR, true); if (!(localFS.mkdirs(HDFS_TEST_DIR))) { throw new IOException("Failed to create " + HDFS_TEST_DIR + " on local filesystem"); } } synchronized (this) { if (!localFS.exists(HDFS_SHLIB)) { if (!FileUtil.copy(fs, HDFS_SHLIB, localFS, HDFS_SHLIB, false, fsConfig)) { throw new IOException("Failed to copy " + HDFS_SHLIB + " to local filesystem"); } String chmodCmd = new String(CHMOD + " a+x " + HDFS_SHLIB); Process process = runTime.exec(chmodCmd); int exitStatus = process.waitFor(); if (exitStatus != 0) { throw new IOException(chmodCmd + ": Failed with exitStatus: " + exitStatus); } } } synchronized (this) { if (!localFS.exists(HDFS_READ)) { if (!FileUtil.copy(fs, HDFS_READ, localFS, HDFS_READ, false, fsConfig)) { throw new IOException("Failed to copy " + HDFS_READ + " to local filesystem"); } String chmodCmd = new String(CHMOD + " a+x " + HDFS_READ); Process process = runTime.exec(chmodCmd); int exitStatus = process.waitFor(); if (exitStatus != 0) { throw new IOException(chmodCmd + ": Failed with exitStatus: " + exitStatus); } } } // exec the C program Path inFile = new Path(DATA_DIR, name); String readCmd = new String(HDFS_READ + " " + inFile + " " + totalSize + " " + bufferSize); Process process = runTime.exec(readCmd, null, new File(HDFS_TEST_DIR.toString())); int exitStatus = process.waitFor(); if (exitStatus != 0) { throw new IOException(HDFS_READ + ": Failed with exitStatus: " + exitStatus); } } catch (InterruptedException interruptedException) { reporter.setStatus(interruptedException.toString()); } finally { localFS.close(); } return new Long(totalSize); } } private static void readTest(FileSystem fs) throws IOException { fs.delete(READ_DIR, true); runIOTest(ReadMapper.class, READ_DIR); } private static void sequentialTest(FileSystem fs, int testType, int fileSize, int nrFiles) throws Exception { IOStatMapper ioer = null; if (testType == TEST_TYPE_READ) ioer = new ReadMapper(); else if (testType == TEST_TYPE_WRITE) ioer = new WriteMapper(); else return; for (int i = 0; i < nrFiles; i++) ioer.doIO(Reporter.NULL, BASE_FILE_NAME + Integer.toString(i), MEGA * fileSize); } public static void main(String[] args) { int testType = TEST_TYPE_READ; int bufferSize = DEFAULT_BUFFER_SIZE; int fileSize = 1; int nrFiles = 1; String resFileName = DEFAULT_RES_FILE_NAME; boolean isSequential = false; String version = "DFSCIOTest.0.0.1"; String usage = "Usage: DFSCIOTest -read | -write | -clean [-nrFiles N] [-fileSize MB] [-resFile resultFileName] [-bufferSize Bytes] "; System.out.println(version); if (args.length == 0) { System.err.println(usage); System.exit(-1); } for (int i = 0; i < args.length; i++) { // parse command line if (args[i].startsWith("-r")) { testType = TEST_TYPE_READ; } else if (args[i].startsWith("-w")) { testType = TEST_TYPE_WRITE; } else if (args[i].startsWith("-clean")) { testType = TEST_TYPE_CLEANUP; } else if (args[i].startsWith("-seq")) { isSequential = true; } else if (args[i].equals("-nrFiles")) { nrFiles = Integer.parseInt(args[++i]); } else if (args[i].equals("-fileSize")) { fileSize = Integer.parseInt(args[++i]); } else if (args[i].equals("-bufferSize")) { bufferSize = Integer.parseInt(args[++i]); } else if (args[i].equals("-resFile")) { resFileName = args[++i]; } } LOG.info("nrFiles = " + nrFiles); LOG.info("fileSize (MB) = " + fileSize); LOG.info("bufferSize = " + bufferSize); try { fsConfig.setInt("test.io.file.buffer.size", bufferSize); FileSystem fs = FileSystem.get(fsConfig); if (testType != TEST_TYPE_CLEANUP) { fs.delete(HDFS_TEST_DIR, true); if (!fs.mkdirs(HDFS_TEST_DIR)) { throw new IOException("Mkdirs failed to create " + HDFS_TEST_DIR.toString()); } // Copy the executables over to the remote filesystem String hadoopHome = System.getenv("HADOOP_PREFIX"); fs.copyFromLocalFile( new Path(hadoopHome + "/libhdfs/libhdfs.so." + HDFS_LIB_VERSION), HDFS_SHLIB); fs.copyFromLocalFile(new Path(hadoopHome + "/libhdfs/hdfs_read"), HDFS_READ); fs.copyFromLocalFile(new Path(hadoopHome + "/libhdfs/hdfs_write"), HDFS_WRITE); } if (isSequential) { long tStart = System.currentTimeMillis(); sequentialTest(fs, testType, fileSize, nrFiles); long execTime = System.currentTimeMillis() - tStart; String resultLine = "Seq Test exec time sec: " + (float) execTime / 1000; LOG.info(resultLine); return; } if (testType == TEST_TYPE_CLEANUP) { cleanup(fs); return; } createControlFile(fs, fileSize, nrFiles); long tStart = System.currentTimeMillis(); if (testType == TEST_TYPE_WRITE) writeTest(fs); if (testType == TEST_TYPE_READ) readTest(fs); long execTime = System.currentTimeMillis() - tStart; analyzeResult(fs, testType, execTime, resFileName); } catch (Exception e) { System.err.print(e.getLocalizedMessage()); System.exit(-1); } } private static void analyzeResult(FileSystem fs, int testType, long execTime, String resFileName) throws IOException { Path reduceFile; if (testType == TEST_TYPE_WRITE) reduceFile = new Path(WRITE_DIR, "part-00000"); else reduceFile = new Path(READ_DIR, "part-00000"); DataInputStream in; in = new DataInputStream(fs.open(reduceFile)); BufferedReader lines; lines = new BufferedReader(new InputStreamReader(in)); long tasks = 0; long size = 0; long time = 0; float rate = 0; float sqrate = 0; String line; while ((line = lines.readLine()) != null) { StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%"); String attr = tokens.nextToken(); if (attr.endsWith(":tasks")) tasks = Long.parseLong(tokens.nextToken()); else if (attr.endsWith(":size")) size = Long.parseLong(tokens.nextToken()); else if (attr.endsWith(":time")) time = Long.parseLong(tokens.nextToken()); else if (attr.endsWith(":rate")) rate = Float.parseFloat(tokens.nextToken()); else if (attr.endsWith(":sqrate")) sqrate = Float.parseFloat(tokens.nextToken()); } double med = rate / 1000 / tasks; double stdDev = Math.sqrt(Math.abs(sqrate / 1000 / tasks - med * med)); String resultLines[] = { "----- DFSCIOTest ----- : " + ((testType == TEST_TYPE_WRITE) ? "write" : (testType == TEST_TYPE_READ) ? "read" : "unknown"), " Date & time: " + new Date(System.currentTimeMillis()), " Number of files: " + tasks, "Total MBytes processed: " + size / MEGA, " Throughput mb/sec: " + size * 1000.0 / (time * MEGA), "Average IO rate mb/sec: " + med, " Std IO rate deviation: " + stdDev, " Test exec time sec: " + (float) execTime / 1000, "" }; PrintStream res = new PrintStream(new FileOutputStream(new File(resFileName), true)); for (int i = 0; i < resultLines.length; i++) { LOG.info(resultLines[i]); res.println(resultLines[i]); } } private static void cleanup(FileSystem fs) throws Exception { LOG.info("Cleaning up test files"); fs.delete(new Path(TEST_ROOT_DIR), true); fs.delete(HDFS_TEST_DIR, true); } }
public class TestFileSystem extends TestCase { private static final Log LOG = FileSystem.LOG; private static Configuration conf = new Configuration(); private static int BUFFER_SIZE = conf.getInt("io.file.buffer.size", 4096); private static final long MEGA = 1024 * 1024; private static final int SEEKS_PER_FILE = 4; private static String ROOT = System.getProperty("test.build.data", "fs_test"); private static Path CONTROL_DIR = new Path(ROOT, "fs_control"); private static Path WRITE_DIR = new Path(ROOT, "fs_write"); private static Path READ_DIR = new Path(ROOT, "fs_read"); private static Path DATA_DIR = new Path(ROOT, "fs_data"); public void testFs() throws Exception { testFs(10 * MEGA, 100, 0); } public static void testFs(long megaBytes, int numFiles, long seed) throws Exception { FileSystem fs = FileSystem.get(conf); if (seed == 0) seed = new Random().nextLong(); LOG.info("seed = " + seed); createControlFile(fs, megaBytes, numFiles, seed); writeTest(fs, false); readTest(fs, false); seekTest(fs, false); fs.delete(CONTROL_DIR, true); fs.delete(DATA_DIR, true); fs.delete(WRITE_DIR, true); fs.delete(READ_DIR, true); } public static void testCommandFormat() throws Exception { // This should go to TestFsShell.java when it is added. CommandFormat cf; cf = new CommandFormat("copyToLocal", 2, 2, "crc", "ignoreCrc"); assertEquals(cf.parse(new String[] {"-get", "file", "-"}, 1).get(1), "-"); try { cf.parse(new String[] {"-get", "file", "-ignoreCrc", "/foo"}, 1); fail("Expected parsing to fail as it should stop at first non-option"); } catch (Exception e) { // Expected } cf = new CommandFormat("tail", 1, 1, "f"); assertEquals(cf.parse(new String[] {"-tail", "fileName"}, 1).get(0), "fileName"); assertEquals(cf.parse(new String[] {"-tail", "-f", "fileName"}, 1).get(0), "fileName"); cf = new CommandFormat("setrep", 2, 2, "R", "w"); assertEquals(cf.parse(new String[] {"-setrep", "-R", "2", "/foo/bar"}, 1).get(1), "/foo/bar"); cf = new CommandFormat("put", 2, 10000); assertEquals(cf.parse(new String[] {"-put", "-", "dest"}, 1).get(1), "dest"); } public static void createControlFile(FileSystem fs, long megaBytes, int numFiles, long seed) throws Exception { LOG.info("creating control file: " + megaBytes + " bytes, " + numFiles + " files"); Path controlFile = new Path(CONTROL_DIR, "files"); fs.delete(controlFile, true); Random random = new Random(seed); SequenceFile.Writer writer = SequenceFile.createWriter( fs, conf, controlFile, Text.class, LongWritable.class, CompressionType.NONE); long totalSize = 0; long maxSize = ((megaBytes / numFiles) * 2) + 1; try { while (totalSize < megaBytes) { Text name = new Text(Long.toString(random.nextLong())); long size = random.nextLong(); if (size < 0) size = -size; size = size % maxSize; // LOG.info(" adding: name="+name+" size="+size); writer.append(name, new LongWritable(size)); totalSize += size; } } finally { writer.close(); } LOG.info("created control file for: " + totalSize + " bytes"); } public static class WriteMapper extends Configured implements Mapper<Text, LongWritable, Text, LongWritable> { private Random random = new Random(); private byte[] buffer = new byte[BUFFER_SIZE]; private FileSystem fs; private boolean fastCheck; // a random suffix per task private String suffix = "-" + random.nextLong(); { try { fs = FileSystem.get(conf); } catch (IOException e) { throw new RuntimeException(e); } } public WriteMapper() { super(null); } public WriteMapper(Configuration conf) { super(conf); } public void configure(JobConf job) { setConf(job); fastCheck = job.getBoolean("fs.test.fastCheck", false); } public void map( Text key, LongWritable value, OutputCollector<Text, LongWritable> collector, Reporter reporter) throws IOException { String name = key.toString(); long size = value.get(); long seed = Long.parseLong(name); random.setSeed(seed); reporter.setStatus("creating " + name); // write to temp file initially to permit parallel execution Path tempFile = new Path(DATA_DIR, name + suffix); OutputStream out = fs.create(tempFile); long written = 0; try { while (written < size) { if (fastCheck) { Arrays.fill(buffer, (byte) random.nextInt(Byte.MAX_VALUE)); } else { random.nextBytes(buffer); } long remains = size - written; int length = (remains <= buffer.length) ? (int) remains : buffer.length; out.write(buffer, 0, length); written += length; reporter.setStatus("writing " + name + "@" + written + "/" + size); } } finally { out.close(); } // rename to final location fs.rename(tempFile, new Path(DATA_DIR, name)); collector.collect(new Text("bytes"), new LongWritable(written)); reporter.setStatus("wrote " + name); } public void close() {} } public static void writeTest(FileSystem fs, boolean fastCheck) throws Exception { fs.delete(DATA_DIR, true); fs.delete(WRITE_DIR, true); JobConf job = new JobConf(conf, TestFileSystem.class); job.setBoolean("fs.test.fastCheck", fastCheck); FileInputFormat.setInputPaths(job, CONTROL_DIR); job.setInputFormat(SequenceFileInputFormat.class); job.setMapperClass(WriteMapper.class); job.setReducerClass(LongSumReducer.class); FileOutputFormat.setOutputPath(job, WRITE_DIR); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); job.setNumReduceTasks(1); JobClient.runJob(job); } public static class ReadMapper extends Configured implements Mapper<Text, LongWritable, Text, LongWritable> { private Random random = new Random(); private byte[] buffer = new byte[BUFFER_SIZE]; private byte[] check = new byte[BUFFER_SIZE]; private FileSystem fs; private boolean fastCheck; { try { fs = FileSystem.get(conf); } catch (IOException e) { throw new RuntimeException(e); } } public ReadMapper() { super(null); } public ReadMapper(Configuration conf) { super(conf); } public void configure(JobConf job) { setConf(job); fastCheck = job.getBoolean("fs.test.fastCheck", false); } public void map( Text key, LongWritable value, OutputCollector<Text, LongWritable> collector, Reporter reporter) throws IOException { String name = key.toString(); long size = value.get(); long seed = Long.parseLong(name); random.setSeed(seed); reporter.setStatus("opening " + name); DataInputStream in = new DataInputStream(fs.open(new Path(DATA_DIR, name))); long read = 0; try { while (read < size) { long remains = size - read; int n = (remains <= buffer.length) ? (int) remains : buffer.length; in.readFully(buffer, 0, n); read += n; if (fastCheck) { Arrays.fill(check, (byte) random.nextInt(Byte.MAX_VALUE)); } else { random.nextBytes(check); } if (n != buffer.length) { Arrays.fill(buffer, n, buffer.length, (byte) 0); Arrays.fill(check, n, check.length, (byte) 0); } assertTrue(Arrays.equals(buffer, check)); reporter.setStatus("reading " + name + "@" + read + "/" + size); } } finally { in.close(); } collector.collect(new Text("bytes"), new LongWritable(read)); reporter.setStatus("read " + name); } public void close() {} } public static void readTest(FileSystem fs, boolean fastCheck) throws Exception { fs.delete(READ_DIR, true); JobConf job = new JobConf(conf, TestFileSystem.class); job.setBoolean("fs.test.fastCheck", fastCheck); FileInputFormat.setInputPaths(job, CONTROL_DIR); job.setInputFormat(SequenceFileInputFormat.class); job.setMapperClass(ReadMapper.class); job.setReducerClass(LongSumReducer.class); FileOutputFormat.setOutputPath(job, READ_DIR); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); job.setNumReduceTasks(1); JobClient.runJob(job); } public static class SeekMapper<K> extends Configured implements Mapper<Text, LongWritable, K, LongWritable> { private Random random = new Random(); private byte[] check = new byte[BUFFER_SIZE]; private FileSystem fs; private boolean fastCheck; { try { fs = FileSystem.get(conf); } catch (IOException e) { throw new RuntimeException(e); } } public SeekMapper() { super(null); } public SeekMapper(Configuration conf) { super(conf); } public void configure(JobConf job) { setConf(job); fastCheck = job.getBoolean("fs.test.fastCheck", false); } public void map( Text key, LongWritable value, OutputCollector<K, LongWritable> collector, Reporter reporter) throws IOException { String name = key.toString(); long size = value.get(); long seed = Long.parseLong(name); if (size == 0) return; reporter.setStatus("opening " + name); FSDataInputStream in = fs.open(new Path(DATA_DIR, name)); try { for (int i = 0; i < SEEKS_PER_FILE; i++) { // generate a random position long position = Math.abs(random.nextLong()) % size; // seek file to that position reporter.setStatus("seeking " + name); in.seek(position); byte b = in.readByte(); // check that byte matches byte checkByte = 0; // advance random state to that position random.setSeed(seed); for (int p = 0; p <= position; p += check.length) { reporter.setStatus("generating data for " + name); if (fastCheck) { checkByte = (byte) random.nextInt(Byte.MAX_VALUE); } else { random.nextBytes(check); checkByte = check[(int) (position % check.length)]; } } assertEquals(b, checkByte); } } finally { in.close(); } } public void close() {} } public static void seekTest(FileSystem fs, boolean fastCheck) throws Exception { fs.delete(READ_DIR, true); JobConf job = new JobConf(conf, TestFileSystem.class); job.setBoolean("fs.test.fastCheck", fastCheck); FileInputFormat.setInputPaths(job, CONTROL_DIR); job.setInputFormat(SequenceFileInputFormat.class); job.setMapperClass(SeekMapper.class); job.setReducerClass(LongSumReducer.class); FileOutputFormat.setOutputPath(job, READ_DIR); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); job.setNumReduceTasks(1); JobClient.runJob(job); } public static void main(String[] args) throws Exception { int megaBytes = 10; int files = 100; boolean noRead = false; boolean noWrite = false; boolean noSeek = false; boolean fastCheck = false; long seed = new Random().nextLong(); String usage = "Usage: TestFileSystem -files N -megaBytes M [-noread] [-nowrite] [-noseek] [-fastcheck]"; if (args.length == 0) { System.err.println(usage); System.exit(-1); } for (int i = 0; i < args.length; i++) { // parse command line if (args[i].equals("-files")) { files = Integer.parseInt(args[++i]); } else if (args[i].equals("-megaBytes")) { megaBytes = Integer.parseInt(args[++i]); } else if (args[i].equals("-noread")) { noRead = true; } else if (args[i].equals("-nowrite")) { noWrite = true; } else if (args[i].equals("-noseek")) { noSeek = true; } else if (args[i].equals("-fastcheck")) { fastCheck = true; } } LOG.info("seed = " + seed); LOG.info("files = " + files); LOG.info("megaBytes = " + megaBytes); FileSystem fs = FileSystem.get(conf); if (!noWrite) { createControlFile(fs, megaBytes * MEGA, files, seed); writeTest(fs, fastCheck); } if (!noRead) { readTest(fs, fastCheck); } if (!noSeek) { seekTest(fs, fastCheck); } } public void testFsCache() throws Exception { { long now = System.currentTimeMillis(); String[] users = new String[] {"foo", "bar"}; final Configuration conf = new Configuration(); FileSystem[] fs = new FileSystem[users.length]; for (int i = 0; i < users.length; i++) { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(users[i]); fs[i] = ugi.doAs( new PrivilegedExceptionAction<FileSystem>() { public FileSystem run() throws IOException { return FileSystem.get(conf); } }); for (int j = 0; j < i; j++) { assertFalse(fs[j] == fs[i]); } } FileSystem.closeAll(); } { try { runTestCache(HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); } catch (java.net.BindException be) { LOG.warn( "Cannot test HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT (=" + HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT + ")", be); } runTestCache(0); } } static void runTestCache(int port) throws Exception { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).nameNodePort(port).numDataNodes(2).build(); URI uri = cluster.getFileSystem().getUri(); LOG.info("uri=" + uri); { FileSystem fs = FileSystem.get(uri, new Configuration()); checkPath(cluster, fs); for (int i = 0; i < 100; i++) { assertTrue(fs == FileSystem.get(uri, new Configuration())); } } if (port == HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT) { // test explicit default port URI uri2 = new URI( uri.getScheme(), uri.getUserInfo(), uri.getHost(), HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT, uri.getPath(), uri.getQuery(), uri.getFragment()); LOG.info("uri2=" + uri2); FileSystem fs = FileSystem.get(uri2, conf); checkPath(cluster, fs); for (int i = 0; i < 100; i++) { assertTrue(fs == FileSystem.get(uri2, new Configuration())); } } } finally { if (cluster != null) cluster.shutdown(); } } static void checkPath(MiniDFSCluster cluster, FileSystem fileSys) throws IOException { InetSocketAddress add = cluster.getNameNode().getNameNodeAddress(); // Test upper/lower case fileSys.checkPath( new Path("hdfs://" + StringUtils.toUpperCase(add.getHostName()) + ":" + add.getPort())); } public void testFsClose() throws Exception { { Configuration conf = new Configuration(); new Path("file:///").getFileSystem(conf); FileSystem.closeAll(); } } public void testFsShutdownHook() throws Exception { final Set<FileSystem> closed = Collections.synchronizedSet(new HashSet<FileSystem>()); Configuration conf = new Configuration(); Configuration confNoAuto = new Configuration(); conf.setClass("fs.test.impl", TestShutdownFileSystem.class, FileSystem.class); confNoAuto.setClass("fs.test.impl", TestShutdownFileSystem.class, FileSystem.class); confNoAuto.setBoolean("fs.automatic.close", false); TestShutdownFileSystem fsWithAuto = (TestShutdownFileSystem) (new Path("test://a/").getFileSystem(conf)); TestShutdownFileSystem fsWithoutAuto = (TestShutdownFileSystem) (new Path("test://b/").getFileSystem(confNoAuto)); fsWithAuto.setClosedSet(closed); fsWithoutAuto.setClosedSet(closed); // Different URIs should result in different FS instances assertNotSame(fsWithAuto, fsWithoutAuto); FileSystem.CACHE.closeAll(true); assertEquals(1, closed.size()); assertTrue(closed.contains(fsWithAuto)); closed.clear(); FileSystem.closeAll(); assertEquals(1, closed.size()); assertTrue(closed.contains(fsWithoutAuto)); } public void testCacheKeysAreCaseInsensitive() throws Exception { Configuration conf = new Configuration(); // check basic equality FileSystem.Cache.Key lowercaseCachekey1 = new FileSystem.Cache.Key(new URI("hdfs://localhost:12345/"), conf); FileSystem.Cache.Key lowercaseCachekey2 = new FileSystem.Cache.Key(new URI("hdfs://localhost:12345/"), conf); assertEquals(lowercaseCachekey1, lowercaseCachekey2); // check insensitive equality FileSystem.Cache.Key uppercaseCachekey = new FileSystem.Cache.Key(new URI("HDFS://Localhost:12345/"), conf); assertEquals(lowercaseCachekey2, uppercaseCachekey); // check behaviour with collections List<FileSystem.Cache.Key> list = new ArrayList<FileSystem.Cache.Key>(); list.add(uppercaseCachekey); assertTrue(list.contains(uppercaseCachekey)); assertTrue(list.contains(lowercaseCachekey2)); Set<FileSystem.Cache.Key> set = new HashSet<FileSystem.Cache.Key>(); set.add(uppercaseCachekey); assertTrue(set.contains(uppercaseCachekey)); assertTrue(set.contains(lowercaseCachekey2)); Map<FileSystem.Cache.Key, String> map = new HashMap<FileSystem.Cache.Key, String>(); map.put(uppercaseCachekey, ""); assertTrue(map.containsKey(uppercaseCachekey)); assertTrue(map.containsKey(lowercaseCachekey2)); } public static void testFsUniqueness(long megaBytes, int numFiles, long seed) throws Exception { // multiple invocations of FileSystem.get return the same object. FileSystem fs1 = FileSystem.get(conf); FileSystem fs2 = FileSystem.get(conf); assertTrue(fs1 == fs2); // multiple invocations of FileSystem.newInstance return different objects fs1 = FileSystem.newInstance(conf); fs2 = FileSystem.newInstance(conf); assertTrue(fs1 != fs2 && !fs1.equals(fs2)); fs1.close(); fs2.close(); } public static class TestShutdownFileSystem extends RawLocalFileSystem { private Set<FileSystem> closedSet; public void setClosedSet(Set<FileSystem> closedSet) { this.closedSet = closedSet; } public void close() throws IOException { if (closedSet != null) { closedSet.add(this); } super.close(); } } }