@Test public void testNumSamplesOneFile() { try { final String tempFile = TestFileUtils.createTempFile(TEST_DATA1); final Configuration conf = new Configuration(); final TestDelimitedInputFormat format = new TestDelimitedInputFormat(); format.setFilePath(tempFile.replace("file", "test")); format.configure(conf); TestFileSystem.resetStreamOpenCounter(); format.getStatistics(null); Assert.assertEquals( "Wrong number of samples taken.", DEFAULT_NUM_SAMPLES, TestFileSystem.getNumtimeStreamOpened()); TestDelimitedInputFormat format2 = new TestDelimitedInputFormat(); format2.setFilePath(tempFile.replace("file", "test")); format2.setNumLineSamples(8); format2.configure(conf); TestFileSystem.resetStreamOpenCounter(); format2.getStatistics(null); Assert.assertEquals( "Wrong number of samples taken.", 8, TestFileSystem.getNumtimeStreamOpened()); } catch (Exception e) { e.printStackTrace(); Assert.fail(e.getMessage()); } }
@Test public void testCachedStatistics() { try { final String tempFile = TestFileUtils.createTempFile(TEST_DATA1); final Configuration conf = new Configuration(); final TestDelimitedInputFormat format = new TestDelimitedInputFormat(); format.setFilePath("test://" + tempFile); format.configure(conf); TestFileSystem.resetStreamOpenCounter(); BaseStatistics stats = format.getStatistics(null); Assert.assertEquals( "Wrong number of samples taken.", DEFAULT_NUM_SAMPLES, TestFileSystem.getNumtimeStreamOpened()); final TestDelimitedInputFormat format2 = new TestDelimitedInputFormat(); format2.setFilePath("test://" + tempFile); format2.configure(conf); TestFileSystem.resetStreamOpenCounter(); BaseStatistics stats2 = format2.getStatistics(stats); Assert.assertTrue( "Using cached statistics should cicumvent sampling.", 0 == TestFileSystem.getNumtimeStreamOpened()); Assert.assertTrue("Using cached statistics should cicumvent sampling.", stats == stats2); } catch (Exception e) { e.printStackTrace(); Assert.fail(e.getMessage()); } }