@Test
public void testNumSamplesOneFile() {
try {
final String tempFile = TestFileUtils.createTempFile(TEST_DATA1);
final Configuration conf = new Configuration();
final TestDelimitedInputFormat format = new TestDelimitedInputFormat();
format.setFilePath(tempFile.replace("file", "test"));
format.configure(conf);
TestFileSystem.resetStreamOpenCounter();
format.getStatistics(null);
Assert.assertEquals(
"Wrong number of samples taken.",
DEFAULT_NUM_SAMPLES,
TestFileSystem.getNumtimeStreamOpened());
TestDelimitedInputFormat format2 = new TestDelimitedInputFormat();
format2.setFilePath(tempFile.replace("file", "test"));
format2.setNumLineSamples(8);
format2.configure(conf);
TestFileSystem.resetStreamOpenCounter();
format2.getStatistics(null);
Assert.assertEquals(
"Wrong number of samples taken.", 8, TestFileSystem.getNumtimeStreamOpened());
} catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
@Test
public void testCachedStatistics() {
try {
final String tempFile = TestFileUtils.createTempFile(TEST_DATA1);
final Configuration conf = new Configuration();
final TestDelimitedInputFormat format = new TestDelimitedInputFormat();
format.setFilePath("test://" + tempFile);
format.configure(conf);
TestFileSystem.resetStreamOpenCounter();
BaseStatistics stats = format.getStatistics(null);
Assert.assertEquals(
"Wrong number of samples taken.",
DEFAULT_NUM_SAMPLES,
TestFileSystem.getNumtimeStreamOpened());
final TestDelimitedInputFormat format2 = new TestDelimitedInputFormat();
format2.setFilePath("test://" + tempFile);
format2.configure(conf);
TestFileSystem.resetStreamOpenCounter();
BaseStatistics stats2 = format2.getStatistics(stats);
Assert.assertTrue(
"Using cached statistics should cicumvent sampling.",
0 == TestFileSystem.getNumtimeStreamOpened());
Assert.assertTrue("Using cached statistics should cicumvent sampling.", stats == stats2);
} catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
@BeforeClass
public static void initialize() {
LogUtils.initializeDefaultConsoleLogger(Level.ERROR);
try {
TestFileSystem.registerTestFileSysten();
} catch (Throwable t) {
Assert.fail("Could not setup the mock test filesystem.");
}
try {
// make sure we do 4 samples
TestConfigUtils.loadGlobalConf(
new String[] {
PactConfigConstants.DELIMITED_FORMAT_MIN_LINE_SAMPLES_KEY,
PactConfigConstants.DELIMITED_FORMAT_MAX_LINE_SAMPLES_KEY
},
new String[] {"4", "4"});
TestDelimitedInputFormat.prepare();
} catch (Throwable t) {
Assert.fail("Could not load the global configuration.");
}
}
