@Test public void testDifferentDelimiter() { try { final String DELIMITER = "12345678-"; String testData = TEST_DATA1.replace("\n", DELIMITER); final String tempFile = TestFileUtils.createTempFile(testData); final Configuration conf = new Configuration(); final TestDelimitedInputFormat format = new TestDelimitedInputFormat(); format.setFilePath(tempFile); format.setDelimiter(DELIMITER); format.configure(conf); BaseStatistics stats = format.getStatistics(null); final int numLines = TEST_DATA_1_LINES; final float avgWidth = ((float) testData.length()) / TEST_DATA_1_LINES; Assert.assertTrue( "Wrong record count.", stats.getNumberOfRecords() < numLines + 1 & stats.getNumberOfRecords() > numLines - 1); Assert.assertTrue( "Wrong avg record size.", stats.getAverageRecordWidth() < avgWidth + 1 & stats.getAverageRecordWidth() > avgWidth - 1); } catch (Exception e) { e.printStackTrace(); Assert.fail(e.getMessage()); } }
@Test public void testSamplingDirectory() { try { final String tempFile = TestFileUtils.createTempFileDir(TEST_DATA1, TEST_DATA2); final Configuration conf = new Configuration(); final TestDelimitedInputFormat format = new TestDelimitedInputFormat(); format.setFilePath(tempFile); format.configure(conf); BaseStatistics stats = format.getStatistics(null); final int maxNumLines = (int) Math.ceil( TOTAL_SIZE / ((double) Math.min(TEST_DATA_1_LINEWIDTH, TEST_DATA_2_LINEWIDTH))); final int minNumLines = (int) (TOTAL_SIZE / ((double) Math.max(TEST_DATA_1_LINEWIDTH, TEST_DATA_2_LINEWIDTH))); final float maxAvgWidth = ((float) (TOTAL_SIZE)) / minNumLines; final float minAvgWidth = ((float) (TOTAL_SIZE)) / maxNumLines; if (!(stats.getNumberOfRecords() <= maxNumLines & stats.getNumberOfRecords() >= minNumLines)) { System.err.println( "Records: " + stats.getNumberOfRecords() + " out of (" + minNumLines + ", " + maxNumLines + ")."); Assert.fail("Wrong record count."); } if (!(stats.getAverageRecordWidth() <= maxAvgWidth & stats.getAverageRecordWidth() >= minAvgWidth)) { Assert.fail("Wrong avg record size."); } } catch (Exception e) { e.printStackTrace(); Assert.fail(e.getMessage()); } }