/**
* simple input.
*
* @throws Exception if failed
*/
@Test
public void input() throws Exception {
put(new File(mapping, "input/file.txt"), "Hello, world!");
profile.setMinimumFragmentSize(-1);
HadoopDataSourceCore core = new HadoopDataSourceCore(profile);
List<DirectInputFragment> fragments =
core.findInputFragments(StringBuilder.class, format, "input", FilePattern.compile("**"));
assertThat(fragments.size(), is(1));
List<String> results = consume(core, fragments);
assertThat(counter.get(), is(greaterThan(0L)));
assertThat(results.size(), is(1));
assertThat(results, hasItem("Hello, world!"));
}
/**
* input multiple records.
*
* @throws Exception if failed
*/
@Test
public void input_large() throws Exception {
long fragmentSize = 1 * 1024 * 1024;
int fragmentCount = 20;
put(new File(mapping, "input/file.txt"), fragmentSize * fragmentCount);
profile.setMinimumFragmentSize(1);
profile.setPreferredFragmentSize(fragmentSize);
HadoopDataSourceCore core = new HadoopDataSourceCore(profile);
List<DirectInputFragment> fragments =
core.findInputFragments(StringBuilder.class, format, "input", FilePattern.compile("**"));
assertThat(fragments.size(), is(greaterThanOrEqualTo(fragmentCount / 2)));
for (DirectInputFragment fragment : fragments) {
assertThat(fragment.getSize(), is(greaterThanOrEqualTo(fragmentSize / 2)));
assertThat(fragment.getSize(), is(lessThanOrEqualTo(fragmentSize * 2)));
}
}
