/** * output without streaming nor staging. * * @throws Exception if failed */ @Test public void output_nomove() throws Exception { profile.setOutputStaging(false); profile.setOutputStreaming(false); profile .getLocalFileSystem() .getConf() .set(HadoopDataSourceUtil.KEY_LOCAL_TEMPDIR, localtemp.getPath()); HadoopDataSourceCore core = new HadoopDataSourceCore(profile); setup(core); ModelOutput<StringBuilder> output = core.openOutput(context, StringBuilder.class, format, "output", "file.txt", counter); try { output.write(new StringBuilder("Hello, world!")); } finally { output.close(); } assertThat(counter.get(), is(greaterThan(0L))); File target = new File(mapping, "output/file.txt"); assertThat(target.exists(), is(false)); commitAttempt(core); assertThat(target.exists(), is(true)); commitTransaction(core); assertThat(target.exists(), is(true)); assertThat(get(target), is(Arrays.asList("Hello, world!"))); }
/** * rollback output. * * @throws Exception if failed */ @Test public void output_rollback() throws Exception { HadoopDataSourceCore core = new HadoopDataSourceCore(profile); setup(core); ModelOutput<StringBuilder> output = core.openOutput(context, StringBuilder.class, format, "output", "file.txt", counter); try { output.write(new StringBuilder("Hello, world!")); } finally { output.close(); } cleanup(core); assertThat(new File(mapping, "output/file.txt").exists(), is(false)); }
/** * output multiple files. * * @throws Exception if failed */ @Test public void output_multifile() throws Exception { HadoopDataSourceCore core = new HadoopDataSourceCore(profile); setup(core); for (int i = 0; i < 3; i++) { ModelOutput<StringBuilder> output = core.openOutput( context, StringBuilder.class, format, "output", "file" + i + ".txt", counter); try { for (int j = 0; j < i + 1; j++) { output.write(new StringBuilder("Hello" + j)); } } finally { output.close(); } } commit(core); assertThat(get(new File(mapping, "output/file0.txt")), is(Arrays.asList("Hello0"))); assertThat(get(new File(mapping, "output/file1.txt")), is(Arrays.asList("Hello0", "Hello1"))); assertThat( get(new File(mapping, "output/file2.txt")), is(Arrays.asList("Hello0", "Hello1", "Hello2"))); }
/** * output multiple records. * * @throws Exception if failed */ @Test public void output_multirecord() throws Exception { HadoopDataSourceCore core = new HadoopDataSourceCore(profile); setup(core); ModelOutput<StringBuilder> output = core.openOutput(context, StringBuilder.class, format, "output", "file.txt", counter); try { output.write(new StringBuilder("Hello, world!")); } finally { output.close(); } File target = new File(mapping, "output/file.txt"); assertThat(target.exists(), is(false)); commitAttempt(core); assertThat(target.exists(), is(false)); commitTransaction(core); assertThat(target.exists(), is(true)); assertThat(get(target), is(Arrays.asList("Hello, world!"))); }