コード例 #1
0
  /**
   * output without streaming nor staging.
   *
   * @throws Exception if failed
   */
  @Test
  public void output_nomove() throws Exception {
    profile.setOutputStaging(false);
    profile.setOutputStreaming(false);
    profile
        .getLocalFileSystem()
        .getConf()
        .set(HadoopDataSourceUtil.KEY_LOCAL_TEMPDIR, localtemp.getPath());
    HadoopDataSourceCore core = new HadoopDataSourceCore(profile);
    setup(core);
    ModelOutput<StringBuilder> output =
        core.openOutput(context, StringBuilder.class, format, "output", "file.txt", counter);
    try {
      output.write(new StringBuilder("Hello, world!"));
    } finally {
      output.close();
    }
    assertThat(counter.get(), is(greaterThan(0L)));

    File target = new File(mapping, "output/file.txt");
    assertThat(target.exists(), is(false));

    commitAttempt(core);
    assertThat(target.exists(), is(true));

    commitTransaction(core);
    assertThat(target.exists(), is(true));

    assertThat(get(target), is(Arrays.asList("Hello, world!")));
  }
コード例 #2
0
  /**
   * simple input.
   *
   * @throws Exception if failed
   */
  @Test
  public void input() throws Exception {
    put(new File(mapping, "input/file.txt"), "Hello, world!");
    profile.setMinimumFragmentSize(-1);

    HadoopDataSourceCore core = new HadoopDataSourceCore(profile);
    List<DirectInputFragment> fragments =
        core.findInputFragments(StringBuilder.class, format, "input", FilePattern.compile("**"));
    assertThat(fragments.size(), is(1));

    List<String> results = consume(core, fragments);
    assertThat(counter.get(), is(greaterThan(0L)));
    assertThat(results.size(), is(1));
    assertThat(results, hasItem("Hello, world!"));
  }
コード例 #3
0
  /**
   * simple output.
   *
   * @throws Exception if failed
   */
  @Test
  public void output() throws Exception {
    HadoopDataSourceCore core = new HadoopDataSourceCore(profile);
    setup(core);
    ModelOutput<StringBuilder> output =
        core.openOutput(context, StringBuilder.class, format, "output", "file.txt", counter);
    try {
      output.write(new StringBuilder("Hello, world!"));
    } finally {
      output.close();
    }
    assertThat(counter.get(), is(greaterThan(0L)));

    File target = new File(mapping, "output/file.txt");
    assertThat(target.exists(), is(false));
    commitAttempt(core);

    assertThat(target.exists(), is(false));
    commitTransaction(core);

    assertThat(target.exists(), is(true));

    assertThat(get(target), is(Arrays.asList("Hello, world!")));
  }