コード例 #1
0
  private void testMultiFileFailedWriteHelper(
      EvenOddHDFSExactlyOnceWriter writer, ProcessingMode mode) {
    File meta = new File(testMeta.getDir());
    writer.setFilePath(meta.getAbsolutePath());

    writer.setup(new DummyContext(0, mode));

    writer.beginWindow(0);
    writer.input.put(0);
    writer.input.put(1);
    writer.input.put(2);
    writer.input.put(3);
    writer.endWindow();

    CheckPointWriter checkPointWriter = checkpoint(writer);

    writer.beginWindow(1);
    writer.input.put(4);
    writer.input.put(5);
    writer.endWindow();
    writer.teardown();

    restoreCheckPoint(checkPointWriter, writer);
    writer.setup(new DummyContext(0, mode));

    writer.beginWindow(2);
    writer.input.put(6);
    writer.input.put(7);
    writer.input.put(8);
    writer.input.put(9);
    writer.endWindow();
  }
コード例 #2
0
  // @Ignore
  @Test
  public void testMultiRollingFileFailedWriteOverwriteCache1() {
    prepareTest();
    EvenOddHDFSExactlyOnceWriter writer = new EvenOddHDFSExactlyOnceWriter();
    writer.setAppend(false);
    writer.setMaxOpenFiles(1);

    testMultiRollingFileFailedWriteOverwriteHelperCache1(writer, ProcessingMode.EXACTLY_ONCE);

    // Even file

    String evenFileName = testMeta.getDir() + File.separator + EVEN_FILE;
    String correctContents = "0\n" + "4\n" + "6\n";
    checkOutput(0, evenFileName, correctContents);

    correctContents = "8\n" + "6\n";
    checkOutput(1, evenFileName, correctContents);

    // Odd file

    String oddFileName = testMeta.getDir() + File.separator + ODD_FILE;

    correctContents = "1\n" + "5\n" + "7\n";
    checkOutput(0, oddFileName, correctContents);

    correctContents = "9\n" + "7\n";
    checkOutput(1, oddFileName, correctContents);
  }
コード例 #3
0
  // @Ignore
  @Test
  public void testMultiRollingFileFailedWriteOverwrite() {
    prepareTest();
    EvenOddHDFSExactlyOnceWriter writer = new EvenOddHDFSExactlyOnceWriter();
    writer.setAppend(false);

    testMultiRollingFileFailedWriteOverwriteHelper(writer, ProcessingMode.EXACTLY_ONCE);
  }
コード例 #4
0
  // @Ignore
  @Test
  public void testMultiRollingFileFailedWriteCache1() {
    prepareTest();
    EvenOddHDFSExactlyOnceWriter writer = new EvenOddHDFSExactlyOnceWriter();
    writer.setAppend(true);
    writer.setMaxOpenFiles(1);

    testMultiRollingFileFailedWriteHelperHelper(writer, ProcessingMode.EXACTLY_ONCE);
  }
コード例 #5
0
  // @Ignore
  @Test
  public void testMultiFileFailedWriteOverwrite() {
    prepareTest();
    EvenOddHDFSExactlyOnceWriter writer = new EvenOddHDFSExactlyOnceWriter();
    writer.setAppend(false);

    testMultiFileFailedWriteHelper(writer, ProcessingMode.EXACTLY_ONCE);

    String evenFileName = testMeta.getDir() + File.separator + EVEN_FILE;

    String correctContents = "0\n" + "2\n" + "6\n" + "8\n";

    checkOutput(-1, evenFileName, correctContents);

    String oddFileName = testMeta.getDir() + File.separator + ODD_FILE;

    correctContents = "1\n" + "3\n" + "7\n" + "9\n";

    checkOutput(-1, oddFileName, correctContents);
  }
コード例 #6
0
  // @Ignore
  @Test
  public void testMultiFileCompletedWriteInitial() {
    prepareTest();
    populateFile(EVEN_FILE, "0\n" + "2\n");
    populateFile(ODD_FILE, "1\n" + "3\n");
    EvenOddHDFSExactlyOnceWriter writer = new EvenOddHDFSExactlyOnceWriter();
    writer.setAppend(true);

    testMultiFileCompletedWriteHelper(writer, ProcessingMode.EXACTLY_ONCE);

    String evenFileName = testMeta.getDir() + File.separator + EVEN_FILE;

    String correctContents = "0\n" + "2\n" + "0\n" + "2\n" + "4\n" + "6\n";

    checkOutput(-1, evenFileName, correctContents);

    String oddFileName = testMeta.getDir() + File.separator + ODD_FILE;

    correctContents = "1\n" + "3\n" + "1\n" + "3\n" + "5\n" + "7\n";

    checkOutput(-1, oddFileName, correctContents);
  }
コード例 #7
0
  private void testMultiRollingFileCompletedWriteHelper(
      EvenOddHDFSExactlyOnceWriter writer, ProcessingMode mode) {
    writer.setMaxLength(4);
    File meta = new File(testMeta.getDir());
    writer.setFilePath(meta.getAbsolutePath());

    writer.setup(new DummyContext(0, mode));

    writer.beginWindow(0);
    writer.input.put(0);
    writer.input.put(1);
    writer.input.put(2);
    writer.input.put(3);
    writer.input.put(4);
    writer.input.put(5);
    writer.endWindow();

    writer.beginWindow(1);
    writer.input.put(6);
    writer.input.put(7);
    writer.input.put(8);
    writer.input.put(9);
    writer.input.put(6);
    writer.input.put(7);
    writer.endWindow();

    // Even file

    String evenFileName = testMeta.getDir() + File.separator + EVEN_FILE;

    String correctContents = "0\n" + "2\n" + "4\n";

    checkOutput(0, evenFileName, correctContents);

    correctContents = "6\n" + "8\n" + "6\n";

    checkOutput(1, evenFileName, correctContents);

    // Odd file

    String oddFileName = testMeta.getDir() + File.separator + ODD_FILE;

    correctContents = "1\n" + "3\n" + "5\n";

    checkOutput(0, oddFileName, correctContents);

    correctContents = "7\n" + "9\n" + "7\n";

    checkOutput(1, oddFileName, correctContents);
  }
コード例 #8
0
  private void testMultiFileCompletedWriteHelper(
      EvenOddHDFSExactlyOnceWriter writer, ProcessingMode mode) {
    File meta = new File(testMeta.getDir());
    writer.setFilePath(meta.getAbsolutePath());

    writer.setup(new DummyContext(0, mode));

    writer.beginWindow(0);
    writer.input.put(0);
    writer.input.put(1);
    writer.input.put(2);
    writer.input.put(3);
    writer.endWindow();

    writer.beginWindow(1);
    writer.input.put(4);
    writer.input.put(5);
    writer.input.put(6);
    writer.input.put(7);
    writer.endWindow();
  }