Exemplo n.º 1
0
  // @Ignore
  @Test
  public void testSingleRollingFileFailedWrite() {
    prepareTest();
    SingleHDFSExactlyOnceWriter writer = new SingleHDFSExactlyOnceWriter();
    writer.setAppend(true);

    testSingleRollingFileFailedWriteHelper(writer, ProcessingMode.EXACTLY_ONCE);

    // Rolling file 0

    String singleFileName = testMeta.getDir() + File.separator + SINGLE_FILE;

    String correctContents = "0\n" + "1\n" + "2\n";

    checkOutput(0, singleFileName, correctContents);

    // Rolling file 1

    correctContents = "3\n" + "4\n" + "5\n";

    checkOutput(1, singleFileName, correctContents);

    // Rolling file 2

    correctContents = "6\n" + "7\n" + "8\n";

    checkOutput(2, singleFileName, correctContents);
  }
Exemplo n.º 2
0
  // @Ignore
  @Test
  public void testSingleRollingFileCompletedWriteOverwriteInitial() {
    prepareTest();

    populateFile(SINGLE_FILE + ".0", "0\n" + "1\n" + "2\n");

    populateFile(SINGLE_FILE + ".1", "0\n" + "1\n" + "2\n");

    populateFile(SINGLE_FILE + ".2", "0\n" + "1\n" + "2\n");

    SingleHDFSExactlyOnceWriter writer = new SingleHDFSExactlyOnceWriter();
    writer.setAppend(false);

    testSingleRollingFileCompletedWriteHelper(writer, ProcessingMode.EXACTLY_ONCE);

    // Rolling file 0

    String singleFileName = testMeta.getDir() + File.separator + SINGLE_FILE;

    String correctContents = "0\n" + "1\n" + "2\n";

    checkOutput(0, singleFileName, correctContents);

    // Rolling file 1

    correctContents = "3\n" + "4\n" + "5\n";

    checkOutput(1, singleFileName, correctContents);
  }
Exemplo n.º 3
0
  // @Ignore
  @Test
  public void testSingleFileFailedWriteOverwrite() {
    prepareTest();
    SingleHDFSExactlyOnceWriter writer = new SingleHDFSExactlyOnceWriter();
    writer.setAppend(false);

    testSingleFileFailedWriteHelper(writer, ProcessingMode.EXACTLY_ONCE);

    String singleFileName = testMeta.getDir() + File.separator + SINGLE_FILE;

    String correctContents = "0\n" + "1\n" + "4\n" + "5\n" + "6\n" + "7\n";

    checkOutput(-1, singleFileName, correctContents);
  }
Exemplo n.º 4
0
  // @Ignore
  @Test
  public void testSingleFileCompletedWriteInitial() {
    prepareTest();
    populateFile(SINGLE_FILE, "0\n" + "1\n" + "2\n");
    SingleHDFSExactlyOnceWriter writer = new SingleHDFSExactlyOnceWriter();
    writer.setAppend(true);

    testSingleFileCompletedWriteHelper(writer, ProcessingMode.EXACTLY_ONCE);

    String singleFileName = testMeta.getDir() + File.separator + SINGLE_FILE;

    String correctContents = "0\n" + "1\n" + "2\n" + "0\n" + "1\n" + "2\n" + "3\n";

    checkOutput(-1, singleFileName, correctContents);
  }
Exemplo n.º 5
0
  // @Ignore
  @Test
  public void singleFileMultiRollingFailureOverwrite() {
    prepareTest();
    SingleHDFSExactlyOnceWriter writer = new SingleHDFSExactlyOnceWriter();
    File meta = new File(testMeta.getDir());
    writer.setFilePath(meta.getAbsolutePath());
    writer.setAppend(false);
    writer.setMaxLength(4);

    singleFileMultiRollingFailureHelper(writer, ProcessingMode.EXACTLY_ONCE);

    String singleFilePath = testMeta.getDir() + File.separator + SINGLE_FILE;

    // Rolling file 0

    String correctContents = "0\n" + "1\n" + "2\n";
    checkOutput(0, singleFilePath, correctContents);

    // Rolling file 1

    correctContents = "3\n" + "4\n" + "0\n";

    checkOutput(1, singleFilePath, correctContents);

    // Rolling file 2

    correctContents = "1\n" + "2\n" + "3\n";

    checkOutput(2, singleFilePath, correctContents);

    // Rolling file 2

    correctContents = "4\n" + "5\n";

    checkOutput(3, singleFilePath, correctContents);
  }
Exemplo n.º 6
0
  private void testSingleFileCompletedWriteHelper(
      SingleHDFSExactlyOnceWriter writer, ProcessingMode mode) {
    writer.setFilePath(testMeta.getDir());
    writer.setup(new DummyContext(0, mode));

    writer.beginWindow(0);
    writer.input.put(0);
    writer.input.put(1);
    writer.endWindow();

    writer.beginWindow(1);
    writer.input.put(2);
    writer.input.put(3);
    writer.endWindow();

    writer.teardown();
  }
Exemplo n.º 7
0
  private void testSingleFileFailedWriteHelper(
      SingleHDFSExactlyOnceWriter writer, ProcessingMode mode) {
    File meta = new File(testMeta.getDir());
    writer.setFilePath(meta.getAbsolutePath());
    writer.setup(new DummyContext(0, mode));

    writer.beginWindow(0);
    writer.input.put(0);
    writer.input.put(1);
    writer.endWindow();

    CheckPointWriter checkPointWriter = checkpoint(writer);

    writer.beginWindow(1);
    writer.input.put(2);

    writer.teardown();

    restoreCheckPoint(checkPointWriter, writer);
    writer.setup(new DummyContext(0, mode));

    writer.beginWindow(1);
    writer.input.put(4);
    writer.input.put(5);
    writer.endWindow();

    writer.beginWindow(2);
    writer.input.put(6);
    writer.input.put(7);
    writer.endWindow();

    writer.teardown();
  }
Exemplo n.º 8
0
  private void singleFileMultiRollingFailureHelper(
      SingleHDFSExactlyOnceWriter writer, ProcessingMode mode) {
    writer.setup(new DummyContext(0, mode));

    writer.beginWindow(0);
    writer.input.put(0);
    writer.input.put(1);
    writer.input.put(2);
    writer.endWindow();

    writer.beginWindow(1);
    writer.input.put(3);
    writer.input.put(4);

    CheckPointWriter checkPointWriter = checkpoint(writer);

    writer.input.put(3);
    writer.input.put(4);
    writer.input.put(5);
    writer.endWindow();

    writer.beginWindow(2);
    writer.input.put(6);
    writer.input.put(7);
    writer.input.put(8);
    writer.endWindow();
    writer.teardown();

    restoreCheckPoint(checkPointWriter, writer);
    writer.setup(new DummyContext(0, mode));

    writer.beginWindow(1);
    writer.input.put(0);
    writer.input.put(1);
    writer.input.put(2);
    writer.endWindow();

    writer.beginWindow(2);
    writer.input.put(3);
    writer.input.put(4);
    writer.input.put(5);
    writer.endWindow();
  }
Exemplo n.º 9
0
  // @Ignore
  @Test
  public void testSingleRollingFileFailedWrite1() {
    prepareTest();
    SingleHDFSExactlyOnceWriter writer = new SingleHDFSExactlyOnceWriter();
    writer.setAppend(true);
    writer.setFilePath(testMeta.getDir());
    writer.setMaxLength(4);

    writer.setup(new DummyContext(0, ProcessingMode.EXACTLY_ONCE));

    writer.beginWindow(0);
    writer.input.put(0);
    writer.input.put(1);
    writer.input.put(2);
    writer.endWindow();

    writer.beginWindow(1);
    writer.input.put(3);
    writer.input.put(4);
    writer.endWindow();

    CheckPointWriter checkPointWriter = checkpoint(writer);
    CheckPointWriter checkPointWriter1 = checkpoint(writer);

    LOG.debug("Checkpoint endOffsets={}", checkPointWriter.endOffsets);

    writer.beginWindow(2);
    writer.input.put(5);
    writer.teardown();

    restoreCheckPoint(checkPointWriter, writer);
    LOG.debug("Checkpoint endOffsets={}", checkPointWriter.endOffsets);
    writer.setup(new DummyContext(0, ProcessingMode.EXACTLY_ONCE));

    writer.beginWindow(2);
    writer.input.put(5);
    writer.endWindow();

    writer.beginWindow(3);
    writer.input.put(6);
    writer.input.put(7);
    writer.input.put(8);
    writer.endWindow();

    writer.teardown();

    restoreCheckPoint(checkPointWriter1, writer);
    writer.setup(new DummyContext(0, ProcessingMode.EXACTLY_ONCE));

    String singleFilePath = testMeta.getDir() + File.separator + SINGLE_FILE;

    // Rolling file 0

    String correctContents = "0\n" + "1\n" + "2\n";
    checkOutput(0, singleFilePath, correctContents);

    // Rolling file 1

    correctContents = "3\n" + "4\n";

    checkOutput(1, singleFilePath, correctContents);
  }
Exemplo n.º 10
0
  private void testSingleRollingFileFailedWriteOverwriteHelper(
      SingleHDFSExactlyOnceWriter writer, ProcessingMode mode) {
    writer.setMaxLength(4);
    writer.setFilePath(testMeta.getDir());
    writer.setup(new DummyContext(0, mode));

    writer.beginWindow(0);
    writer.input.put(0);
    writer.input.put(1);
    writer.input.put(2);
    writer.endWindow();

    writer.beginWindow(1);
    writer.input.put(3);
    writer.input.put(4);

    CheckPointWriter checkPointWriter = checkpoint(writer);

    writer.teardown();

    restoreCheckPoint(checkPointWriter, writer);
    writer.setup(new DummyContext(0, mode));

    writer.beginWindow(1);
    writer.input.put(3);
    writer.endWindow();

    writer.beginWindow(2);
    writer.input.put(6);
    writer.input.put(7);
    writer.input.put(8);
    writer.endWindow();

    writer.teardown();
  }