// @Ignore
@Test
public void testSingleRollingFileFailedWrite() {
prepareTest();
SingleHDFSExactlyOnceWriter writer = new SingleHDFSExactlyOnceWriter();
writer.setAppend(true);
testSingleRollingFileFailedWriteHelper(writer, ProcessingMode.EXACTLY_ONCE);
// Rolling file 0
String singleFileName = testMeta.getDir() + File.separator + SINGLE_FILE;
String correctContents = "0\n" + "1\n" + "2\n";
checkOutput(0, singleFileName, correctContents);
// Rolling file 1
correctContents = "3\n" + "4\n" + "5\n";
checkOutput(1, singleFileName, correctContents);
// Rolling file 2
correctContents = "6\n" + "7\n" + "8\n";
checkOutput(2, singleFileName, correctContents);
}
// @Ignore
@Test
public void testSingleRollingFileCompletedWriteOverwriteInitial() {
prepareTest();
populateFile(SINGLE_FILE + ".0", "0\n" + "1\n" + "2\n");
populateFile(SINGLE_FILE + ".1", "0\n" + "1\n" + "2\n");
populateFile(SINGLE_FILE + ".2", "0\n" + "1\n" + "2\n");
SingleHDFSExactlyOnceWriter writer = new SingleHDFSExactlyOnceWriter();
writer.setAppend(false);
testSingleRollingFileCompletedWriteHelper(writer, ProcessingMode.EXACTLY_ONCE);
// Rolling file 0
String singleFileName = testMeta.getDir() + File.separator + SINGLE_FILE;
String correctContents = "0\n" + "1\n" + "2\n";
checkOutput(0, singleFileName, correctContents);
// Rolling file 1
correctContents = "3\n" + "4\n" + "5\n";
checkOutput(1, singleFileName, correctContents);
}
// @Ignore
@Test
public void testSingleFileFailedWriteOverwrite() {
prepareTest();
SingleHDFSExactlyOnceWriter writer = new SingleHDFSExactlyOnceWriter();
writer.setAppend(false);
testSingleFileFailedWriteHelper(writer, ProcessingMode.EXACTLY_ONCE);
String singleFileName = testMeta.getDir() + File.separator + SINGLE_FILE;
String correctContents = "0\n" + "1\n" + "4\n" + "5\n" + "6\n" + "7\n";
checkOutput(-1, singleFileName, correctContents);
}
// @Ignore
@Test
public void testSingleFileCompletedWriteInitial() {
prepareTest();
populateFile(SINGLE_FILE, "0\n" + "1\n" + "2\n");
SingleHDFSExactlyOnceWriter writer = new SingleHDFSExactlyOnceWriter();
writer.setAppend(true);
testSingleFileCompletedWriteHelper(writer, ProcessingMode.EXACTLY_ONCE);
String singleFileName = testMeta.getDir() + File.separator + SINGLE_FILE;
String correctContents = "0\n" + "1\n" + "2\n" + "0\n" + "1\n" + "2\n" + "3\n";
checkOutput(-1, singleFileName, correctContents);
}
// @Ignore
@Test
public void singleFileMultiRollingFailureOverwrite() {
prepareTest();
SingleHDFSExactlyOnceWriter writer = new SingleHDFSExactlyOnceWriter();
File meta = new File(testMeta.getDir());
writer.setFilePath(meta.getAbsolutePath());
writer.setAppend(false);
writer.setMaxLength(4);
singleFileMultiRollingFailureHelper(writer, ProcessingMode.EXACTLY_ONCE);
String singleFilePath = testMeta.getDir() + File.separator + SINGLE_FILE;
// Rolling file 0
String correctContents = "0\n" + "1\n" + "2\n";
checkOutput(0, singleFilePath, correctContents);
// Rolling file 1
correctContents = "3\n" + "4\n" + "0\n";
checkOutput(1, singleFilePath, correctContents);
// Rolling file 2
correctContents = "1\n" + "2\n" + "3\n";
checkOutput(2, singleFilePath, correctContents);
// Rolling file 2
correctContents = "4\n" + "5\n";
checkOutput(3, singleFilePath, correctContents);
}
private void testSingleFileCompletedWriteHelper(
SingleHDFSExactlyOnceWriter writer, ProcessingMode mode) {
writer.setFilePath(testMeta.getDir());
writer.setup(new DummyContext(0, mode));
writer.beginWindow(0);
writer.input.put(0);
writer.input.put(1);
writer.endWindow();
writer.beginWindow(1);
writer.input.put(2);
writer.input.put(3);
writer.endWindow();
writer.teardown();
}
private void testSingleFileFailedWriteHelper(
SingleHDFSExactlyOnceWriter writer, ProcessingMode mode) {
File meta = new File(testMeta.getDir());
writer.setFilePath(meta.getAbsolutePath());
writer.setup(new DummyContext(0, mode));
writer.beginWindow(0);
writer.input.put(0);
writer.input.put(1);
writer.endWindow();
CheckPointWriter checkPointWriter = checkpoint(writer);
writer.beginWindow(1);
writer.input.put(2);
writer.teardown();
restoreCheckPoint(checkPointWriter, writer);
writer.setup(new DummyContext(0, mode));
writer.beginWindow(1);
writer.input.put(4);
writer.input.put(5);
writer.endWindow();
writer.beginWindow(2);
writer.input.put(6);
writer.input.put(7);
writer.endWindow();
writer.teardown();
}
private void singleFileMultiRollingFailureHelper(
SingleHDFSExactlyOnceWriter writer, ProcessingMode mode) {
writer.setup(new DummyContext(0, mode));
writer.beginWindow(0);
writer.input.put(0);
writer.input.put(1);
writer.input.put(2);
writer.endWindow();
writer.beginWindow(1);
writer.input.put(3);
writer.input.put(4);
CheckPointWriter checkPointWriter = checkpoint(writer);
writer.input.put(3);
writer.input.put(4);
writer.input.put(5);
writer.endWindow();
writer.beginWindow(2);
writer.input.put(6);
writer.input.put(7);
writer.input.put(8);
writer.endWindow();
writer.teardown();
restoreCheckPoint(checkPointWriter, writer);
writer.setup(new DummyContext(0, mode));
writer.beginWindow(1);
writer.input.put(0);
writer.input.put(1);
writer.input.put(2);
writer.endWindow();
writer.beginWindow(2);
writer.input.put(3);
writer.input.put(4);
writer.input.put(5);
writer.endWindow();
}
// @Ignore
@Test
public void testSingleRollingFileFailedWrite1() {
prepareTest();
SingleHDFSExactlyOnceWriter writer = new SingleHDFSExactlyOnceWriter();
writer.setAppend(true);
writer.setFilePath(testMeta.getDir());
writer.setMaxLength(4);
writer.setup(new DummyContext(0, ProcessingMode.EXACTLY_ONCE));
writer.beginWindow(0);
writer.input.put(0);
writer.input.put(1);
writer.input.put(2);
writer.endWindow();
writer.beginWindow(1);
writer.input.put(3);
writer.input.put(4);
writer.endWindow();
CheckPointWriter checkPointWriter = checkpoint(writer);
CheckPointWriter checkPointWriter1 = checkpoint(writer);
LOG.debug("Checkpoint endOffsets={}", checkPointWriter.endOffsets);
writer.beginWindow(2);
writer.input.put(5);
writer.teardown();
restoreCheckPoint(checkPointWriter, writer);
LOG.debug("Checkpoint endOffsets={}", checkPointWriter.endOffsets);
writer.setup(new DummyContext(0, ProcessingMode.EXACTLY_ONCE));
writer.beginWindow(2);
writer.input.put(5);
writer.endWindow();
writer.beginWindow(3);
writer.input.put(6);
writer.input.put(7);
writer.input.put(8);
writer.endWindow();
writer.teardown();
restoreCheckPoint(checkPointWriter1, writer);
writer.setup(new DummyContext(0, ProcessingMode.EXACTLY_ONCE));
String singleFilePath = testMeta.getDir() + File.separator + SINGLE_FILE;
// Rolling file 0
String correctContents = "0\n" + "1\n" + "2\n";
checkOutput(0, singleFilePath, correctContents);
// Rolling file 1
correctContents = "3\n" + "4\n";
checkOutput(1, singleFilePath, correctContents);
}
private void testSingleRollingFileFailedWriteOverwriteHelper(
SingleHDFSExactlyOnceWriter writer, ProcessingMode mode) {
writer.setMaxLength(4);
writer.setFilePath(testMeta.getDir());
writer.setup(new DummyContext(0, mode));
writer.beginWindow(0);
writer.input.put(0);
writer.input.put(1);
writer.input.put(2);
writer.endWindow();
writer.beginWindow(1);
writer.input.put(3);
writer.input.put(4);
CheckPointWriter checkPointWriter = checkpoint(writer);
writer.teardown();
restoreCheckPoint(checkPointWriter, writer);
writer.setup(new DummyContext(0, mode));
writer.beginWindow(1);
writer.input.put(3);
writer.endWindow();
writer.beginWindow(2);
writer.input.put(6);
writer.input.put(7);
writer.input.put(8);
writer.endWindow();
writer.teardown();
}