private void testMultiFileFailedWriteHelper(
EvenOddHDFSExactlyOnceWriter writer, ProcessingMode mode) {
File meta = new File(testMeta.getDir());
writer.setFilePath(meta.getAbsolutePath());
writer.setup(new DummyContext(0, mode));
writer.beginWindow(0);
writer.input.put(0);
writer.input.put(1);
writer.input.put(2);
writer.input.put(3);
writer.endWindow();
CheckPointWriter checkPointWriter = checkpoint(writer);
writer.beginWindow(1);
writer.input.put(4);
writer.input.put(5);
writer.endWindow();
writer.teardown();
restoreCheckPoint(checkPointWriter, writer);
writer.setup(new DummyContext(0, mode));
writer.beginWindow(2);
writer.input.put(6);
writer.input.put(7);
writer.input.put(8);
writer.input.put(9);
writer.endWindow();
}
// @Ignore
@Test
public void testMultiRollingFileFailedWriteOverwriteCache1() {
prepareTest();
EvenOddHDFSExactlyOnceWriter writer = new EvenOddHDFSExactlyOnceWriter();
writer.setAppend(false);
writer.setMaxOpenFiles(1);
testMultiRollingFileFailedWriteOverwriteHelperCache1(writer, ProcessingMode.EXACTLY_ONCE);
// Even file
String evenFileName = testMeta.getDir() + File.separator + EVEN_FILE;
String correctContents = "0\n" + "4\n" + "6\n";
checkOutput(0, evenFileName, correctContents);
correctContents = "8\n" + "6\n";
checkOutput(1, evenFileName, correctContents);
// Odd file
String oddFileName = testMeta.getDir() + File.separator + ODD_FILE;
correctContents = "1\n" + "5\n" + "7\n";
checkOutput(0, oddFileName, correctContents);
correctContents = "9\n" + "7\n";
checkOutput(1, oddFileName, correctContents);
}
// @Ignore
@Test
public void testMultiRollingFileFailedWriteOverwrite() {
prepareTest();
EvenOddHDFSExactlyOnceWriter writer = new EvenOddHDFSExactlyOnceWriter();
writer.setAppend(false);
testMultiRollingFileFailedWriteOverwriteHelper(writer, ProcessingMode.EXACTLY_ONCE);
}
// @Ignore
@Test
public void testMultiRollingFileFailedWriteCache1() {
prepareTest();
EvenOddHDFSExactlyOnceWriter writer = new EvenOddHDFSExactlyOnceWriter();
writer.setAppend(true);
writer.setMaxOpenFiles(1);
testMultiRollingFileFailedWriteHelperHelper(writer, ProcessingMode.EXACTLY_ONCE);
}
// @Ignore
@Test
public void testMultiFileFailedWriteOverwrite() {
prepareTest();
EvenOddHDFSExactlyOnceWriter writer = new EvenOddHDFSExactlyOnceWriter();
writer.setAppend(false);
testMultiFileFailedWriteHelper(writer, ProcessingMode.EXACTLY_ONCE);
String evenFileName = testMeta.getDir() + File.separator + EVEN_FILE;
String correctContents = "0\n" + "2\n" + "6\n" + "8\n";
checkOutput(-1, evenFileName, correctContents);
String oddFileName = testMeta.getDir() + File.separator + ODD_FILE;
correctContents = "1\n" + "3\n" + "7\n" + "9\n";
checkOutput(-1, oddFileName, correctContents);
}
// @Ignore
@Test
public void testMultiFileCompletedWriteInitial() {
prepareTest();
populateFile(EVEN_FILE, "0\n" + "2\n");
populateFile(ODD_FILE, "1\n" + "3\n");
EvenOddHDFSExactlyOnceWriter writer = new EvenOddHDFSExactlyOnceWriter();
writer.setAppend(true);
testMultiFileCompletedWriteHelper(writer, ProcessingMode.EXACTLY_ONCE);
String evenFileName = testMeta.getDir() + File.separator + EVEN_FILE;
String correctContents = "0\n" + "2\n" + "0\n" + "2\n" + "4\n" + "6\n";
checkOutput(-1, evenFileName, correctContents);
String oddFileName = testMeta.getDir() + File.separator + ODD_FILE;
correctContents = "1\n" + "3\n" + "1\n" + "3\n" + "5\n" + "7\n";
checkOutput(-1, oddFileName, correctContents);
}
private void testMultiRollingFileCompletedWriteHelper(
EvenOddHDFSExactlyOnceWriter writer, ProcessingMode mode) {
writer.setMaxLength(4);
File meta = new File(testMeta.getDir());
writer.setFilePath(meta.getAbsolutePath());
writer.setup(new DummyContext(0, mode));
writer.beginWindow(0);
writer.input.put(0);
writer.input.put(1);
writer.input.put(2);
writer.input.put(3);
writer.input.put(4);
writer.input.put(5);
writer.endWindow();
writer.beginWindow(1);
writer.input.put(6);
writer.input.put(7);
writer.input.put(8);
writer.input.put(9);
writer.input.put(6);
writer.input.put(7);
writer.endWindow();
// Even file
String evenFileName = testMeta.getDir() + File.separator + EVEN_FILE;
String correctContents = "0\n" + "2\n" + "4\n";
checkOutput(0, evenFileName, correctContents);
correctContents = "6\n" + "8\n" + "6\n";
checkOutput(1, evenFileName, correctContents);
// Odd file
String oddFileName = testMeta.getDir() + File.separator + ODD_FILE;
correctContents = "1\n" + "3\n" + "5\n";
checkOutput(0, oddFileName, correctContents);
correctContents = "7\n" + "9\n" + "7\n";
checkOutput(1, oddFileName, correctContents);
}
private void testMultiFileCompletedWriteHelper(
EvenOddHDFSExactlyOnceWriter writer, ProcessingMode mode) {
File meta = new File(testMeta.getDir());
writer.setFilePath(meta.getAbsolutePath());
writer.setup(new DummyContext(0, mode));
writer.beginWindow(0);
writer.input.put(0);
writer.input.put(1);
writer.input.put(2);
writer.input.put(3);
writer.endWindow();
writer.beginWindow(1);
writer.input.put(4);
writer.input.put(5);
writer.input.put(6);
writer.input.put(7);
writer.endWindow();
}
