@Test
public void testEmptyFileIndexLessThanOtherFileBadNewFile()
throws IOException, InterruptedException {
writeFile(new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0));
fileSystem.createNewFile(new Path(shardPath.getParent(), "data-002.json"));
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue());
writeFile(new Path(shardPath.getParent(), "data-003.json"), ImmutableList.of(RECORD_1));
expectedException.expect(IllegalStateException.class);
recordReader.nextKeyValue();
}
@Test
public void testEmptyFileThenDataFile() throws IOException, InterruptedException {
checkNextKeyValueWouldBlock();
fileSystem.createNewFile(new Path(shardPath.getParent(), "data-001.json"));
checkNextKeyValueWouldBlock();
writeFile(new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0));
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue());
assertFalse(recordReader.nextKeyValue());
}
@Test
public void testBadFilename() throws IOException, InterruptedException {
String outOfBounds = String.format("data-%d.json", 1L + Integer.MAX_VALUE);
fileSystem.createNewFile(new Path(shardPath.getParent(), outOfBounds));
expectedException.expect(IndexOutOfBoundsException.class);
recordReader.nextKeyValue();
}
@Test
public void testEmptyFileIsOnlyFileAndZeroIndex() throws IOException, InterruptedException {
checkNextKeyValueWouldBlock();
fileSystem.createNewFile(new Path(shardPath.getParent(), "data-000.json"));
assertFalse(recordReader.nextKeyValue());
assertNull(recordReader.getCurrentKey());
assertNull(recordReader.getCurrentValue());
}
/**
* Since we set up the mockSleeper to throw an exception with a test-idenfiable string, this
* helper method checks that invoking nextKeyValue would've blocked but threw the fake exception
* instead.
*/
private void checkNextKeyValueWouldBlock() throws IOException, InterruptedException {
try {
recordReader.nextKeyValue();
fail("nextKeyValue should have thrown");
} catch (RuntimeException re) {
assertTrue(re.getMessage().contains("test-sleep-id-12345"));
}
}
@Test
public void testMultipleDataFilesInSingleList() throws IOException, InterruptedException {
writeFile(new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0));
writeFile(
new Path(shardPath.getParent(), "data-001.json"), ImmutableList.of(RECORD_1, RECORD_2));
fileSystem.createNewFile(new Path(shardPath.getParent(), "data-002.json"));
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue());
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_1), recordReader.getCurrentValue());
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(RECORD_1.length() + 1), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_2), recordReader.getCurrentValue());
assertFalse(recordReader.nextKeyValue());
}
@Test
public void testEmptyFileIsOnlyFileAndNotZeroIndex() throws IOException, InterruptedException {
fileSystem.createNewFile(new Path(shardPath.getParent(), "data-001.json"));
checkNextKeyValueWouldBlock();
fileSystem.createNewFile(new Path(shardPath.getParent(), "data-002.json"));
// Second file-marker with different index causes IllegalStateException.
expectedException.expect(IllegalStateException.class);
recordReader.nextKeyValue();
}
@Test
public void testEmptyFileIndexLessThanOtherFileBadKnownFile()
throws IOException, InterruptedException {
writeFile(new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0));
writeFile(new Path(shardPath.getParent(), "data-002.json"), ImmutableList.of(RECORD_1));
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue());
fileSystem.createNewFile(new Path(shardPath.getParent(), "data-001.json"));
// We will successfully read the remaining available file before discovering the bad one.
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_1), recordReader.getCurrentValue());
expectedException.expect(IllegalStateException.class);
recordReader.nextKeyValue();
}
@Test
public void testCloseBeforeEnd() throws IOException, InterruptedException {
writeFile(
new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0, RECORD_1));
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue());
recordReader.close();
}