@Test
public void testMultipleFilesThenHangBeforeEmptyFileAppears()
throws IOException, InterruptedException {
writeFile(new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0));
writeFile(new Path(shardPath.getParent(), "data-001.json"), ImmutableList.of(RECORD_1));
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue());
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_1), recordReader.getCurrentValue());
checkNextKeyValueWouldBlock();
fileSystem.createNewFile(new Path(shardPath.getParent(), "data-002.json"));
assertFalse(recordReader.nextKeyValue());
}
@Test
public void testEmptyFileIsOnlyFileAndZeroIndex() throws IOException, InterruptedException {
checkNextKeyValueWouldBlock();
fileSystem.createNewFile(new Path(shardPath.getParent(), "data-000.json"));
assertFalse(recordReader.nextKeyValue());
assertNull(recordReader.getCurrentKey());
assertNull(recordReader.getCurrentValue());
}
@Test
public void testMultipleDataFilesInSingleList() throws IOException, InterruptedException {
writeFile(new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0));
writeFile(
new Path(shardPath.getParent(), "data-001.json"), ImmutableList.of(RECORD_1, RECORD_2));
fileSystem.createNewFile(new Path(shardPath.getParent(), "data-002.json"));
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue());
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_1), recordReader.getCurrentValue());
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(RECORD_1.length() + 1), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_2), recordReader.getCurrentValue());
assertFalse(recordReader.nextKeyValue());
}
@Test
public void testEmptyFileIndexLessThanOtherFileBadKnownFile()
throws IOException, InterruptedException {
writeFile(new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0));
writeFile(new Path(shardPath.getParent(), "data-002.json"), ImmutableList.of(RECORD_1));
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue());
fileSystem.createNewFile(new Path(shardPath.getParent(), "data-001.json"));
// We will successfully read the remaining available file before discovering the bad one.
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_1), recordReader.getCurrentValue());
expectedException.expect(IllegalStateException.class);
recordReader.nextKeyValue();
}
@Test
public void testCloseBeforeEnd() throws IOException, InterruptedException {
writeFile(
new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0, RECORD_1));
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue());
recordReader.close();
}
@Test
public void testThreeBatchesEndFileInMiddleBatch() throws IOException, InterruptedException {
writeFile(new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0));
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue());
checkNextKeyValueWouldBlock();
writeFile(new Path(shardPath.getParent(), "data-001.json"), ImmutableList.of(RECORD_1));
fileSystem.createNewFile(new Path(shardPath.getParent(), "data-003.json"));
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_1), recordReader.getCurrentValue());
checkNextKeyValueWouldBlock();
checkNextKeyValueWouldBlock();
writeFile(new Path(shardPath.getParent(), "data-002.json"), ImmutableList.of(RECORD_2));
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_2), recordReader.getCurrentValue());
assertFalse(recordReader.nextKeyValue());
}
@Test
public void testEmptyFileIndexLessThanOtherFileBadNewFile()
throws IOException, InterruptedException {
writeFile(new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0));
fileSystem.createNewFile(new Path(shardPath.getParent(), "data-002.json"));
assertTrue(recordReader.nextKeyValue());
assertEquals(new LongWritable(0), recordReader.getCurrentKey());
assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue());
writeFile(new Path(shardPath.getParent(), "data-003.json"), ImmutableList.of(RECORD_1));
expectedException.expect(IllegalStateException.class);
recordReader.nextKeyValue();
}
@Test
public void testGetCurrentBeforeFirstRecord() throws IOException {
assertNull(recordReader.getCurrentKey());
assertNull(recordReader.getCurrentValue());
assertEquals(0.0f, recordReader.getProgress(), 0.0f);
}
