@Test public void testMultipleFilesThenHangBeforeEmptyFileAppears() throws IOException, InterruptedException { writeFile(new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0)); writeFile(new Path(shardPath.getParent(), "data-001.json"), ImmutableList.of(RECORD_1)); assertTrue(recordReader.nextKeyValue()); assertEquals(new LongWritable(0), recordReader.getCurrentKey()); assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue()); assertTrue(recordReader.nextKeyValue()); assertEquals(new LongWritable(0), recordReader.getCurrentKey()); assertEquals(jsonParser.parse(RECORD_1), recordReader.getCurrentValue()); checkNextKeyValueWouldBlock(); fileSystem.createNewFile(new Path(shardPath.getParent(), "data-002.json")); assertFalse(recordReader.nextKeyValue()); }
@Test public void testEmptyFileIsOnlyFileAndZeroIndex() throws IOException, InterruptedException { checkNextKeyValueWouldBlock(); fileSystem.createNewFile(new Path(shardPath.getParent(), "data-000.json")); assertFalse(recordReader.nextKeyValue()); assertNull(recordReader.getCurrentKey()); assertNull(recordReader.getCurrentValue()); }
@Test public void testMultipleDataFilesInSingleList() throws IOException, InterruptedException { writeFile(new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0)); writeFile( new Path(shardPath.getParent(), "data-001.json"), ImmutableList.of(RECORD_1, RECORD_2)); fileSystem.createNewFile(new Path(shardPath.getParent(), "data-002.json")); assertTrue(recordReader.nextKeyValue()); assertEquals(new LongWritable(0), recordReader.getCurrentKey()); assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue()); assertTrue(recordReader.nextKeyValue()); assertEquals(new LongWritable(0), recordReader.getCurrentKey()); assertEquals(jsonParser.parse(RECORD_1), recordReader.getCurrentValue()); assertTrue(recordReader.nextKeyValue()); assertEquals(new LongWritable(RECORD_1.length() + 1), recordReader.getCurrentKey()); assertEquals(jsonParser.parse(RECORD_2), recordReader.getCurrentValue()); assertFalse(recordReader.nextKeyValue()); }
@Test public void testEmptyFileIndexLessThanOtherFileBadKnownFile() throws IOException, InterruptedException { writeFile(new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0)); writeFile(new Path(shardPath.getParent(), "data-002.json"), ImmutableList.of(RECORD_1)); assertTrue(recordReader.nextKeyValue()); assertEquals(new LongWritable(0), recordReader.getCurrentKey()); assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue()); fileSystem.createNewFile(new Path(shardPath.getParent(), "data-001.json")); // We will successfully read the remaining available file before discovering the bad one. assertTrue(recordReader.nextKeyValue()); assertEquals(new LongWritable(0), recordReader.getCurrentKey()); assertEquals(jsonParser.parse(RECORD_1), recordReader.getCurrentValue()); expectedException.expect(IllegalStateException.class); recordReader.nextKeyValue(); }
@Test public void testCloseBeforeEnd() throws IOException, InterruptedException { writeFile( new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0, RECORD_1)); assertTrue(recordReader.nextKeyValue()); assertEquals(new LongWritable(0), recordReader.getCurrentKey()); assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue()); recordReader.close(); }
@Test public void testThreeBatchesEndFileInMiddleBatch() throws IOException, InterruptedException { writeFile(new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0)); assertTrue(recordReader.nextKeyValue()); assertEquals(new LongWritable(0), recordReader.getCurrentKey()); assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue()); checkNextKeyValueWouldBlock(); writeFile(new Path(shardPath.getParent(), "data-001.json"), ImmutableList.of(RECORD_1)); fileSystem.createNewFile(new Path(shardPath.getParent(), "data-003.json")); assertTrue(recordReader.nextKeyValue()); assertEquals(new LongWritable(0), recordReader.getCurrentKey()); assertEquals(jsonParser.parse(RECORD_1), recordReader.getCurrentValue()); checkNextKeyValueWouldBlock(); checkNextKeyValueWouldBlock(); writeFile(new Path(shardPath.getParent(), "data-002.json"), ImmutableList.of(RECORD_2)); assertTrue(recordReader.nextKeyValue()); assertEquals(new LongWritable(0), recordReader.getCurrentKey()); assertEquals(jsonParser.parse(RECORD_2), recordReader.getCurrentValue()); assertFalse(recordReader.nextKeyValue()); }
@Test public void testEmptyFileIndexLessThanOtherFileBadNewFile() throws IOException, InterruptedException { writeFile(new Path(shardPath.getParent(), "data-000.json"), ImmutableList.of(RECORD_0)); fileSystem.createNewFile(new Path(shardPath.getParent(), "data-002.json")); assertTrue(recordReader.nextKeyValue()); assertEquals(new LongWritable(0), recordReader.getCurrentKey()); assertEquals(jsonParser.parse(RECORD_0), recordReader.getCurrentValue()); writeFile(new Path(shardPath.getParent(), "data-003.json"), ImmutableList.of(RECORD_1)); expectedException.expect(IllegalStateException.class); recordReader.nextKeyValue(); }
@Test public void testGetCurrentBeforeFirstRecord() throws IOException { assertNull(recordReader.getCurrentKey()); assertNull(recordReader.getCurrentValue()); assertEquals(0.0f, recordReader.getProgress(), 0.0f); }