/**
* Generates the next record batch
*
* @return number of records in the batch
*/
@Override
public int next() {
reader.resetForNextBatch();
int cnt = 0;
try {
while (cnt < MAX_RECORDS_PER_BATCH && reader.parseNext()) {
cnt++;
}
reader.finishBatch();
return cnt;
} catch (IOException | TextParsingException e) {
throw UserException.dataReadError(e)
.addContext(
"Failure while reading file %s. Happened at or shortly before byte position %d.",
split.getPath(), reader.getPos())
.build(logger);
}
}
/**
* Performs the initial setup required for the record reader. Initializes the input stream,
* handling of the output record batch and the actual reader to be used.
*
* @param context operator context from which buffer's will be allocated and managed
* @param outputMutator Used to create the schema in the output record batch
* @throws ExecutionSetupException
*/
@Override
public void setup(OperatorContext context, OutputMutator outputMutator)
throws ExecutionSetupException {
oContext = context;
readBuffer = context.getManagedBuffer(READ_BUFFER);
whitespaceBuffer = context.getManagedBuffer(WHITE_SPACE_BUFFER);
// setup Output, Input, and Reader
try {
TextOutput output = null;
TextInput input = null;
InputStream stream = null;
// setup Output using OutputMutator
if (settings.isHeaderExtractionEnabled()) {
// extract header and use that to setup a set of VarCharVectors
String[] fieldNames = extractHeader();
output = new FieldVarCharOutput(outputMutator, fieldNames, getColumns(), isStarQuery());
} else {
// simply use RepeatedVarCharVector
output = new RepeatedVarCharOutput(outputMutator, getColumns(), isStarQuery());
}
// setup Input using InputStream
stream = dfs.openPossiblyCompressedStream(split.getPath());
input =
new TextInput(
settings, stream, readBuffer, split.getStart(), split.getStart() + split.getLength());
// setup Reader using Input and Output
reader = new TextReader(settings, input, output, whitespaceBuffer);
reader.start();
} catch (SchemaChangeException | IOException e) {
throw new ExecutionSetupException(
String.format("Failure while setting up text reader for file %s", split.getPath()), e);
} catch (IllegalArgumentException e) {
throw UserException.dataReadError(e)
.addContext("File Path", split.getPath().toString())
.build(logger);
}
}
