protected void splitFile(int numSplits) throws Exception {
InputSplit[] inputSplit = databusInputFormat.getSplits(defaultConf, numSplits);
LOG.info("number of splits : " + inputSplit.length);
for (InputSplit split : inputSplit) {
readMessages.addAll(readSplit(databusInputFormat, split, defaultConf, reporter));
}
}
/**
* read the the given input split.
*
* @return List : List of read messages
*/
private List<Message> readSplit(
DatabusInputFormat format, InputSplit split, JobConf job, Reporter reporter)
throws IOException {
List<Message> result = new ArrayList<Message>();
RecordReader<LongWritable, Message> reader = format.getRecordReader(split, job, reporter);
LongWritable key = ((DatabusRecordReader) reader).createKey();
Message value = ((DatabusRecordReader) reader).createValue();
while (((DatabusRecordReader) reader).next(key, value)) {
result.add(value);
value = (Message) ((DatabusRecordReader) reader).createValue();
}
reader.close();
return result;
}
