private void checkRead() throws IOException {
if (current == totalCountLoadedSoFar) {
if (current != 0) {
long timeAssembling = System.currentTimeMillis() - startedAssemblingCurrentBlockAt;
totalTimeSpentProcessingRecords += timeAssembling;
LOG.info(
"Assembled and processed "
+ totalCountLoadedSoFar
+ " records from "
+ columnCount
+ " columns in "
+ totalTimeSpentProcessingRecords
+ " ms: "
+ ((float) totalCountLoadedSoFar / totalTimeSpentProcessingRecords)
+ " rec/ms, "
+ ((float) totalCountLoadedSoFar * columnCount / totalTimeSpentProcessingRecords)
+ " cell/ms");
long totalTime = totalTimeSpentProcessingRecords + totalTimeSpentReadingBytes;
long percentReading = 100 * totalTimeSpentReadingBytes / totalTime;
long percentProcessing = 100 * totalTimeSpentProcessingRecords / totalTime;
LOG.info(
"time spent so far "
+ percentReading
+ "% reading ("
+ totalTimeSpentReadingBytes
+ " ms) and "
+ percentProcessing
+ "% processing ("
+ totalTimeSpentProcessingRecords
+ " ms)");
}
LOG.info("at row " + current + ". reading next block");
long t0 = System.currentTimeMillis();
PageReadStore pages = reader.readNextRowGroup();
if (pages == null) {
throw new IOException(
"expecting more rows but reached last block. Read " + current + " out of " + total);
}
long timeSpentReading = System.currentTimeMillis() - t0;
totalTimeSpentReadingBytes += timeSpentReading;
LOG.info(
"block read in memory in "
+ timeSpentReading
+ " ms. row count = "
+ pages.getRowCount());
if (Log.DEBUG)
LOG.debug("initializing Record assembly with requested schema " + requestedSchema);
MessageColumnIO columnIO = columnIOFactory.getColumnIO(requestedSchema);
recordReader = columnIO.getRecordReader(pages, recordConverter);
startedAssemblingCurrentBlockAt = System.currentTimeMillis();
totalCountLoadedSoFar += pages.getRowCount();
}
}
/** {@inheritDoc} */
@Override
public void close() throws IOException {
reader.close();
}