@Override
public void returnData(OrcEncodedColumnBatch ecb) {
for (ColumnStreamData[] datas : ecb.getColumnData()) {
if (datas == null) continue;
for (ColumnStreamData data : datas) {
if (data == null || data.decRef() != 0) continue;
if (DebugUtils.isTraceLockingEnabled()) {
for (MemoryBuffer buf : data.getCacheBuffers()) {
LlapIoImpl.LOG.info("Unlocking " + buf + " at the end of processing");
}
}
lowLevelCache.releaseBuffers(data.getCacheBuffers());
CSD_POOL.offer(data);
}
}
// We can offer ECB even with some streams not discarded; reset() will clear the arrays.
ECB_POOL.offer(ecb);
}
@Override
public void resetBeforeOffer(OrcEncodedColumnBatch t) {
t.reset();
}
/**
* Takes the data from high-level cache for all stripes and returns to consumer.
*
* @return List of columns to read per stripe, if any columns were fully eliminated by cache.
*/
private List<Integer>[] produceDataFromCache(int rowIndexStride) throws IOException {
OrcCacheKey key = new OrcCacheKey(fileId, -1, -1, -1);
// For each stripe, keep a list of columns that are not fully in cache (null => all of them).
@SuppressWarnings("unchecked")
List<Integer>[] stripeColsNotInCache = new List[readState.length];
for (int stripeIxMod = 0; stripeIxMod < readState.length; ++stripeIxMod) {
key.stripeIx = stripeIxFrom + stripeIxMod;
boolean[][] cols = readState[stripeIxMod];
boolean[] isMissingAnyRgs = new boolean[cols.length];
int totalRgCount = getRgCount(fileMetadata.getStripes().get(key.stripeIx), rowIndexStride);
for (int rgIx = 0; rgIx < totalRgCount; ++rgIx) {
OrcEncodedColumnBatch col = ECB_POOL.take();
col.init(fileId, key.stripeIx, rgIx, cols.length);
boolean hasAnyCached = false;
try {
key.rgIx = rgIx;
for (int colIxMod = 0; colIxMod < cols.length; ++colIxMod) {
boolean[] readMask = cols[colIxMod];
// Check if RG is eliminated by SARG
if ((readMask == SargApplier.READ_NO_RGS)
|| (readMask != SargApplier.READ_ALL_RGS
&& (readMask.length <= rgIx || !readMask[rgIx]))) continue;
key.colIx = columnIds.get(colIxMod);
ColumnStreamData[] cached = cache.get(key);
if (cached == null) {
isMissingAnyRgs[colIxMod] = true;
continue;
}
assert cached.length == OrcEncodedColumnBatch.MAX_DATA_STREAMS;
col.setAllStreamsData(colIxMod, key.colIx, cached);
hasAnyCached = true;
if (readMask == SargApplier.READ_ALL_RGS) {
// We were going to read all RGs, but some were in cache, allocate the mask.
cols[colIxMod] = readMask = new boolean[totalRgCount];
Arrays.fill(readMask, true);
}
readMask[rgIx] = false; // Got from cache, don't read from disk.
}
} catch (Throwable t) {
// TODO: Any cleanup needed to release data in col back to cache should be here.
throw (t instanceof IOException) ? (IOException) t : new IOException(t);
}
if (hasAnyCached) {
consumer.consumeData(col);
}
}
boolean makeStripeColList = false; // By default assume we'll fetch all original columns.
for (int colIxMod = 0; colIxMod < cols.length; ++colIxMod) {
if (isMissingAnyRgs[colIxMod]) {
if (makeStripeColList) {
stripeColsNotInCache[stripeIxMod].add(columnIds.get(colIxMod));
}
} else if (!makeStripeColList) {
// Some columns were fully in cache. Make a per-stripe col list, add previous columns.
makeStripeColList = true;
stripeColsNotInCache[stripeIxMod] = new ArrayList<Integer>(cols.length - 1);
for (int i = 0; i < colIxMod; ++i) {
stripeColsNotInCache[stripeIxMod].add(columnIds.get(i));
}
}
}
}
return stripeColsNotInCache;
}