示例#1
0
 @Override
 public void returnData(OrcEncodedColumnBatch ecb) {
   for (ColumnStreamData[] datas : ecb.getColumnData()) {
     if (datas == null) continue;
     for (ColumnStreamData data : datas) {
       if (data == null || data.decRef() != 0) continue;
       if (DebugUtils.isTraceLockingEnabled()) {
         for (MemoryBuffer buf : data.getCacheBuffers()) {
           LlapIoImpl.LOG.info("Unlocking " + buf + " at the end of processing");
         }
       }
       lowLevelCache.releaseBuffers(data.getCacheBuffers());
       CSD_POOL.offer(data);
     }
   }
   // We can offer ECB even with some streams not discarded; reset() will clear the arrays.
   ECB_POOL.offer(ecb);
 }
示例#2
0
 @Override
 public void resetBeforeOffer(OrcEncodedColumnBatch t) {
   t.reset();
 }
示例#3
0
 /**
  * Takes the data from high-level cache for all stripes and returns to consumer.
  *
  * @return List of columns to read per stripe, if any columns were fully eliminated by cache.
  */
 private List<Integer>[] produceDataFromCache(int rowIndexStride) throws IOException {
   OrcCacheKey key = new OrcCacheKey(fileId, -1, -1, -1);
   // For each stripe, keep a list of columns that are not fully in cache (null => all of them).
   @SuppressWarnings("unchecked")
   List<Integer>[] stripeColsNotInCache = new List[readState.length];
   for (int stripeIxMod = 0; stripeIxMod < readState.length; ++stripeIxMod) {
     key.stripeIx = stripeIxFrom + stripeIxMod;
     boolean[][] cols = readState[stripeIxMod];
     boolean[] isMissingAnyRgs = new boolean[cols.length];
     int totalRgCount = getRgCount(fileMetadata.getStripes().get(key.stripeIx), rowIndexStride);
     for (int rgIx = 0; rgIx < totalRgCount; ++rgIx) {
       OrcEncodedColumnBatch col = ECB_POOL.take();
       col.init(fileId, key.stripeIx, rgIx, cols.length);
       boolean hasAnyCached = false;
       try {
         key.rgIx = rgIx;
         for (int colIxMod = 0; colIxMod < cols.length; ++colIxMod) {
           boolean[] readMask = cols[colIxMod];
           // Check if RG is eliminated by SARG
           if ((readMask == SargApplier.READ_NO_RGS)
               || (readMask != SargApplier.READ_ALL_RGS
                   && (readMask.length <= rgIx || !readMask[rgIx]))) continue;
           key.colIx = columnIds.get(colIxMod);
           ColumnStreamData[] cached = cache.get(key);
           if (cached == null) {
             isMissingAnyRgs[colIxMod] = true;
             continue;
           }
           assert cached.length == OrcEncodedColumnBatch.MAX_DATA_STREAMS;
           col.setAllStreamsData(colIxMod, key.colIx, cached);
           hasAnyCached = true;
           if (readMask == SargApplier.READ_ALL_RGS) {
             // We were going to read all RGs, but some were in cache, allocate the mask.
             cols[colIxMod] = readMask = new boolean[totalRgCount];
             Arrays.fill(readMask, true);
           }
           readMask[rgIx] = false; // Got from cache, don't read from disk.
         }
       } catch (Throwable t) {
         // TODO: Any cleanup needed to release data in col back to cache should be here.
         throw (t instanceof IOException) ? (IOException) t : new IOException(t);
       }
       if (hasAnyCached) {
         consumer.consumeData(col);
       }
     }
     boolean makeStripeColList = false; // By default assume we'll fetch all original columns.
     for (int colIxMod = 0; colIxMod < cols.length; ++colIxMod) {
       if (isMissingAnyRgs[colIxMod]) {
         if (makeStripeColList) {
           stripeColsNotInCache[stripeIxMod].add(columnIds.get(colIxMod));
         }
       } else if (!makeStripeColList) {
         // Some columns were fully in cache. Make a per-stripe col list, add previous columns.
         makeStripeColList = true;
         stripeColsNotInCache[stripeIxMod] = new ArrayList<Integer>(cols.length - 1);
         for (int i = 0; i < colIxMod; ++i) {
           stripeColsNotInCache[stripeIxMod].add(columnIds.get(i));
         }
       }
     }
   }
   return stripeColsNotInCache;
 }