@Override protected void doSetNextReader(AtomicReaderContext context) throws IOException { if (current != null) { missing += current.counts[0]; total += current.total - current.counts[0]; if (current.values.ordinals().getNumOrds() > 1) { aggregators.add(current); } } values = indexFieldData.load(context).getBytesValues(); current = new ReaderAggregator(values); }
private TermIterator( IndexFieldData.WithOrdinals indexFieldData, List<AtomicReaderContext> leaves, AtomicFieldData.WithOrdinals[] withOrdinals) throws IOException { this.sources = new LeafSourceQueue(leaves.size()); for (int i = 0; i < leaves.size(); i++) { AtomicReaderContext atomicReaderContext = leaves.get(i); AtomicFieldData.WithOrdinals afd = indexFieldData.load(atomicReaderContext); withOrdinals[i] = afd; LeafSource leafSource = new LeafSource(afd, atomicReaderContext); if (leafSource.current != null) { sources.add(leafSource); } } }
@Override public IndexFieldData.WithOrdinals build( final IndexReader indexReader, IndexFieldData.WithOrdinals indexFieldData, Settings settings, CircuitBreakerService breakerService) throws IOException { assert indexReader.leaves().size() > 1; long startTime = System.currentTimeMillis(); // It makes sense to make the overhead ratio configurable for the mapping from segment ords to // global ords // However, other mappings are never the bottleneck and only used to get the original value from // an ord, so // it makes sense to force COMPACT for them final float acceptableOverheadRatio = settings.getAsFloat("acceptable_overhead_ratio", PackedInts.FAST); final AppendingPackedLongBuffer globalOrdToFirstSegment = new AppendingPackedLongBuffer(PackedInts.COMPACT); final MonotonicAppendingLongBuffer globalOrdToFirstSegmentDelta = new MonotonicAppendingLongBuffer(PackedInts.COMPACT); FieldDataType fieldDataType = indexFieldData.getFieldDataType(); int defaultThreshold = settings.getAsInt( ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY, ORDINAL_MAPPING_THRESHOLD_DEFAULT); int threshold = fieldDataType.getSettings().getAsInt(ORDINAL_MAPPING_THRESHOLD_KEY, defaultThreshold); OrdinalMappingSourceBuilder ordinalMappingBuilder = new OrdinalMappingSourceBuilder( indexReader.leaves().size(), acceptableOverheadRatio, threshold); long currentGlobalOrdinal = 0; final AtomicFieldData.WithOrdinals[] withOrdinals = new AtomicFieldData.WithOrdinals[indexReader.leaves().size()]; TermIterator termIterator = new TermIterator(indexFieldData, indexReader.leaves(), withOrdinals); for (BytesRef term = termIterator.next(); term != null; term = termIterator.next()) { globalOrdToFirstSegment.add(termIterator.firstReaderIndex()); long globalOrdinalDelta = currentGlobalOrdinal - termIterator.firstLocalOrdinal(); globalOrdToFirstSegmentDelta.add(globalOrdinalDelta); for (TermIterator.LeafSource leafSource : termIterator.competitiveLeafs()) { ordinalMappingBuilder.onOrdinal( leafSource.context.ord, leafSource.tenum.ord(), currentGlobalOrdinal); } currentGlobalOrdinal++; } // ram used for the globalOrd to segmentOrd and segmentOrd to firstReaderIndex lookups long memorySizeInBytesCounter = 0; globalOrdToFirstSegment.freeze(); memorySizeInBytesCounter += globalOrdToFirstSegment.ramBytesUsed(); globalOrdToFirstSegmentDelta.freeze(); memorySizeInBytesCounter += globalOrdToFirstSegmentDelta.ramBytesUsed(); final long maxOrd = currentGlobalOrdinal; OrdinalMappingSource[] segmentOrdToGlobalOrdLookups = ordinalMappingBuilder.build(maxOrd); // add ram used for the main segmentOrd to globalOrd lookups memorySizeInBytesCounter += ordinalMappingBuilder.getMemorySizeInBytes(); final long memorySizeInBytes = memorySizeInBytesCounter; breakerService.getBreaker().addWithoutBreaking(memorySizeInBytes); if (logger.isDebugEnabled()) { // this does include the [] from the array in the impl name String implName = segmentOrdToGlobalOrdLookups.getClass().getSimpleName(); logger.debug( "Global-ordinals[{}][{}][{}] took {} ms", implName, indexFieldData.getFieldNames().fullName(), maxOrd, (System.currentTimeMillis() - startTime)); } return new InternalGlobalOrdinalsIndexFieldData( indexFieldData.index(), settings, indexFieldData.getFieldNames(), fieldDataType, withOrdinals, globalOrdToFirstSegment, globalOrdToFirstSegmentDelta, segmentOrdToGlobalOrdLookups, memorySizeInBytes); }