protected AbstractIndexOrdinalsFieldData( Index index, Settings indexSettings, Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache, CircuitBreakerService breakerService) { super(index, indexSettings, fieldNames, fieldDataType, cache); final Map<String, Settings> groups = fieldDataType.getSettings().getGroups("filter"); frequency = groups.get("frequency"); regex = groups.get("regex"); this.breakerService = breakerService; }
@Override public IndexFieldData.WithOrdinals build( final IndexReader indexReader, IndexFieldData.WithOrdinals indexFieldData, Settings settings, CircuitBreakerService breakerService) throws IOException { assert indexReader.leaves().size() > 1; long startTime = System.currentTimeMillis(); // It makes sense to make the overhead ratio configurable for the mapping from segment ords to // global ords // However, other mappings are never the bottleneck and only used to get the original value from // an ord, so // it makes sense to force COMPACT for them final float acceptableOverheadRatio = settings.getAsFloat("acceptable_overhead_ratio", PackedInts.FAST); final AppendingPackedLongBuffer globalOrdToFirstSegment = new AppendingPackedLongBuffer(PackedInts.COMPACT); final MonotonicAppendingLongBuffer globalOrdToFirstSegmentDelta = new MonotonicAppendingLongBuffer(PackedInts.COMPACT); FieldDataType fieldDataType = indexFieldData.getFieldDataType(); int defaultThreshold = settings.getAsInt( ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY, ORDINAL_MAPPING_THRESHOLD_DEFAULT); int threshold = fieldDataType.getSettings().getAsInt(ORDINAL_MAPPING_THRESHOLD_KEY, defaultThreshold); OrdinalMappingSourceBuilder ordinalMappingBuilder = new OrdinalMappingSourceBuilder( indexReader.leaves().size(), acceptableOverheadRatio, threshold); long currentGlobalOrdinal = 0; final AtomicFieldData.WithOrdinals[] withOrdinals = new AtomicFieldData.WithOrdinals[indexReader.leaves().size()]; TermIterator termIterator = new TermIterator(indexFieldData, indexReader.leaves(), withOrdinals); for (BytesRef term = termIterator.next(); term != null; term = termIterator.next()) { globalOrdToFirstSegment.add(termIterator.firstReaderIndex()); long globalOrdinalDelta = currentGlobalOrdinal - termIterator.firstLocalOrdinal(); globalOrdToFirstSegmentDelta.add(globalOrdinalDelta); for (TermIterator.LeafSource leafSource : termIterator.competitiveLeafs()) { ordinalMappingBuilder.onOrdinal( leafSource.context.ord, leafSource.tenum.ord(), currentGlobalOrdinal); } currentGlobalOrdinal++; } // ram used for the globalOrd to segmentOrd and segmentOrd to firstReaderIndex lookups long memorySizeInBytesCounter = 0; globalOrdToFirstSegment.freeze(); memorySizeInBytesCounter += globalOrdToFirstSegment.ramBytesUsed(); globalOrdToFirstSegmentDelta.freeze(); memorySizeInBytesCounter += globalOrdToFirstSegmentDelta.ramBytesUsed(); final long maxOrd = currentGlobalOrdinal; OrdinalMappingSource[] segmentOrdToGlobalOrdLookups = ordinalMappingBuilder.build(maxOrd); // add ram used for the main segmentOrd to globalOrd lookups memorySizeInBytesCounter += ordinalMappingBuilder.getMemorySizeInBytes(); final long memorySizeInBytes = memorySizeInBytesCounter; breakerService.getBreaker().addWithoutBreaking(memorySizeInBytes); if (logger.isDebugEnabled()) { // this does include the [] from the array in the impl name String implName = segmentOrdToGlobalOrdLookups.getClass().getSimpleName(); logger.debug( "Global-ordinals[{}][{}][{}] took {} ms", implName, indexFieldData.getFieldNames().fullName(), maxOrd, (System.currentTimeMillis() - startTime)); } return new InternalGlobalOrdinalsIndexFieldData( indexFieldData.index(), settings, indexFieldData.getFieldNames(), fieldDataType, withOrdinals, globalOrdToFirstSegment, globalOrdToFirstSegmentDelta, segmentOrdToGlobalOrdLookups, memorySizeInBytes); }