protected AbstractIndexOrdinalsFieldData(
Index index,
Settings indexSettings,
Names fieldNames,
FieldDataType fieldDataType,
IndexFieldDataCache cache,
CircuitBreakerService breakerService) {
super(index, indexSettings, fieldNames, fieldDataType, cache);
final Map<String, Settings> groups = fieldDataType.getSettings().getGroups("filter");
frequency = groups.get("frequency");
regex = groups.get("regex");
this.breakerService = breakerService;
}
@Override
public IndexFieldData.WithOrdinals build(
final IndexReader indexReader,
IndexFieldData.WithOrdinals indexFieldData,
Settings settings,
CircuitBreakerService breakerService)
throws IOException {
assert indexReader.leaves().size() > 1;
long startTime = System.currentTimeMillis();
// It makes sense to make the overhead ratio configurable for the mapping from segment ords to
// global ords
// However, other mappings are never the bottleneck and only used to get the original value from
// an ord, so
// it makes sense to force COMPACT for them
final float acceptableOverheadRatio =
settings.getAsFloat("acceptable_overhead_ratio", PackedInts.FAST);
final AppendingPackedLongBuffer globalOrdToFirstSegment =
new AppendingPackedLongBuffer(PackedInts.COMPACT);
final MonotonicAppendingLongBuffer globalOrdToFirstSegmentDelta =
new MonotonicAppendingLongBuffer(PackedInts.COMPACT);
FieldDataType fieldDataType = indexFieldData.getFieldDataType();
int defaultThreshold =
settings.getAsInt(
ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY, ORDINAL_MAPPING_THRESHOLD_DEFAULT);
int threshold =
fieldDataType.getSettings().getAsInt(ORDINAL_MAPPING_THRESHOLD_KEY, defaultThreshold);
OrdinalMappingSourceBuilder ordinalMappingBuilder =
new OrdinalMappingSourceBuilder(
indexReader.leaves().size(), acceptableOverheadRatio, threshold);
long currentGlobalOrdinal = 0;
final AtomicFieldData.WithOrdinals[] withOrdinals =
new AtomicFieldData.WithOrdinals[indexReader.leaves().size()];
TermIterator termIterator =
new TermIterator(indexFieldData, indexReader.leaves(), withOrdinals);
for (BytesRef term = termIterator.next(); term != null; term = termIterator.next()) {
globalOrdToFirstSegment.add(termIterator.firstReaderIndex());
long globalOrdinalDelta = currentGlobalOrdinal - termIterator.firstLocalOrdinal();
globalOrdToFirstSegmentDelta.add(globalOrdinalDelta);
for (TermIterator.LeafSource leafSource : termIterator.competitiveLeafs()) {
ordinalMappingBuilder.onOrdinal(
leafSource.context.ord, leafSource.tenum.ord(), currentGlobalOrdinal);
}
currentGlobalOrdinal++;
}
// ram used for the globalOrd to segmentOrd and segmentOrd to firstReaderIndex lookups
long memorySizeInBytesCounter = 0;
globalOrdToFirstSegment.freeze();
memorySizeInBytesCounter += globalOrdToFirstSegment.ramBytesUsed();
globalOrdToFirstSegmentDelta.freeze();
memorySizeInBytesCounter += globalOrdToFirstSegmentDelta.ramBytesUsed();
final long maxOrd = currentGlobalOrdinal;
OrdinalMappingSource[] segmentOrdToGlobalOrdLookups = ordinalMappingBuilder.build(maxOrd);
// add ram used for the main segmentOrd to globalOrd lookups
memorySizeInBytesCounter += ordinalMappingBuilder.getMemorySizeInBytes();
final long memorySizeInBytes = memorySizeInBytesCounter;
breakerService.getBreaker().addWithoutBreaking(memorySizeInBytes);
if (logger.isDebugEnabled()) {
// this does include the [] from the array in the impl name
String implName = segmentOrdToGlobalOrdLookups.getClass().getSimpleName();
logger.debug(
"Global-ordinals[{}][{}][{}] took {} ms",
implName,
indexFieldData.getFieldNames().fullName(),
maxOrd,
(System.currentTimeMillis() - startTime));
}
return new InternalGlobalOrdinalsIndexFieldData(
indexFieldData.index(),
settings,
indexFieldData.getFieldNames(),
fieldDataType,
withOrdinals,
globalOrdToFirstSegment,
globalOrdToFirstSegmentDelta,
segmentOrdToGlobalOrdLookups,
memorySizeInBytes);
}