@Override
protected void doSetNextReader(AtomicReaderContext context) throws IOException {
if (current != null) {
missing += current.counts[0];
total += current.total - current.counts[0];
if (current.values.ordinals().getNumOrds() > 1) {
aggregators.add(current);
}
}
values = indexFieldData.load(context).getBytesValues();
current = new ReaderAggregator(values);
}
private TermIterator(
IndexFieldData.WithOrdinals indexFieldData,
List<AtomicReaderContext> leaves,
AtomicFieldData.WithOrdinals[] withOrdinals)
throws IOException {
this.sources = new LeafSourceQueue(leaves.size());
for (int i = 0; i < leaves.size(); i++) {
AtomicReaderContext atomicReaderContext = leaves.get(i);
AtomicFieldData.WithOrdinals afd = indexFieldData.load(atomicReaderContext);
withOrdinals[i] = afd;
LeafSource leafSource = new LeafSource(afd, atomicReaderContext);
if (leafSource.current != null) {
sources.add(leafSource);
}
}
}
@Override
public IndexFieldData.WithOrdinals build(
final IndexReader indexReader,
IndexFieldData.WithOrdinals indexFieldData,
Settings settings,
CircuitBreakerService breakerService)
throws IOException {
assert indexReader.leaves().size() > 1;
long startTime = System.currentTimeMillis();
// It makes sense to make the overhead ratio configurable for the mapping from segment ords to
// global ords
// However, other mappings are never the bottleneck and only used to get the original value from
// an ord, so
// it makes sense to force COMPACT for them
final float acceptableOverheadRatio =
settings.getAsFloat("acceptable_overhead_ratio", PackedInts.FAST);
final AppendingPackedLongBuffer globalOrdToFirstSegment =
new AppendingPackedLongBuffer(PackedInts.COMPACT);
final MonotonicAppendingLongBuffer globalOrdToFirstSegmentDelta =
new MonotonicAppendingLongBuffer(PackedInts.COMPACT);
FieldDataType fieldDataType = indexFieldData.getFieldDataType();
int defaultThreshold =
settings.getAsInt(
ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY, ORDINAL_MAPPING_THRESHOLD_DEFAULT);
int threshold =
fieldDataType.getSettings().getAsInt(ORDINAL_MAPPING_THRESHOLD_KEY, defaultThreshold);
OrdinalMappingSourceBuilder ordinalMappingBuilder =
new OrdinalMappingSourceBuilder(
indexReader.leaves().size(), acceptableOverheadRatio, threshold);
long currentGlobalOrdinal = 0;
final AtomicFieldData.WithOrdinals[] withOrdinals =
new AtomicFieldData.WithOrdinals[indexReader.leaves().size()];
TermIterator termIterator =
new TermIterator(indexFieldData, indexReader.leaves(), withOrdinals);
for (BytesRef term = termIterator.next(); term != null; term = termIterator.next()) {
globalOrdToFirstSegment.add(termIterator.firstReaderIndex());
long globalOrdinalDelta = currentGlobalOrdinal - termIterator.firstLocalOrdinal();
globalOrdToFirstSegmentDelta.add(globalOrdinalDelta);
for (TermIterator.LeafSource leafSource : termIterator.competitiveLeafs()) {
ordinalMappingBuilder.onOrdinal(
leafSource.context.ord, leafSource.tenum.ord(), currentGlobalOrdinal);
}
currentGlobalOrdinal++;
}
// ram used for the globalOrd to segmentOrd and segmentOrd to firstReaderIndex lookups
long memorySizeInBytesCounter = 0;
globalOrdToFirstSegment.freeze();
memorySizeInBytesCounter += globalOrdToFirstSegment.ramBytesUsed();
globalOrdToFirstSegmentDelta.freeze();
memorySizeInBytesCounter += globalOrdToFirstSegmentDelta.ramBytesUsed();
final long maxOrd = currentGlobalOrdinal;
OrdinalMappingSource[] segmentOrdToGlobalOrdLookups = ordinalMappingBuilder.build(maxOrd);
// add ram used for the main segmentOrd to globalOrd lookups
memorySizeInBytesCounter += ordinalMappingBuilder.getMemorySizeInBytes();
final long memorySizeInBytes = memorySizeInBytesCounter;
breakerService.getBreaker().addWithoutBreaking(memorySizeInBytes);
if (logger.isDebugEnabled()) {
// this does include the [] from the array in the impl name
String implName = segmentOrdToGlobalOrdLookups.getClass().getSimpleName();
logger.debug(
"Global-ordinals[{}][{}][{}] took {} ms",
implName,
indexFieldData.getFieldNames().fullName(),
maxOrd,
(System.currentTimeMillis() - startTime));
}
return new InternalGlobalOrdinalsIndexFieldData(
indexFieldData.index(),
settings,
indexFieldData.getFieldNames(),
fieldDataType,
withOrdinals,
globalOrdToFirstSegment,
globalOrdToFirstSegmentDelta,
segmentOrdToGlobalOrdLookups,
memorySizeInBytes);
}