protected AbstractIndexOrdinalsFieldData(
Index index,
Settings indexSettings,
Names fieldNames,
FieldDataType fieldDataType,
IndexFieldDataCache cache,
CircuitBreakerService breakerService) {
super(index, indexSettings, fieldNames, fieldDataType, cache);
final Map<String, Settings> groups = fieldDataType.getSettings().getGroups("filter");
frequency = groups.get("frequency");
regex = groups.get("regex");
this.breakerService = breakerService;
}
@Override
public TerminationHandle warmTopReader(
final IndexShard indexShard,
IndexMetaData indexMetaData,
final WarmerContext context,
ThreadPool threadPool) {
final MapperService mapperService = indexShard.mapperService();
final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
for (DocumentMapper docMapper : mapperService.docMappers(false)) {
for (FieldMapper fieldMapper : docMapper.mappers()) {
final FieldDataType fieldDataType = fieldMapper.fieldType().fieldDataType();
if (fieldDataType == null) {
continue;
}
if (fieldDataType.getLoading() != Loading.EAGER_GLOBAL_ORDINALS) {
continue;
}
final String indexName = fieldMapper.fieldType().names().indexName();
if (warmUpGlobalOrdinals.containsKey(indexName)) {
continue;
}
warmUpGlobalOrdinals.put(indexName, fieldMapper.fieldType());
}
}
final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
final Executor executor = threadPool.executor(executor());
final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size());
for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) {
executor.execute(
new Runnable() {
@Override
public void run() {
try {
final long start = System.nanoTime();
IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType);
ifd.loadGlobal(context.reader());
if (indexShard.warmerService().logger().isTraceEnabled()) {
indexShard
.warmerService()
.logger()
.trace(
"warmed global ordinals for [{}], took [{}]",
fieldType.names().fullName(),
TimeValue.timeValueNanos(System.nanoTime() - start));
}
} catch (Throwable t) {
indexShard
.warmerService()
.logger()
.warn(
"failed to warm-up global ordinals for [{}]",
t,
fieldType.names().fullName());
} finally {
latch.countDown();
}
}
});
}
return new TerminationHandle() {
@Override
public void awaitTermination() throws InterruptedException {
latch.await();
}
};
}
@Override
public IndexFieldData.WithOrdinals build(
final IndexReader indexReader,
IndexFieldData.WithOrdinals indexFieldData,
Settings settings,
CircuitBreakerService breakerService)
throws IOException {
assert indexReader.leaves().size() > 1;
long startTime = System.currentTimeMillis();
// It makes sense to make the overhead ratio configurable for the mapping from segment ords to
// global ords
// However, other mappings are never the bottleneck and only used to get the original value from
// an ord, so
// it makes sense to force COMPACT for them
final float acceptableOverheadRatio =
settings.getAsFloat("acceptable_overhead_ratio", PackedInts.FAST);
final AppendingPackedLongBuffer globalOrdToFirstSegment =
new AppendingPackedLongBuffer(PackedInts.COMPACT);
final MonotonicAppendingLongBuffer globalOrdToFirstSegmentDelta =
new MonotonicAppendingLongBuffer(PackedInts.COMPACT);
FieldDataType fieldDataType = indexFieldData.getFieldDataType();
int defaultThreshold =
settings.getAsInt(
ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY, ORDINAL_MAPPING_THRESHOLD_DEFAULT);
int threshold =
fieldDataType.getSettings().getAsInt(ORDINAL_MAPPING_THRESHOLD_KEY, defaultThreshold);
OrdinalMappingSourceBuilder ordinalMappingBuilder =
new OrdinalMappingSourceBuilder(
indexReader.leaves().size(), acceptableOverheadRatio, threshold);
long currentGlobalOrdinal = 0;
final AtomicFieldData.WithOrdinals[] withOrdinals =
new AtomicFieldData.WithOrdinals[indexReader.leaves().size()];
TermIterator termIterator =
new TermIterator(indexFieldData, indexReader.leaves(), withOrdinals);
for (BytesRef term = termIterator.next(); term != null; term = termIterator.next()) {
globalOrdToFirstSegment.add(termIterator.firstReaderIndex());
long globalOrdinalDelta = currentGlobalOrdinal - termIterator.firstLocalOrdinal();
globalOrdToFirstSegmentDelta.add(globalOrdinalDelta);
for (TermIterator.LeafSource leafSource : termIterator.competitiveLeafs()) {
ordinalMappingBuilder.onOrdinal(
leafSource.context.ord, leafSource.tenum.ord(), currentGlobalOrdinal);
}
currentGlobalOrdinal++;
}
// ram used for the globalOrd to segmentOrd and segmentOrd to firstReaderIndex lookups
long memorySizeInBytesCounter = 0;
globalOrdToFirstSegment.freeze();
memorySizeInBytesCounter += globalOrdToFirstSegment.ramBytesUsed();
globalOrdToFirstSegmentDelta.freeze();
memorySizeInBytesCounter += globalOrdToFirstSegmentDelta.ramBytesUsed();
final long maxOrd = currentGlobalOrdinal;
OrdinalMappingSource[] segmentOrdToGlobalOrdLookups = ordinalMappingBuilder.build(maxOrd);
// add ram used for the main segmentOrd to globalOrd lookups
memorySizeInBytesCounter += ordinalMappingBuilder.getMemorySizeInBytes();
final long memorySizeInBytes = memorySizeInBytesCounter;
breakerService.getBreaker().addWithoutBreaking(memorySizeInBytes);
if (logger.isDebugEnabled()) {
// this does include the [] from the array in the impl name
String implName = segmentOrdToGlobalOrdLookups.getClass().getSimpleName();
logger.debug(
"Global-ordinals[{}][{}][{}] took {} ms",
implName,
indexFieldData.getFieldNames().fullName(),
maxOrd,
(System.currentTimeMillis() - startTime));
}
return new InternalGlobalOrdinalsIndexFieldData(
indexFieldData.index(),
settings,
indexFieldData.getFieldNames(),
fieldDataType,
withOrdinals,
globalOrdToFirstSegment,
globalOrdToFirstSegmentDelta,
segmentOrdToGlobalOrdLookups,
memorySizeInBytes);
}