@Override
public TerminationHandle warmReader(
final IndexShard indexShard, final Engine.Searcher searcher) {
final MapperService mapperService = indexShard.mapperService();
final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
for (DocumentMapper docMapper : mapperService.docMappers(false)) {
for (FieldMapper fieldMapper : docMapper.mappers()) {
final MappedFieldType fieldType = fieldMapper.fieldType();
final String indexName = fieldType.name();
if (fieldType.eagerGlobalOrdinals() == false) {
continue;
}
warmUpGlobalOrdinals.put(indexName, fieldType);
}
}
final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size());
for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) {
executor.execute(
() -> {
try {
final long start = System.nanoTime();
IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType);
DirectoryReader reader = searcher.getDirectoryReader();
IndexFieldData<?> global = ifd.loadGlobal(reader);
if (reader.leaves().isEmpty() == false) {
global.load(reader.leaves().get(0));
}
if (indexShard.warmerService().logger().isTraceEnabled()) {
indexShard
.warmerService()
.logger()
.trace(
"warmed global ordinals for [{}], took [{}]",
fieldType.name(),
TimeValue.timeValueNanos(System.nanoTime() - start));
}
} catch (Exception e) {
indexShard
.warmerService()
.logger()
.warn("failed to warm-up global ordinals for [{}]", e, fieldType.name());
} finally {
latch.countDown();
}
});
}
return () -> latch.await();
}
private void purgeShards(List<IndexShard> shardsToPurge) {
for (IndexShard shardToPurge : shardsToPurge) {
Query query =
shardToPurge
.mapperService()
.fullName(TTLFieldMapper.NAME)
.rangeQuery(null, System.currentTimeMillis(), false, true);
Engine.Searcher searcher = shardToPurge.acquireSearcher("indices_ttl");
try {
logger.debug(
"[{}][{}] purging shard",
shardToPurge.routingEntry().index(),
shardToPurge.routingEntry().id());
ExpiredDocsCollector expiredDocsCollector = new ExpiredDocsCollector();
searcher.searcher().search(query, expiredDocsCollector);
List<DocToPurge> docsToPurge = expiredDocsCollector.getDocsToPurge();
BulkRequest bulkRequest = new BulkRequest();
for (DocToPurge docToPurge : docsToPurge) {
bulkRequest.add(
new DeleteRequest()
.index(shardToPurge.routingEntry().getIndexName())
.type(docToPurge.type)
.id(docToPurge.id)
.version(docToPurge.version)
.routing(docToPurge.routing));
bulkRequest = processBulkIfNeeded(bulkRequest, false);
}
processBulkIfNeeded(bulkRequest, true);
} catch (Exception e) {
logger.warn("failed to purge", e);
} finally {
searcher.close();
}
}
}
@Override
public TerminationHandle warmNewReaders(
final IndexShard indexShard,
IndexMetaData indexMetaData,
final WarmerContext context,
ThreadPool threadPool) {
final Loading defaultLoading =
Loading.parse(indexMetaData.settings().get(NORMS_LOADING_KEY), Loading.LAZY);
final MapperService mapperService = indexShard.mapperService();
final ObjectSet<String> warmUp = new ObjectHashSet<>();
for (DocumentMapper docMapper : mapperService.docMappers(false)) {
for (FieldMapper fieldMapper : docMapper.mappers()) {
final String indexName = fieldMapper.fieldType().names().indexName();
Loading normsLoading = fieldMapper.fieldType().normsLoading();
if (normsLoading == null) {
normsLoading = defaultLoading;
}
if (fieldMapper.fieldType().indexOptions() != IndexOptions.NONE
&& !fieldMapper.fieldType().omitNorms()
&& normsLoading == Loading.EAGER) {
warmUp.add(indexName);
}
}
}
final CountDownLatch latch = new CountDownLatch(1);
// Norms loading may be I/O intensive but is not CPU intensive, so we execute it in a single
// task
threadPool
.executor(executor())
.execute(
new Runnable() {
@Override
public void run() {
try {
for (ObjectCursor<String> stringObjectCursor : warmUp) {
final String indexName = stringObjectCursor.value;
final long start = System.nanoTime();
for (final LeafReaderContext ctx : context.searcher().reader().leaves()) {
final NumericDocValues values = ctx.reader().getNormValues(indexName);
if (values != null) {
values.get(0);
}
}
if (indexShard.warmerService().logger().isTraceEnabled()) {
indexShard
.warmerService()
.logger()
.trace(
"warmed norms for [{}], took [{}]",
indexName,
TimeValue.timeValueNanos(System.nanoTime() - start));
}
}
} catch (Throwable t) {
indexShard.warmerService().logger().warn("failed to warm-up norms", t);
} finally {
latch.countDown();
}
}
});
return new TerminationHandle() {
@Override
public void awaitTermination() throws InterruptedException {
latch.await();
}
};
}
@Override
public TerminationHandle warmTopReader(
final IndexShard indexShard,
IndexMetaData indexMetaData,
final WarmerContext context,
ThreadPool threadPool) {
final MapperService mapperService = indexShard.mapperService();
final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
for (DocumentMapper docMapper : mapperService.docMappers(false)) {
for (FieldMapper fieldMapper : docMapper.mappers()) {
final FieldDataType fieldDataType = fieldMapper.fieldType().fieldDataType();
if (fieldDataType == null) {
continue;
}
if (fieldDataType.getLoading() != Loading.EAGER_GLOBAL_ORDINALS) {
continue;
}
final String indexName = fieldMapper.fieldType().names().indexName();
if (warmUpGlobalOrdinals.containsKey(indexName)) {
continue;
}
warmUpGlobalOrdinals.put(indexName, fieldMapper.fieldType());
}
}
final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
final Executor executor = threadPool.executor(executor());
final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size());
for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) {
executor.execute(
new Runnable() {
@Override
public void run() {
try {
final long start = System.nanoTime();
IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType);
ifd.loadGlobal(context.reader());
if (indexShard.warmerService().logger().isTraceEnabled()) {
indexShard
.warmerService()
.logger()
.trace(
"warmed global ordinals for [{}], took [{}]",
fieldType.names().fullName(),
TimeValue.timeValueNanos(System.nanoTime() - start));
}
} catch (Throwable t) {
indexShard
.warmerService()
.logger()
.warn(
"failed to warm-up global ordinals for [{}]",
t,
fieldType.names().fullName());
} finally {
latch.countDown();
}
}
});
}
return new TerminationHandle() {
@Override
public void awaitTermination() throws InterruptedException {
latch.await();
}
};
}