@Override
public TerminationHandle warmReader(
final IndexShard indexShard, final Engine.Searcher searcher) {
final MapperService mapperService = indexShard.mapperService();
final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
for (DocumentMapper docMapper : mapperService.docMappers(false)) {
for (FieldMapper fieldMapper : docMapper.mappers()) {
final MappedFieldType fieldType = fieldMapper.fieldType();
final String indexName = fieldType.name();
if (fieldType.eagerGlobalOrdinals() == false) {
continue;
}
warmUpGlobalOrdinals.put(indexName, fieldType);
}
}
final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size());
for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) {
executor.execute(
() -> {
try {
final long start = System.nanoTime();
IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType);
DirectoryReader reader = searcher.getDirectoryReader();
IndexFieldData<?> global = ifd.loadGlobal(reader);
if (reader.leaves().isEmpty() == false) {
global.load(reader.leaves().get(0));
}
if (indexShard.warmerService().logger().isTraceEnabled()) {
indexShard
.warmerService()
.logger()
.trace(
"warmed global ordinals for [{}], took [{}]",
fieldType.name(),
TimeValue.timeValueNanos(System.nanoTime() - start));
}
} catch (Exception e) {
indexShard
.warmerService()
.logger()
.warn("failed to warm-up global ordinals for [{}]", e, fieldType.name());
} finally {
latch.countDown();
}
});
}
return () -> latch.await();
}
public IndexService(
IndexSettings indexSettings,
NodeEnvironment nodeEnv,
SimilarityService similarityService,
ShardStoreDeleter shardStoreDeleter,
AnalysisRegistry registry,
@Nullable EngineFactory engineFactory,
NodeServicesProvider nodeServicesProvider,
QueryCache queryCache,
IndexStore indexStore,
IndexEventListener eventListener,
IndexModule.IndexSearcherWrapperFactory wrapperFactory,
MapperRegistry mapperRegistry,
IndexingOperationListener... listenersIn)
throws IOException {
super(indexSettings);
this.indexSettings = indexSettings;
this.analysisService = registry.build(indexSettings);
this.similarityService = similarityService;
this.mapperService =
new MapperService(
indexSettings,
analysisService,
similarityService,
mapperRegistry,
IndexService.this::getQueryShardContext);
this.indexFieldData =
new IndexFieldDataService(
indexSettings,
nodeServicesProvider.getIndicesFieldDataCache(),
nodeServicesProvider.getCircuitBreakerService(),
mapperService);
this.shardStoreDeleter = shardStoreDeleter;
this.eventListener = eventListener;
this.nodeEnv = nodeEnv;
this.nodeServicesProvider = nodeServicesProvider;
this.indexStore = indexStore;
indexFieldData.setListener(new FieldDataCacheListener(this));
this.bitsetFilterCache =
new BitsetFilterCache(
indexSettings, nodeServicesProvider.getWarmer(), new BitsetCacheListener(this));
this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache);
this.engineFactory = engineFactory;
// initialize this last -- otherwise if the wrapper requires any other member to be non-null we
// fail with an NPE
this.searcherWrapper = wrapperFactory.newWrapper(this);
this.slowLog = new IndexingSlowLog(indexSettings.getSettings());
// Add our slowLog to the incoming IndexingOperationListeners:
this.listeners = new IndexingOperationListener[1 + listenersIn.length];
this.listeners[0] = slowLog;
System.arraycopy(listenersIn, 0, this.listeners, 1, listenersIn.length);
// kick off async ops for the first shard in this index
if (this.indexSettings.getTranslogSyncInterval().millis() != 0) {
this.fsyncTask = new AsyncTranslogFSync(this);
} else {
this.fsyncTask = null;
}
this.refreshTask = new AsyncRefreshTask(this);
}
@After
public void tearDown() throws Exception {
super.tearDown();
reader.close();
writer.close();
ifdService.clear();
SearchContext.removeCurrent();
}
public IndexService(
IndexSettings indexSettings,
NodeEnvironment nodeEnv,
SimilarityService similarityService,
ShardStoreDeleter shardStoreDeleter,
AnalysisRegistry registry,
@Nullable EngineFactory engineFactory,
NodeServicesProvider nodeServicesProvider,
QueryCache queryCache,
IndexStore indexStore,
IndexEventListener eventListener,
IndexModule.IndexSearcherWrapperFactory wrapperFactory,
MapperRegistry mapperRegistry,
IndicesFieldDataCache indicesFieldDataCache,
List<SearchOperationListener> searchOperationListeners,
List<IndexingOperationListener> indexingOperationListeners)
throws IOException {
super(indexSettings);
this.indexSettings = indexSettings;
this.analysisService = registry.build(indexSettings);
this.similarityService = similarityService;
this.mapperService =
new MapperService(
indexSettings,
analysisService,
similarityService,
mapperRegistry,
IndexService.this::newQueryShardContext);
this.indexFieldData =
new IndexFieldDataService(
indexSettings,
indicesFieldDataCache,
nodeServicesProvider.getCircuitBreakerService(),
mapperService);
this.shardStoreDeleter = shardStoreDeleter;
this.bigArrays = nodeServicesProvider.getBigArrays();
this.threadPool = nodeServicesProvider.getThreadPool();
this.eventListener = eventListener;
this.nodeEnv = nodeEnv;
this.nodeServicesProvider = nodeServicesProvider;
this.indexStore = indexStore;
indexFieldData.setListener(new FieldDataCacheListener(this));
this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
this.warmer =
new IndexWarmer(
indexSettings.getSettings(), threadPool, bitsetFilterCache.createListener(threadPool));
this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache);
this.engineFactory = engineFactory;
// initialize this last -- otherwise if the wrapper requires any other member to be non-null we
// fail with an NPE
this.searcherWrapper = wrapperFactory.newWrapper(this);
this.indexingOperationListeners = Collections.unmodifiableList(indexingOperationListeners);
this.searchOperationListeners = Collections.unmodifiableList(searchOperationListeners);
// kick off async ops for the first shard in this index
this.refreshTask = new AsyncRefreshTask(this);
rescheduleFsyncTask(indexSettings.getTranslogDurability());
}
@Before
public void setup() throws Exception {
super.setUp();
// setup field mappers
strMapper =
new StringFieldMapper.Builder("str_value")
.build(new Mapper.BuilderContext(null, new ContentPath(1)));
lngMapper =
new LongFieldMapper.Builder("lng_value")
.build(new Mapper.BuilderContext(null, new ContentPath(1)));
dblMapper =
new DoubleFieldMapper.Builder("dbl_value")
.build(new Mapper.BuilderContext(null, new ContentPath(1)));
// create index and fielddata service
ifdService = new IndexFieldDataService(new Index("test"), new DummyCircuitBreakerService());
MapperService mapperService =
MapperTestUtils.newMapperService(
ifdService.index(), ImmutableSettings.Builder.EMPTY_SETTINGS);
ifdService.setIndexService(new StubIndexService(mapperService));
writer =
new IndexWriter(
new RAMDirectory(),
new IndexWriterConfig(Lucene.VERSION, new StandardAnalyzer(Lucene.VERSION)));
int numDocs = 10;
for (int i = 0; i < numDocs; i++) {
Document d = new Document();
d.add(new StringField(strMapper.names().indexName(), "str" + i, Field.Store.NO));
d.add(new LongField(lngMapper.names().indexName(), i, Field.Store.NO));
d.add(new DoubleField(dblMapper.names().indexName(), Double.valueOf(i), Field.Store.NO));
writer.addDocument(d);
}
reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(writer, true));
}
public <IFD extends IndexFieldData<?>> IFD getForField(FieldDataType type, String fieldName) {
final FieldMapper<?> mapper;
final BuilderContext context =
new BuilderContext(indexService.settingsService().getSettings(), new ContentPath(1));
if (type.getType().equals("string")) {
mapper =
MapperBuilders.stringField(fieldName)
.tokenized(false)
.fieldDataSettings(type.getSettings())
.build(context);
} else if (type.getType().equals("float")) {
mapper =
MapperBuilders.floatField(fieldName).fieldDataSettings(type.getSettings()).build(context);
} else if (type.getType().equals("double")) {
mapper =
MapperBuilders.doubleField(fieldName)
.fieldDataSettings(type.getSettings())
.build(context);
} else if (type.getType().equals("long")) {
mapper =
MapperBuilders.longField(fieldName).fieldDataSettings(type.getSettings()).build(context);
} else if (type.getType().equals("int")) {
mapper =
MapperBuilders.integerField(fieldName)
.fieldDataSettings(type.getSettings())
.build(context);
} else if (type.getType().equals("short")) {
mapper =
MapperBuilders.shortField(fieldName).fieldDataSettings(type.getSettings()).build(context);
} else if (type.getType().equals("byte")) {
mapper =
MapperBuilders.byteField(fieldName).fieldDataSettings(type.getSettings()).build(context);
} else if (type.getType().equals("geo_point")) {
mapper =
MapperBuilders.geoPointField(fieldName)
.fieldDataSettings(type.getSettings())
.build(context);
} else if (type.getType().equals("_parent")) {
mapper = MapperBuilders.parent().type(fieldName).build(context);
} else if (type.getType().equals("binary")) {
mapper =
MapperBuilders.binaryField(fieldName)
.fieldDataSettings(type.getSettings())
.build(context);
} else {
throw new UnsupportedOperationException(type.getType());
}
return ifdService.getForField(mapper);
}
@Override
public Filter rangeFilter(
IndexFieldDataService fieldData,
Object lowerTerm,
Object upperTerm,
boolean includeLower,
boolean includeUpper,
@Nullable QueryParseContext context) {
return NumericRangeFieldDataFilter.newIntRange(
(IndexNumericFieldData) fieldData.getForField(this),
lowerTerm == null ? null : parseValue(lowerTerm),
upperTerm == null ? null : parseValue(upperTerm),
includeLower,
includeUpper);
}
protected <IFD extends IndexNumericFieldData> IFD getFieldData(NumberFieldMapper fieldMapper) {
return ifdService.getForField(fieldMapper);
}