@Override public TerminationHandle warmReader( final IndexShard indexShard, final Engine.Searcher searcher) { final MapperService mapperService = indexShard.mapperService(); final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>(); for (DocumentMapper docMapper : mapperService.docMappers(false)) { for (FieldMapper fieldMapper : docMapper.mappers()) { final MappedFieldType fieldType = fieldMapper.fieldType(); final String indexName = fieldType.name(); if (fieldType.eagerGlobalOrdinals() == false) { continue; } warmUpGlobalOrdinals.put(indexName, fieldType); } } final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService(); final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size()); for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) { executor.execute( () -> { try { final long start = System.nanoTime(); IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType); DirectoryReader reader = searcher.getDirectoryReader(); IndexFieldData<?> global = ifd.loadGlobal(reader); if (reader.leaves().isEmpty() == false) { global.load(reader.leaves().get(0)); } if (indexShard.warmerService().logger().isTraceEnabled()) { indexShard .warmerService() .logger() .trace( "warmed global ordinals for [{}], took [{}]", fieldType.name(), TimeValue.timeValueNanos(System.nanoTime() - start)); } } catch (Exception e) { indexShard .warmerService() .logger() .warn("failed to warm-up global ordinals for [{}]", e, fieldType.name()); } finally { latch.countDown(); } }); } return () -> latch.await(); }
public IndexService( IndexSettings indexSettings, NodeEnvironment nodeEnv, SimilarityService similarityService, ShardStoreDeleter shardStoreDeleter, AnalysisRegistry registry, @Nullable EngineFactory engineFactory, NodeServicesProvider nodeServicesProvider, QueryCache queryCache, IndexStore indexStore, IndexEventListener eventListener, IndexModule.IndexSearcherWrapperFactory wrapperFactory, MapperRegistry mapperRegistry, IndexingOperationListener... listenersIn) throws IOException { super(indexSettings); this.indexSettings = indexSettings; this.analysisService = registry.build(indexSettings); this.similarityService = similarityService; this.mapperService = new MapperService( indexSettings, analysisService, similarityService, mapperRegistry, IndexService.this::getQueryShardContext); this.indexFieldData = new IndexFieldDataService( indexSettings, nodeServicesProvider.getIndicesFieldDataCache(), nodeServicesProvider.getCircuitBreakerService(), mapperService); this.shardStoreDeleter = shardStoreDeleter; this.eventListener = eventListener; this.nodeEnv = nodeEnv; this.nodeServicesProvider = nodeServicesProvider; this.indexStore = indexStore; indexFieldData.setListener(new FieldDataCacheListener(this)); this.bitsetFilterCache = new BitsetFilterCache( indexSettings, nodeServicesProvider.getWarmer(), new BitsetCacheListener(this)); this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache); this.engineFactory = engineFactory; // initialize this last -- otherwise if the wrapper requires any other member to be non-null we // fail with an NPE this.searcherWrapper = wrapperFactory.newWrapper(this); this.slowLog = new IndexingSlowLog(indexSettings.getSettings()); // Add our slowLog to the incoming IndexingOperationListeners: this.listeners = new IndexingOperationListener[1 + listenersIn.length]; this.listeners[0] = slowLog; System.arraycopy(listenersIn, 0, this.listeners, 1, listenersIn.length); // kick off async ops for the first shard in this index if (this.indexSettings.getTranslogSyncInterval().millis() != 0) { this.fsyncTask = new AsyncTranslogFSync(this); } else { this.fsyncTask = null; } this.refreshTask = new AsyncRefreshTask(this); }
@After public void tearDown() throws Exception { super.tearDown(); reader.close(); writer.close(); ifdService.clear(); SearchContext.removeCurrent(); }
public IndexService( IndexSettings indexSettings, NodeEnvironment nodeEnv, SimilarityService similarityService, ShardStoreDeleter shardStoreDeleter, AnalysisRegistry registry, @Nullable EngineFactory engineFactory, NodeServicesProvider nodeServicesProvider, QueryCache queryCache, IndexStore indexStore, IndexEventListener eventListener, IndexModule.IndexSearcherWrapperFactory wrapperFactory, MapperRegistry mapperRegistry, IndicesFieldDataCache indicesFieldDataCache, List<SearchOperationListener> searchOperationListeners, List<IndexingOperationListener> indexingOperationListeners) throws IOException { super(indexSettings); this.indexSettings = indexSettings; this.analysisService = registry.build(indexSettings); this.similarityService = similarityService; this.mapperService = new MapperService( indexSettings, analysisService, similarityService, mapperRegistry, IndexService.this::newQueryShardContext); this.indexFieldData = new IndexFieldDataService( indexSettings, indicesFieldDataCache, nodeServicesProvider.getCircuitBreakerService(), mapperService); this.shardStoreDeleter = shardStoreDeleter; this.bigArrays = nodeServicesProvider.getBigArrays(); this.threadPool = nodeServicesProvider.getThreadPool(); this.eventListener = eventListener; this.nodeEnv = nodeEnv; this.nodeServicesProvider = nodeServicesProvider; this.indexStore = indexStore; indexFieldData.setListener(new FieldDataCacheListener(this)); this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this)); this.warmer = new IndexWarmer( indexSettings.getSettings(), threadPool, bitsetFilterCache.createListener(threadPool)); this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache); this.engineFactory = engineFactory; // initialize this last -- otherwise if the wrapper requires any other member to be non-null we // fail with an NPE this.searcherWrapper = wrapperFactory.newWrapper(this); this.indexingOperationListeners = Collections.unmodifiableList(indexingOperationListeners); this.searchOperationListeners = Collections.unmodifiableList(searchOperationListeners); // kick off async ops for the first shard in this index this.refreshTask = new AsyncRefreshTask(this); rescheduleFsyncTask(indexSettings.getTranslogDurability()); }
@Before public void setup() throws Exception { super.setUp(); // setup field mappers strMapper = new StringFieldMapper.Builder("str_value") .build(new Mapper.BuilderContext(null, new ContentPath(1))); lngMapper = new LongFieldMapper.Builder("lng_value") .build(new Mapper.BuilderContext(null, new ContentPath(1))); dblMapper = new DoubleFieldMapper.Builder("dbl_value") .build(new Mapper.BuilderContext(null, new ContentPath(1))); // create index and fielddata service ifdService = new IndexFieldDataService(new Index("test"), new DummyCircuitBreakerService()); MapperService mapperService = MapperTestUtils.newMapperService( ifdService.index(), ImmutableSettings.Builder.EMPTY_SETTINGS); ifdService.setIndexService(new StubIndexService(mapperService)); writer = new IndexWriter( new RAMDirectory(), new IndexWriterConfig(Lucene.VERSION, new StandardAnalyzer(Lucene.VERSION))); int numDocs = 10; for (int i = 0; i < numDocs; i++) { Document d = new Document(); d.add(new StringField(strMapper.names().indexName(), "str" + i, Field.Store.NO)); d.add(new LongField(lngMapper.names().indexName(), i, Field.Store.NO)); d.add(new DoubleField(dblMapper.names().indexName(), Double.valueOf(i), Field.Store.NO)); writer.addDocument(d); } reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(writer, true)); }
public <IFD extends IndexFieldData<?>> IFD getForField(FieldDataType type, String fieldName) { final FieldMapper<?> mapper; final BuilderContext context = new BuilderContext(indexService.settingsService().getSettings(), new ContentPath(1)); if (type.getType().equals("string")) { mapper = MapperBuilders.stringField(fieldName) .tokenized(false) .fieldDataSettings(type.getSettings()) .build(context); } else if (type.getType().equals("float")) { mapper = MapperBuilders.floatField(fieldName).fieldDataSettings(type.getSettings()).build(context); } else if (type.getType().equals("double")) { mapper = MapperBuilders.doubleField(fieldName) .fieldDataSettings(type.getSettings()) .build(context); } else if (type.getType().equals("long")) { mapper = MapperBuilders.longField(fieldName).fieldDataSettings(type.getSettings()).build(context); } else if (type.getType().equals("int")) { mapper = MapperBuilders.integerField(fieldName) .fieldDataSettings(type.getSettings()) .build(context); } else if (type.getType().equals("short")) { mapper = MapperBuilders.shortField(fieldName).fieldDataSettings(type.getSettings()).build(context); } else if (type.getType().equals("byte")) { mapper = MapperBuilders.byteField(fieldName).fieldDataSettings(type.getSettings()).build(context); } else if (type.getType().equals("geo_point")) { mapper = MapperBuilders.geoPointField(fieldName) .fieldDataSettings(type.getSettings()) .build(context); } else if (type.getType().equals("_parent")) { mapper = MapperBuilders.parent().type(fieldName).build(context); } else if (type.getType().equals("binary")) { mapper = MapperBuilders.binaryField(fieldName) .fieldDataSettings(type.getSettings()) .build(context); } else { throw new UnsupportedOperationException(type.getType()); } return ifdService.getForField(mapper); }
@Override public Filter rangeFilter( IndexFieldDataService fieldData, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, @Nullable QueryParseContext context) { return NumericRangeFieldDataFilter.newIntRange( (IndexNumericFieldData) fieldData.getForField(this), lowerTerm == null ? null : parseValue(lowerTerm), upperTerm == null ? null : parseValue(upperTerm), includeLower, includeUpper); }
protected <IFD extends IndexNumericFieldData> IFD getFieldData(NumberFieldMapper fieldMapper) { return ifdService.getForField(fieldMapper); }