Пример #1
0
    @Override
    public TerminationHandle warmReader(
        final IndexShard indexShard, final Engine.Searcher searcher) {
      final MapperService mapperService = indexShard.mapperService();
      final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
      for (DocumentMapper docMapper : mapperService.docMappers(false)) {
        for (FieldMapper fieldMapper : docMapper.mappers()) {
          final MappedFieldType fieldType = fieldMapper.fieldType();
          final String indexName = fieldType.name();
          if (fieldType.eagerGlobalOrdinals() == false) {
            continue;
          }
          warmUpGlobalOrdinals.put(indexName, fieldType);
        }
      }
      final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
      final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size());
      for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) {
        executor.execute(
            () -> {
              try {
                final long start = System.nanoTime();
                IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType);
                DirectoryReader reader = searcher.getDirectoryReader();
                IndexFieldData<?> global = ifd.loadGlobal(reader);
                if (reader.leaves().isEmpty() == false) {
                  global.load(reader.leaves().get(0));
                }

                if (indexShard.warmerService().logger().isTraceEnabled()) {
                  indexShard
                      .warmerService()
                      .logger()
                      .trace(
                          "warmed global ordinals for [{}], took [{}]",
                          fieldType.name(),
                          TimeValue.timeValueNanos(System.nanoTime() - start));
                }
              } catch (Exception e) {
                indexShard
                    .warmerService()
                    .logger()
                    .warn("failed to warm-up global ordinals for [{}]", e, fieldType.name());
              } finally {
                latch.countDown();
              }
            });
      }
      return () -> latch.await();
    }
Пример #2
0
  public IndexService(
      IndexSettings indexSettings,
      NodeEnvironment nodeEnv,
      SimilarityService similarityService,
      ShardStoreDeleter shardStoreDeleter,
      AnalysisRegistry registry,
      @Nullable EngineFactory engineFactory,
      NodeServicesProvider nodeServicesProvider,
      QueryCache queryCache,
      IndexStore indexStore,
      IndexEventListener eventListener,
      IndexModule.IndexSearcherWrapperFactory wrapperFactory,
      MapperRegistry mapperRegistry,
      IndexingOperationListener... listenersIn)
      throws IOException {
    super(indexSettings);
    this.indexSettings = indexSettings;
    this.analysisService = registry.build(indexSettings);
    this.similarityService = similarityService;
    this.mapperService =
        new MapperService(
            indexSettings,
            analysisService,
            similarityService,
            mapperRegistry,
            IndexService.this::getQueryShardContext);
    this.indexFieldData =
        new IndexFieldDataService(
            indexSettings,
            nodeServicesProvider.getIndicesFieldDataCache(),
            nodeServicesProvider.getCircuitBreakerService(),
            mapperService);
    this.shardStoreDeleter = shardStoreDeleter;
    this.eventListener = eventListener;
    this.nodeEnv = nodeEnv;
    this.nodeServicesProvider = nodeServicesProvider;
    this.indexStore = indexStore;
    indexFieldData.setListener(new FieldDataCacheListener(this));
    this.bitsetFilterCache =
        new BitsetFilterCache(
            indexSettings, nodeServicesProvider.getWarmer(), new BitsetCacheListener(this));
    this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache);
    this.engineFactory = engineFactory;
    // initialize this last -- otherwise if the wrapper requires any other member to be non-null we
    // fail with an NPE
    this.searcherWrapper = wrapperFactory.newWrapper(this);
    this.slowLog = new IndexingSlowLog(indexSettings.getSettings());

    // Add our slowLog to the incoming IndexingOperationListeners:
    this.listeners = new IndexingOperationListener[1 + listenersIn.length];
    this.listeners[0] = slowLog;
    System.arraycopy(listenersIn, 0, this.listeners, 1, listenersIn.length);
    // kick off async ops for the first shard in this index
    if (this.indexSettings.getTranslogSyncInterval().millis() != 0) {
      this.fsyncTask = new AsyncTranslogFSync(this);
    } else {
      this.fsyncTask = null;
    }
    this.refreshTask = new AsyncRefreshTask(this);
  }
 @After
 public void tearDown() throws Exception {
   super.tearDown();
   reader.close();
   writer.close();
   ifdService.clear();
   SearchContext.removeCurrent();
 }
Пример #4
0
 public IndexService(
     IndexSettings indexSettings,
     NodeEnvironment nodeEnv,
     SimilarityService similarityService,
     ShardStoreDeleter shardStoreDeleter,
     AnalysisRegistry registry,
     @Nullable EngineFactory engineFactory,
     NodeServicesProvider nodeServicesProvider,
     QueryCache queryCache,
     IndexStore indexStore,
     IndexEventListener eventListener,
     IndexModule.IndexSearcherWrapperFactory wrapperFactory,
     MapperRegistry mapperRegistry,
     IndicesFieldDataCache indicesFieldDataCache,
     List<SearchOperationListener> searchOperationListeners,
     List<IndexingOperationListener> indexingOperationListeners)
     throws IOException {
   super(indexSettings);
   this.indexSettings = indexSettings;
   this.analysisService = registry.build(indexSettings);
   this.similarityService = similarityService;
   this.mapperService =
       new MapperService(
           indexSettings,
           analysisService,
           similarityService,
           mapperRegistry,
           IndexService.this::newQueryShardContext);
   this.indexFieldData =
       new IndexFieldDataService(
           indexSettings,
           indicesFieldDataCache,
           nodeServicesProvider.getCircuitBreakerService(),
           mapperService);
   this.shardStoreDeleter = shardStoreDeleter;
   this.bigArrays = nodeServicesProvider.getBigArrays();
   this.threadPool = nodeServicesProvider.getThreadPool();
   this.eventListener = eventListener;
   this.nodeEnv = nodeEnv;
   this.nodeServicesProvider = nodeServicesProvider;
   this.indexStore = indexStore;
   indexFieldData.setListener(new FieldDataCacheListener(this));
   this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
   this.warmer =
       new IndexWarmer(
           indexSettings.getSettings(), threadPool, bitsetFilterCache.createListener(threadPool));
   this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache);
   this.engineFactory = engineFactory;
   // initialize this last -- otherwise if the wrapper requires any other member to be non-null we
   // fail with an NPE
   this.searcherWrapper = wrapperFactory.newWrapper(this);
   this.indexingOperationListeners = Collections.unmodifiableList(indexingOperationListeners);
   this.searchOperationListeners = Collections.unmodifiableList(searchOperationListeners);
   // kick off async ops for the first shard in this index
   this.refreshTask = new AsyncRefreshTask(this);
   rescheduleFsyncTask(indexSettings.getTranslogDurability());
 }
  @Before
  public void setup() throws Exception {
    super.setUp();

    // setup field mappers
    strMapper =
        new StringFieldMapper.Builder("str_value")
            .build(new Mapper.BuilderContext(null, new ContentPath(1)));

    lngMapper =
        new LongFieldMapper.Builder("lng_value")
            .build(new Mapper.BuilderContext(null, new ContentPath(1)));

    dblMapper =
        new DoubleFieldMapper.Builder("dbl_value")
            .build(new Mapper.BuilderContext(null, new ContentPath(1)));

    // create index and fielddata service
    ifdService = new IndexFieldDataService(new Index("test"), new DummyCircuitBreakerService());
    MapperService mapperService =
        MapperTestUtils.newMapperService(
            ifdService.index(), ImmutableSettings.Builder.EMPTY_SETTINGS);
    ifdService.setIndexService(new StubIndexService(mapperService));
    writer =
        new IndexWriter(
            new RAMDirectory(),
            new IndexWriterConfig(Lucene.VERSION, new StandardAnalyzer(Lucene.VERSION)));

    int numDocs = 10;
    for (int i = 0; i < numDocs; i++) {
      Document d = new Document();
      d.add(new StringField(strMapper.names().indexName(), "str" + i, Field.Store.NO));
      d.add(new LongField(lngMapper.names().indexName(), i, Field.Store.NO));
      d.add(new DoubleField(dblMapper.names().indexName(), Double.valueOf(i), Field.Store.NO));
      writer.addDocument(d);
    }

    reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(writer, true));
  }
Пример #6
0
 public <IFD extends IndexFieldData<?>> IFD getForField(FieldDataType type, String fieldName) {
   final FieldMapper<?> mapper;
   final BuilderContext context =
       new BuilderContext(indexService.settingsService().getSettings(), new ContentPath(1));
   if (type.getType().equals("string")) {
     mapper =
         MapperBuilders.stringField(fieldName)
             .tokenized(false)
             .fieldDataSettings(type.getSettings())
             .build(context);
   } else if (type.getType().equals("float")) {
     mapper =
         MapperBuilders.floatField(fieldName).fieldDataSettings(type.getSettings()).build(context);
   } else if (type.getType().equals("double")) {
     mapper =
         MapperBuilders.doubleField(fieldName)
             .fieldDataSettings(type.getSettings())
             .build(context);
   } else if (type.getType().equals("long")) {
     mapper =
         MapperBuilders.longField(fieldName).fieldDataSettings(type.getSettings()).build(context);
   } else if (type.getType().equals("int")) {
     mapper =
         MapperBuilders.integerField(fieldName)
             .fieldDataSettings(type.getSettings())
             .build(context);
   } else if (type.getType().equals("short")) {
     mapper =
         MapperBuilders.shortField(fieldName).fieldDataSettings(type.getSettings()).build(context);
   } else if (type.getType().equals("byte")) {
     mapper =
         MapperBuilders.byteField(fieldName).fieldDataSettings(type.getSettings()).build(context);
   } else if (type.getType().equals("geo_point")) {
     mapper =
         MapperBuilders.geoPointField(fieldName)
             .fieldDataSettings(type.getSettings())
             .build(context);
   } else if (type.getType().equals("_parent")) {
     mapper = MapperBuilders.parent().type(fieldName).build(context);
   } else if (type.getType().equals("binary")) {
     mapper =
         MapperBuilders.binaryField(fieldName)
             .fieldDataSettings(type.getSettings())
             .build(context);
   } else {
     throw new UnsupportedOperationException(type.getType());
   }
   return ifdService.getForField(mapper);
 }
 @Override
 public Filter rangeFilter(
     IndexFieldDataService fieldData,
     Object lowerTerm,
     Object upperTerm,
     boolean includeLower,
     boolean includeUpper,
     @Nullable QueryParseContext context) {
   return NumericRangeFieldDataFilter.newIntRange(
       (IndexNumericFieldData) fieldData.getForField(this),
       lowerTerm == null ? null : parseValue(lowerTerm),
       upperTerm == null ? null : parseValue(upperTerm),
       includeLower,
       includeUpper);
 }
 protected <IFD extends IndexNumericFieldData> IFD getFieldData(NumberFieldMapper fieldMapper) {
   return ifdService.getForField(fieldMapper);
 }