@Override
public synchronized void updateMetaData(final IndexMetaData metadata) {
final Translog.Durability oldTranslogDurability = indexSettings.getTranslogDurability();
if (indexSettings.updateIndexMetaData(metadata)) {
for (final IndexShard shard : this.shards.values()) {
try {
shard.onSettingsChanged();
} catch (Exception e) {
logger.warn("[{}] failed to notify shard about setting change", e, shard.shardId().id());
}
}
if (refreshTask.getInterval().equals(indexSettings.getRefreshInterval()) == false) {
rescheduleRefreshTasks();
}
final Translog.Durability durability = indexSettings.getTranslogDurability();
if (durability != oldTranslogDurability) {
rescheduleFsyncTask(durability);
}
}
// update primary terms
for (final IndexShard shard : this.shards.values()) {
shard.updatePrimaryTerm(metadata.primaryTerm(shard.shardId().id()));
}
}
public synchronized void updateMetaData(final IndexMetaData metadata) {
if (indexSettings.updateIndexMetaData(metadata)) {
final Settings settings = indexSettings.getSettings();
for (final IndexShard shard : this.shards.values()) {
try {
shard.onRefreshSettings(settings);
} catch (Exception e) {
logger.warn("[{}] failed to refresh shard settings", e, shard.shardId().id());
}
}
try {
indexStore.onRefreshSettings(settings);
} catch (Exception e) {
logger.warn("failed to refresh index store settings", e);
}
try {
slowLog.onRefreshSettings(
settings); // this will be refactored soon anyway so duplication is ok here
} catch (Exception e) {
logger.warn("failed to refresh slowlog settings", e);
}
if (refreshTask.getInterval().equals(indexSettings.getRefreshInterval()) == false) {
rescheduleRefreshTasks();
}
}
}
public IndexService(
IndexSettings indexSettings,
NodeEnvironment nodeEnv,
SimilarityService similarityService,
ShardStoreDeleter shardStoreDeleter,
AnalysisRegistry registry,
@Nullable EngineFactory engineFactory,
NodeServicesProvider nodeServicesProvider,
QueryCache queryCache,
IndexStore indexStore,
IndexEventListener eventListener,
IndexModule.IndexSearcherWrapperFactory wrapperFactory,
MapperRegistry mapperRegistry,
IndicesFieldDataCache indicesFieldDataCache,
List<SearchOperationListener> searchOperationListeners,
List<IndexingOperationListener> indexingOperationListeners)
throws IOException {
super(indexSettings);
this.indexSettings = indexSettings;
this.analysisService = registry.build(indexSettings);
this.similarityService = similarityService;
this.mapperService =
new MapperService(
indexSettings,
analysisService,
similarityService,
mapperRegistry,
IndexService.this::newQueryShardContext);
this.indexFieldData =
new IndexFieldDataService(
indexSettings,
indicesFieldDataCache,
nodeServicesProvider.getCircuitBreakerService(),
mapperService);
this.shardStoreDeleter = shardStoreDeleter;
this.bigArrays = nodeServicesProvider.getBigArrays();
this.threadPool = nodeServicesProvider.getThreadPool();
this.eventListener = eventListener;
this.nodeEnv = nodeEnv;
this.nodeServicesProvider = nodeServicesProvider;
this.indexStore = indexStore;
indexFieldData.setListener(new FieldDataCacheListener(this));
this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
this.warmer =
new IndexWarmer(
indexSettings.getSettings(), threadPool, bitsetFilterCache.createListener(threadPool));
this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache);
this.engineFactory = engineFactory;
// initialize this last -- otherwise if the wrapper requires any other member to be non-null we
// fail with an NPE
this.searcherWrapper = wrapperFactory.newWrapper(this);
this.indexingOperationListeners = Collections.unmodifiableList(indexingOperationListeners);
this.searchOperationListeners = Collections.unmodifiableList(searchOperationListeners);
// kick off async ops for the first shard in this index
this.refreshTask = new AsyncRefreshTask(this);
rescheduleFsyncTask(indexSettings.getTranslogDurability());
}
public IndexService newIndexService(
NodeEnvironment environment,
IndexService.ShardStoreDeleter shardStoreDeleter,
NodeServicesProvider servicesProvider,
MapperRegistry mapperRegistry,
IndexingOperationListener... listeners)
throws IOException {
IndexSearcherWrapperFactory searcherWrapperFactory =
indexSearcherWrapper.get() == null ? (shard) -> null : indexSearcherWrapper.get();
IndexEventListener eventListener = freeze();
final String storeType = indexSettings.getValue(INDEX_STORE_TYPE_SETTING);
final IndexStore store;
if (Strings.isEmpty(storeType) || isBuiltinType(storeType)) {
store = new IndexStore(indexSettings, indexStoreConfig);
} else {
BiFunction<IndexSettings, IndexStoreConfig, IndexStore> factory = storeTypes.get(storeType);
if (factory == null) {
throw new IllegalArgumentException("Unknown store type [" + storeType + "]");
}
store = factory.apply(indexSettings, indexStoreConfig);
if (store == null) {
throw new IllegalStateException("store must not be null");
}
}
indexSettings
.getScopedSettings()
.addSettingsUpdateConsumer(
IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING, store::setMaxRate);
indexSettings
.getScopedSettings()
.addSettingsUpdateConsumer(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING, store::setType);
final String queryCacheType = indexSettings.getValue(INDEX_QUERY_CACHE_TYPE_SETTING);
final BiFunction<IndexSettings, IndicesQueryCache, QueryCache> queryCacheProvider =
queryCaches.get(queryCacheType);
final QueryCache queryCache =
queryCacheProvider.apply(indexSettings, servicesProvider.getIndicesQueryCache());
return new IndexService(
indexSettings,
environment,
new SimilarityService(indexSettings, similarities),
shardStoreDeleter,
analysisRegistry,
engineFactory.get(),
servicesProvider,
queryCache,
store,
eventListener,
searcherWrapperFactory,
mapperRegistry,
listeners);
}
public IndexService(
IndexSettings indexSettings,
NodeEnvironment nodeEnv,
SimilarityService similarityService,
ShardStoreDeleter shardStoreDeleter,
AnalysisRegistry registry,
@Nullable EngineFactory engineFactory,
NodeServicesProvider nodeServicesProvider,
QueryCache queryCache,
IndexStore indexStore,
IndexEventListener eventListener,
IndexModule.IndexSearcherWrapperFactory wrapperFactory,
MapperRegistry mapperRegistry,
IndexingOperationListener... listenersIn)
throws IOException {
super(indexSettings);
this.indexSettings = indexSettings;
this.analysisService = registry.build(indexSettings);
this.similarityService = similarityService;
this.mapperService =
new MapperService(
indexSettings,
analysisService,
similarityService,
mapperRegistry,
IndexService.this::getQueryShardContext);
this.indexFieldData =
new IndexFieldDataService(
indexSettings,
nodeServicesProvider.getIndicesFieldDataCache(),
nodeServicesProvider.getCircuitBreakerService(),
mapperService);
this.shardStoreDeleter = shardStoreDeleter;
this.eventListener = eventListener;
this.nodeEnv = nodeEnv;
this.nodeServicesProvider = nodeServicesProvider;
this.indexStore = indexStore;
indexFieldData.setListener(new FieldDataCacheListener(this));
this.bitsetFilterCache =
new BitsetFilterCache(
indexSettings, nodeServicesProvider.getWarmer(), new BitsetCacheListener(this));
this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache);
this.engineFactory = engineFactory;
// initialize this last -- otherwise if the wrapper requires any other member to be non-null we
// fail with an NPE
this.searcherWrapper = wrapperFactory.newWrapper(this);
this.slowLog = new IndexingSlowLog(indexSettings.getSettings());
// Add our slowLog to the incoming IndexingOperationListeners:
this.listeners = new IndexingOperationListener[1 + listenersIn.length];
this.listeners[0] = slowLog;
System.arraycopy(listenersIn, 0, this.listeners, 1, listenersIn.length);
// kick off async ops for the first shard in this index
if (this.indexSettings.getTranslogSyncInterval().millis() != 0) {
this.fsyncTask = new AsyncTranslogFSync(this);
} else {
this.fsyncTask = null;
}
this.refreshTask = new AsyncRefreshTask(this);
}
private void maybeRefreshEngine() {
if (indexSettings.getRefreshInterval().millis() > 0) {
for (IndexShard shard : this.shards.values()) {
switch (shard.state()) {
case CREATED:
case RECOVERING:
case CLOSED:
continue;
case POST_RECOVERY:
case STARTED:
case RELOCATED:
try {
if (shard.isRefreshNeeded()) {
shard.refresh("schedule");
}
} catch (EngineClosedException | AlreadyClosedException ex) {
// fine - continue;
}
continue;
default:
throw new IllegalStateException("unknown state: " + shard.state());
}
}
}
}
private void onShardClose(ShardLock lock, boolean ownsShard) {
if (deleted.get()) { // we remove that shards content if this index has been deleted
try {
if (ownsShard) {
try {
eventListener.beforeIndexShardDeleted(lock.getShardId(), indexSettings.getSettings());
} finally {
shardStoreDeleter.deleteShardStore("delete index", lock, indexSettings);
eventListener.afterIndexShardDeleted(lock.getShardId(), indexSettings.getSettings());
}
}
} catch (IOException e) {
shardStoreDeleter.addPendingDelete(lock.getShardId(), indexSettings);
logger.debug(
"[{}] failed to delete shard content - scheduled a retry", e, lock.getShardId().id());
}
}
}
public IndexService newIndexService(
NodeEnvironment environment,
IndexService.ShardStoreDeleter shardStoreDeleter,
NodeServicesProvider servicesProvider)
throws IOException {
final IndexSettings settings = indexSettings.newWithListener(settingsConsumers);
IndexSearcherWrapperFactory searcherWrapperFactory =
indexSearcherWrapper.get() == null ? (shard) -> null : indexSearcherWrapper.get();
IndexEventListener eventListener = freeze();
final String storeType = settings.getSettings().get(STORE_TYPE);
final IndexStore store;
if (storeType == null || isBuiltinType(storeType)) {
store = new IndexStore(settings, indexStoreConfig);
} else {
BiFunction<IndexSettings, IndexStoreConfig, IndexStore> factory = storeTypes.get(storeType);
if (factory == null) {
throw new IllegalArgumentException("Unknown store type [" + storeType + "]");
}
store = factory.apply(settings, indexStoreConfig);
if (store == null) {
throw new IllegalStateException("store must not be null");
}
}
final String queryCacheType =
settings.getSettings().get(IndexModule.QUERY_CACHE_TYPE, IndexModule.INDEX_QUERY_CACHE);
final BiFunction<IndexSettings, IndicesQueryCache, QueryCache> queryCacheProvider =
queryCaches.get(queryCacheType);
final QueryCache queryCache =
queryCacheProvider.apply(settings, servicesProvider.getIndicesQueryCache());
return new IndexService(
settings,
environment,
new SimilarityService(settings, similarities),
shardStoreDeleter,
analysisRegistry,
engineFactory.get(),
servicesProvider,
queryCache,
store,
eventListener,
searcherWrapperFactory);
}
/**
* Returns the filter associated with listed filtering aliases.
*
* <p>The list of filtering aliases should be obtained by calling MetaData.filteringAliases.
* Returns <tt>null</tt> if no filtering is required.
*/
public Query aliasFilter(QueryShardContext context, String... aliasNames) {
if (aliasNames == null || aliasNames.length == 0) {
return null;
}
final ImmutableOpenMap<String, AliasMetaData> aliases =
indexSettings.getIndexMetaData().getAliases();
if (aliasNames.length == 1) {
AliasMetaData alias = aliases.get(aliasNames[0]);
if (alias == null) {
// This shouldn't happen unless alias disappeared after filteringAliases was called.
throw new InvalidAliasNameException(
index(), aliasNames[0], "Unknown alias name was passed to alias Filter");
}
return parse(alias, context);
} else {
// we need to bench here a bit, to see maybe it makes sense to use OrFilter
BooleanQuery.Builder combined = new BooleanQuery.Builder();
for (String aliasName : aliasNames) {
AliasMetaData alias = aliases.get(aliasName);
if (alias == null) {
// This shouldn't happen unless alias disappeared after filteringAliases was called.
throw new InvalidAliasNameException(
indexSettings.getIndex(),
aliasNames[0],
"Unknown alias name was passed to alias Filter");
}
Query parsedFilter = parse(alias, context);
if (parsedFilter != null) {
combined.add(parsedFilter, BooleanClause.Occur.SHOULD);
} else {
// The filter might be null only if filter was removed after filteringAliases was called
return null;
}
}
return combined.build();
}
}
private void maybeFSyncTranslogs() {
if (indexSettings.getTranslogDurability() == Translog.Durability.ASYNC) {
for (IndexShard shard : this.shards.values()) {
try {
Translog translog = shard.getTranslog();
if (translog.syncNeeded()) {
translog.sync();
}
} catch (EngineClosedException | AlreadyClosedException ex) {
// fine - continue;
} catch (IOException e) {
logger.warn("failed to sync translog", e);
}
}
}
}
void warm(Engine.Searcher searcher, IndexShard shard, IndexSettings settings) {
if (shard.state() == IndexShardState.CLOSED) {
return;
}
if (settings.isWarmerEnabled() == false) {
return;
}
if (logger.isTraceEnabled()) {
logger.trace("{} top warming [{}]", shard.shardId(), searcher.reader());
}
shard.warmerService().onPreWarm();
long time = System.nanoTime();
final List<TerminationHandle> terminationHandles = new ArrayList<>();
// get a handle on pending tasks
for (final Listener listener : listeners) {
terminationHandles.add(listener.warmReader(shard, searcher));
}
// wait for termination
for (TerminationHandle terminationHandle : terminationHandles) {
try {
terminationHandle.awaitTermination();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
logger.warn("top warming has been interrupted", e);
break;
}
}
long took = System.nanoTime() - time;
shard.warmerService().onPostWarm(took);
if (shard.warmerService().logger().isTraceEnabled()) {
shard
.warmerService()
.logger()
.trace("top warming took [{}]", new TimeValue(took, TimeUnit.NANOSECONDS));
}
}
/** Adds a Setting and it's consumer for this index. */
public <T> void addSettingsUpdateConsumer(Setting<T> setting, Consumer<T> consumer) {
if (setting == null) {
throw new IllegalArgumentException("setting must not be null");
}
indexSettings.getScopedSettings().addSettingsUpdateConsumer(setting, consumer);
}
/** Returns the index {@link Settings} for this index */
public Settings getSettings() {
return indexSettings.getSettings();
}
/** Returns the index this module is associated with */
public Index getIndex() {
return indexSettings.getIndex();
}
public IndexMetaData getMetaData() {
return indexSettings.getIndexMetaData();
}
MergeSchedulerConfig(IndexSettings indexSettings) {
maxThreadCount = indexSettings.getValue(MAX_THREAD_COUNT_SETTING);
maxMergeCount = indexSettings.getValue(MAX_MERGE_COUNT_SETTING);
this.autoThrottle = indexSettings.getValue(AUTO_THROTTLE_SETTING);
}
public String indexUUID() {
return indexSettings.getUUID();
}