public IndexService(
IndexSettings indexSettings,
NodeEnvironment nodeEnv,
SimilarityService similarityService,
ShardStoreDeleter shardStoreDeleter,
AnalysisRegistry registry,
@Nullable EngineFactory engineFactory,
NodeServicesProvider nodeServicesProvider,
QueryCache queryCache,
IndexStore indexStore,
IndexEventListener eventListener,
IndexModule.IndexSearcherWrapperFactory wrapperFactory,
MapperRegistry mapperRegistry,
IndexingOperationListener... listenersIn)
throws IOException {
super(indexSettings);
this.indexSettings = indexSettings;
this.analysisService = registry.build(indexSettings);
this.similarityService = similarityService;
this.mapperService =
new MapperService(
indexSettings,
analysisService,
similarityService,
mapperRegistry,
IndexService.this::getQueryShardContext);
this.indexFieldData =
new IndexFieldDataService(
indexSettings,
nodeServicesProvider.getIndicesFieldDataCache(),
nodeServicesProvider.getCircuitBreakerService(),
mapperService);
this.shardStoreDeleter = shardStoreDeleter;
this.eventListener = eventListener;
this.nodeEnv = nodeEnv;
this.nodeServicesProvider = nodeServicesProvider;
this.indexStore = indexStore;
indexFieldData.setListener(new FieldDataCacheListener(this));
this.bitsetFilterCache =
new BitsetFilterCache(
indexSettings, nodeServicesProvider.getWarmer(), new BitsetCacheListener(this));
this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache);
this.engineFactory = engineFactory;
// initialize this last -- otherwise if the wrapper requires any other member to be non-null we
// fail with an NPE
this.searcherWrapper = wrapperFactory.newWrapper(this);
this.slowLog = new IndexingSlowLog(indexSettings.getSettings());
// Add our slowLog to the incoming IndexingOperationListeners:
this.listeners = new IndexingOperationListener[1 + listenersIn.length];
this.listeners[0] = slowLog;
System.arraycopy(listenersIn, 0, this.listeners, 1, listenersIn.length);
// kick off async ops for the first shard in this index
if (this.indexSettings.getTranslogSyncInterval().millis() != 0) {
this.fsyncTask = new AsyncTranslogFSync(this);
} else {
this.fsyncTask = null;
}
this.refreshTask = new AsyncRefreshTask(this);
}
public IndexService(
IndexSettings indexSettings,
NodeEnvironment nodeEnv,
SimilarityService similarityService,
ShardStoreDeleter shardStoreDeleter,
AnalysisRegistry registry,
@Nullable EngineFactory engineFactory,
NodeServicesProvider nodeServicesProvider,
QueryCache queryCache,
IndexStore indexStore,
IndexEventListener eventListener,
IndexModule.IndexSearcherWrapperFactory wrapperFactory,
MapperRegistry mapperRegistry,
IndicesFieldDataCache indicesFieldDataCache,
List<SearchOperationListener> searchOperationListeners,
List<IndexingOperationListener> indexingOperationListeners)
throws IOException {
super(indexSettings);
this.indexSettings = indexSettings;
this.analysisService = registry.build(indexSettings);
this.similarityService = similarityService;
this.mapperService =
new MapperService(
indexSettings,
analysisService,
similarityService,
mapperRegistry,
IndexService.this::newQueryShardContext);
this.indexFieldData =
new IndexFieldDataService(
indexSettings,
indicesFieldDataCache,
nodeServicesProvider.getCircuitBreakerService(),
mapperService);
this.shardStoreDeleter = shardStoreDeleter;
this.bigArrays = nodeServicesProvider.getBigArrays();
this.threadPool = nodeServicesProvider.getThreadPool();
this.eventListener = eventListener;
this.nodeEnv = nodeEnv;
this.nodeServicesProvider = nodeServicesProvider;
this.indexStore = indexStore;
indexFieldData.setListener(new FieldDataCacheListener(this));
this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
this.warmer =
new IndexWarmer(
indexSettings.getSettings(), threadPool, bitsetFilterCache.createListener(threadPool));
this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache);
this.engineFactory = engineFactory;
// initialize this last -- otherwise if the wrapper requires any other member to be non-null we
// fail with an NPE
this.searcherWrapper = wrapperFactory.newWrapper(this);
this.indexingOperationListeners = Collections.unmodifiableList(indexingOperationListeners);
this.searchOperationListeners = Collections.unmodifiableList(searchOperationListeners);
// kick off async ops for the first shard in this index
this.refreshTask = new AsyncRefreshTask(this);
rescheduleFsyncTask(indexSettings.getTranslogDurability());
}
public QueryShardContext getQueryShardContext() {
return new QueryShardContext(
indexSettings,
nodeServicesProvider.getClient(),
indexCache.bitsetFilterCache(),
indexFieldData,
mapperService(),
similarityService(),
nodeServicesProvider.getScriptService(),
nodeServicesProvider.getIndicesQueriesRegistry());
}
public IndexService newIndexService(
NodeEnvironment environment,
IndexService.ShardStoreDeleter shardStoreDeleter,
NodeServicesProvider servicesProvider,
MapperRegistry mapperRegistry,
IndexingOperationListener... listeners)
throws IOException {
IndexSearcherWrapperFactory searcherWrapperFactory =
indexSearcherWrapper.get() == null ? (shard) -> null : indexSearcherWrapper.get();
IndexEventListener eventListener = freeze();
final String storeType = indexSettings.getValue(INDEX_STORE_TYPE_SETTING);
final IndexStore store;
if (Strings.isEmpty(storeType) || isBuiltinType(storeType)) {
store = new IndexStore(indexSettings, indexStoreConfig);
} else {
BiFunction<IndexSettings, IndexStoreConfig, IndexStore> factory = storeTypes.get(storeType);
if (factory == null) {
throw new IllegalArgumentException("Unknown store type [" + storeType + "]");
}
store = factory.apply(indexSettings, indexStoreConfig);
if (store == null) {
throw new IllegalStateException("store must not be null");
}
}
indexSettings
.getScopedSettings()
.addSettingsUpdateConsumer(
IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING, store::setMaxRate);
indexSettings
.getScopedSettings()
.addSettingsUpdateConsumer(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING, store::setType);
final String queryCacheType = indexSettings.getValue(INDEX_QUERY_CACHE_TYPE_SETTING);
final BiFunction<IndexSettings, IndicesQueryCache, QueryCache> queryCacheProvider =
queryCaches.get(queryCacheType);
final QueryCache queryCache =
queryCacheProvider.apply(indexSettings, servicesProvider.getIndicesQueryCache());
return new IndexService(
indexSettings,
environment,
new SimilarityService(indexSettings, similarities),
shardStoreDeleter,
analysisRegistry,
engineFactory.get(),
servicesProvider,
queryCache,
store,
eventListener,
searcherWrapperFactory,
mapperRegistry,
listeners);
}
public IndexService newIndexService(
NodeEnvironment environment,
IndexService.ShardStoreDeleter shardStoreDeleter,
NodeServicesProvider servicesProvider)
throws IOException {
final IndexSettings settings = indexSettings.newWithListener(settingsConsumers);
IndexSearcherWrapperFactory searcherWrapperFactory =
indexSearcherWrapper.get() == null ? (shard) -> null : indexSearcherWrapper.get();
IndexEventListener eventListener = freeze();
final String storeType = settings.getSettings().get(STORE_TYPE);
final IndexStore store;
if (storeType == null || isBuiltinType(storeType)) {
store = new IndexStore(settings, indexStoreConfig);
} else {
BiFunction<IndexSettings, IndexStoreConfig, IndexStore> factory = storeTypes.get(storeType);
if (factory == null) {
throw new IllegalArgumentException("Unknown store type [" + storeType + "]");
}
store = factory.apply(settings, indexStoreConfig);
if (store == null) {
throw new IllegalStateException("store must not be null");
}
}
final String queryCacheType =
settings.getSettings().get(IndexModule.QUERY_CACHE_TYPE, IndexModule.INDEX_QUERY_CACHE);
final BiFunction<IndexSettings, IndicesQueryCache, QueryCache> queryCacheProvider =
queryCaches.get(queryCacheType);
final QueryCache queryCache =
queryCacheProvider.apply(settings, servicesProvider.getIndicesQueryCache());
return new IndexService(
settings,
environment,
new SimilarityService(settings, similarities),
shardStoreDeleter,
analysisRegistry,
engineFactory.get(),
servicesProvider,
queryCache,
store,
eventListener,
searcherWrapperFactory);
}
ThreadPool getThreadPool() {
return nodeServicesProvider.getThreadPool();
}
public synchronized IndexShard createShard(ShardRouting routing) throws IOException {
final boolean primary = routing.primary();
/*
* TODO: we execute this in parallel but it's a synced method. Yet, we might
* be able to serialize the execution via the cluster state in the future. for now we just
* keep it synced.
*/
if (closed.get()) {
throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed");
}
final Settings indexSettings = this.indexSettings.getSettings();
final ShardId shardId = routing.shardId();
boolean success = false;
Store store = null;
IndexShard indexShard = null;
final ShardLock lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5));
try {
eventListener.beforeIndexShardCreated(shardId, indexSettings);
ShardPath path;
try {
path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
} catch (IllegalStateException ex) {
logger.warn("{} failed to load shard path, trying to remove leftover", shardId);
try {
ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings);
path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
} catch (Throwable t) {
t.addSuppressed(ex);
throw t;
}
}
if (path == null) {
// TODO: we should, instead, hold a "bytes reserved" of how large we anticipate this shard
// will be, e.g. for a shard
// that's being relocated/replicated we know how large it will become once it's done
// copying:
// Count up how many shards are currently on each data path:
Map<Path, Integer> dataPathToShardCount = new HashMap<>();
for (IndexShard shard : this) {
Path dataPath = shard.shardPath().getRootStatePath();
Integer curCount = dataPathToShardCount.get(dataPath);
if (curCount == null) {
curCount = 0;
}
dataPathToShardCount.put(dataPath, curCount + 1);
}
path =
ShardPath.selectNewPathForShard(
nodeEnv,
shardId,
this.indexSettings,
routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE
? getAvgShardSizeInBytes()
: routing.getExpectedShardSize(),
dataPathToShardCount);
logger.debug("{} creating using a new path [{}]", shardId, path);
} else {
logger.debug("{} creating using an existing path [{}]", shardId, path);
}
if (shards.containsKey(shardId.id())) {
throw new IndexShardAlreadyExistsException(shardId + " already exists");
}
logger.debug("creating shard_id {}", shardId);
// if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the
// primary.
final boolean canDeleteShardContent =
IndexMetaData.isOnSharedFilesystem(indexSettings) == false
|| (primary && IndexMetaData.isOnSharedFilesystem(indexSettings));
store =
new Store(
shardId,
this.indexSettings,
indexStore.newDirectoryService(path),
lock,
new StoreCloseListener(
shardId,
canDeleteShardContent,
() -> nodeServicesProvider.getIndicesQueryCache().onClose(shardId)));
if (useShadowEngine(primary, indexSettings)) {
indexShard =
new ShadowIndexShard(
shardId,
this.indexSettings,
path,
store,
indexCache,
mapperService,
similarityService,
indexFieldData,
engineFactory,
eventListener,
searcherWrapper,
nodeServicesProvider); // no indexing listeners - shadow engines don't index
} else {
indexShard =
new IndexShard(
shardId,
this.indexSettings,
path,
store,
indexCache,
mapperService,
similarityService,
indexFieldData,
engineFactory,
eventListener,
searcherWrapper,
nodeServicesProvider,
listeners);
}
eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
eventListener.afterIndexShardCreated(indexShard);
indexShard.updateRoutingEntry(routing, true);
shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
success = true;
return indexShard;
} finally {
if (success == false) {
IOUtils.closeWhileHandlingException(lock);
closeShard("initialization failed", shardId, indexShard, store, eventListener);
}
}
}
