public synchronized void updateMetaData(final IndexMetaData metadata) {
if (indexSettings.updateIndexMetaData(metadata)) {
final Settings settings = indexSettings.getSettings();
for (final IndexShard shard : this.shards.values()) {
try {
shard.onRefreshSettings(settings);
} catch (Exception e) {
logger.warn("[{}] failed to refresh shard settings", e, shard.shardId().id());
}
}
try {
indexStore.onRefreshSettings(settings);
} catch (Exception e) {
logger.warn("failed to refresh index store settings", e);
}
try {
slowLog.onRefreshSettings(
settings); // this will be refactored soon anyway so duplication is ok here
} catch (Exception e) {
logger.warn("failed to refresh slowlog settings", e);
}
if (refreshTask.getInterval().equals(indexSettings.getRefreshInterval()) == false) {
rescheduleRefreshTasks();
}
}
}
private FsDirectoryService randomDirectorService(IndexStore indexStore, ShardPath path) {
final IndexSettings indexSettings = indexStore.getIndexSettings();
final IndexMetaData build =
IndexMetaData.builder(indexSettings.getIndexMetaData())
.settings(
Settings.builder()
.put(indexSettings.getSettings())
.put(
IndexStoreModule.STORE_TYPE,
RandomPicks.randomFrom(random, IndexStoreModule.Type.values())
.getSettingsKey()))
.build();
final IndexSettings newIndexSettings =
new IndexSettings(build, indexSettings.getNodeSettings(), Collections.EMPTY_LIST);
return new FsDirectoryService(newIndexSettings, indexStore, path);
}
public synchronized IndexShard createShard(ShardRouting routing) throws IOException {
final boolean primary = routing.primary();
/*
* TODO: we execute this in parallel but it's a synced method. Yet, we might
* be able to serialize the execution via the cluster state in the future. for now we just
* keep it synced.
*/
if (closed.get()) {
throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed");
}
final Settings indexSettings = this.indexSettings.getSettings();
final ShardId shardId = routing.shardId();
boolean success = false;
Store store = null;
IndexShard indexShard = null;
ShardLock lock = null;
try {
lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5));
eventListener.beforeIndexShardCreated(shardId, indexSettings);
ShardPath path;
try {
path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
} catch (IllegalStateException ex) {
logger.warn("{} failed to load shard path, trying to remove leftover", shardId);
try {
ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings);
path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
} catch (Exception inner) {
ex.addSuppressed(inner);
throw ex;
}
}
if (path == null) {
// TODO: we should, instead, hold a "bytes reserved" of how large we anticipate this shard
// will be, e.g. for a shard
// that's being relocated/replicated we know how large it will become once it's done
// copying:
// Count up how many shards are currently on each data path:
Map<Path, Integer> dataPathToShardCount = new HashMap<>();
for (IndexShard shard : this) {
Path dataPath = shard.shardPath().getRootStatePath();
Integer curCount = dataPathToShardCount.get(dataPath);
if (curCount == null) {
curCount = 0;
}
dataPathToShardCount.put(dataPath, curCount + 1);
}
path =
ShardPath.selectNewPathForShard(
nodeEnv,
shardId,
this.indexSettings,
routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE
? getAvgShardSizeInBytes()
: routing.getExpectedShardSize(),
dataPathToShardCount);
logger.debug("{} creating using a new path [{}]", shardId, path);
} else {
logger.debug("{} creating using an existing path [{}]", shardId, path);
}
if (shards.containsKey(shardId.id())) {
throw new IndexShardAlreadyExistsException(shardId + " already exists");
}
logger.debug("creating shard_id {}", shardId);
// if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the
// primary.
final boolean canDeleteShardContent =
IndexMetaData.isOnSharedFilesystem(indexSettings) == false
|| (primary && IndexMetaData.isOnSharedFilesystem(indexSettings));
final Engine.Warmer engineWarmer =
(searcher) -> {
IndexShard shard = getShardOrNull(shardId.getId());
if (shard != null) {
warmer.warm(searcher, shard, IndexService.this.indexSettings);
}
};
store =
new Store(
shardId,
this.indexSettings,
indexStore.newDirectoryService(path),
lock,
new StoreCloseListener(
shardId, canDeleteShardContent, () -> eventListener.onStoreClosed(shardId)));
if (useShadowEngine(primary, indexSettings)) {
indexShard =
new ShadowIndexShard(
routing,
this.indexSettings,
path,
store,
indexCache,
mapperService,
similarityService,
indexFieldData,
engineFactory,
eventListener,
searcherWrapper,
threadPool,
bigArrays,
engineWarmer,
searchOperationListeners);
// no indexing listeners - shadow engines don't index
} else {
indexShard =
new IndexShard(
routing,
this.indexSettings,
path,
store,
indexCache,
mapperService,
similarityService,
indexFieldData,
engineFactory,
eventListener,
searcherWrapper,
threadPool,
bigArrays,
engineWarmer,
searchOperationListeners,
indexingOperationListeners);
}
eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
eventListener.afterIndexShardCreated(indexShard);
shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
success = true;
return indexShard;
} catch (ShardLockObtainFailedException e) {
throw new IOException("failed to obtain in-memory shard lock", e);
} finally {
if (success == false) {
if (lock != null) {
IOUtils.closeWhileHandlingException(lock);
}
closeShard("initialization failed", shardId, indexShard, store, eventListener);
}
}
}
private void deleteShard(
int shardId, boolean delete, boolean snapshotGateway, boolean deleteGateway)
throws ElasticSearchException {
Injector shardInjector;
IndexShard indexShard;
synchronized (this) {
Map<Integer, Injector> tmpShardInjectors = newHashMap(shardsInjectors);
shardInjector = tmpShardInjectors.remove(shardId);
if (shardInjector == null) {
if (!delete) {
return;
}
throw new IndexShardMissingException(new ShardId(index, shardId));
}
shardsInjectors = ImmutableMap.copyOf(tmpShardInjectors);
if (delete) {
logger.debug("deleting shard_id [{}]", shardId);
}
Map<Integer, IndexShard> tmpShardsMap = newHashMap(shards);
indexShard = tmpShardsMap.remove(shardId);
shards = ImmutableMap.copyOf(tmpShardsMap);
}
ShardId sId = new ShardId(index, shardId);
indicesLifecycle.beforeIndexShardClosed(sId, indexShard, delete);
for (Class<? extends CloseableIndexComponent> closeable : pluginsService.shardServices()) {
try {
shardInjector.getInstance(closeable).close(delete);
} catch (Exception e) {
logger.debug("failed to clean plugin shard service [{}]", e, closeable);
}
}
try {
// now we can close the translog service, we need to close it before the we close the shard
shardInjector.getInstance(TranslogService.class).close();
} catch (Exception e) {
// ignore
}
// close shard actions
if (indexShard != null) {
shardInjector.getInstance(IndexShardManagement.class).close();
}
// this logic is tricky, we want to close the engine so we rollback the changes done to it
// and close the shard so no operations are allowed to it
if (indexShard != null) {
indexShard.close();
}
try {
shardInjector.getInstance(Engine.class).close();
} catch (Exception e) {
// ignore
}
try {
// now, we can snapshot to the gateway, it will be only the translog
if (snapshotGateway) {
shardInjector.getInstance(IndexShardGatewayService.class).snapshotOnClose();
}
} catch (Exception e) {
// ignore
}
try {
shardInjector.getInstance(IndexShardGatewayService.class).close(deleteGateway);
} catch (Exception e) {
// ignore
}
try {
// now we can close the translog
shardInjector.getInstance(Translog.class).close(delete);
} catch (Exception e) {
// ignore
}
// call this before we close the store, so we can release resources for it
indicesLifecycle.afterIndexShardClosed(sId, delete);
// if we delete or have no gateway or the store is not persistent, clean the store...
Store store = shardInjector.getInstance(Store.class);
if (delete || indexGateway.type().equals(NoneGateway.TYPE) || !indexStore.persistent()) {
try {
store.fullDelete();
} catch (IOException e) {
logger.warn("failed to clean store on shard deletion", e);
}
}
// and close it
try {
store.close();
} catch (IOException e) {
logger.warn("failed to close store on shard deletion", e);
}
Injectors.close(injector);
// delete the shard location if needed
if (delete || indexGateway.type().equals(NoneGateway.TYPE)) {
FileSystemUtils.deleteRecursively(nodeEnv.shardLocation(sId));
}
}