public synchronized void startAll() throws IOException {
final DiscoveryNode pNode = getDiscoveryNode(primary.routingEntry().currentNodeId());
primary.markAsRecovering(
"store",
new RecoveryState(primary.shardId(), true, RecoveryState.Type.STORE, pNode, pNode));
primary.recoverFromStore();
primary.updateRoutingEntry(ShardRoutingHelper.moveToStarted(primary.routingEntry()));
for (IndexShard replicaShard : replicas) {
recoverReplica(
replicaShard,
(replica, sourceNode) ->
new RecoveryTarget(replica, sourceNode, recoveryListener, version -> {}));
}
}
public void recoverReplica(
IndexShard replica,
BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier,
boolean markAsRecovering)
throws IOException {
final DiscoveryNode pNode = getPrimaryNode();
final DiscoveryNode rNode = getDiscoveryNode(replica.routingEntry().currentNodeId());
if (markAsRecovering) {
replica.markAsRecovering(
"remote",
new RecoveryState(replica.shardId(), false, RecoveryState.Type.REPLICA, pNode, rNode));
} else {
assertEquals(replica.state(), IndexShardState.RECOVERING);
}
replica.prepareForIndexRecovery();
RecoveryTarget recoveryTarget = targetSupplier.apply(replica, pNode);
StartRecoveryRequest request =
new StartRecoveryRequest(
replica.shardId(),
pNode,
rNode,
getMetadataSnapshotOrEmpty(replica),
RecoveryState.Type.REPLICA,
0);
RecoverySourceHandler recovery =
new RecoverySourceHandler(
primary,
recoveryTarget,
request,
() -> 0L,
e -> () -> {},
(int) ByteSizeUnit.MB.toKB(1),
logger);
recovery.recoverToTarget();
recoveryTarget.markAsDone();
replica.updateRoutingEntry(ShardRoutingHelper.moveToStarted(replica.routingEntry()));
}
private void applyInitializingShard(
final ClusterState state,
final IndexMetaData indexMetaData,
final ShardRouting shardRouting) {
final IndexService indexService = indicesService.indexService(shardRouting.index());
if (indexService == null) {
// got deleted on us, ignore
return;
}
final RoutingTable routingTable = state.routingTable();
final DiscoveryNodes nodes = state.getNodes();
final int shardId = shardRouting.id();
if (indexService.hasShard(shardId)) {
IndexShard indexShard = indexService.shardSafe(shardId);
if (indexShard.state() == IndexShardState.STARTED
|| indexShard.state() == IndexShardState.POST_RECOVERY) {
// the master thinks we are initializing, but we are already started or on POST_RECOVERY and
// waiting
// for master to confirm a shard started message (either master failover, or a cluster event
// before
// we managed to tell the master we started), mark us as started
if (logger.isTraceEnabled()) {
logger.trace(
"{} master marked shard as initializing, but shard has state [{}], resending shard started to {}",
indexShard.shardId(),
indexShard.state(),
nodes.masterNode());
}
if (nodes.masterNode() != null) {
shardStateAction.shardStarted(
shardRouting,
indexMetaData.getIndexUUID(),
"master "
+ nodes.masterNode()
+ " marked shard as initializing, but shard state is ["
+ indexShard.state()
+ "], mark shard as started",
nodes.masterNode());
}
return;
} else {
if (indexShard.ignoreRecoveryAttempt()) {
logger.trace(
"ignoring recovery instruction for an existing shard {} (shard state: [{}])",
indexShard.shardId(),
indexShard.state());
return;
}
}
}
// if we're in peer recovery, try to find out the source node now so in case it fails, we will
// not create the index shard
DiscoveryNode sourceNode = null;
if (isPeerRecovery(shardRouting)) {
sourceNode = findSourceNodeForPeerRecovery(routingTable, nodes, shardRouting);
if (sourceNode == null) {
logger.trace(
"ignoring initializing shard {} - no source node can be found.",
shardRouting.shardId());
return;
}
}
// if there is no shard, create it
if (!indexService.hasShard(shardId)) {
if (failedShards.containsKey(shardRouting.shardId())) {
if (nodes.masterNode() != null) {
shardStateAction.resendShardFailed(
shardRouting,
indexMetaData.getIndexUUID(),
"master "
+ nodes.masterNode()
+ " marked shard as initializing, but shard is marked as failed, resend shard failure",
nodes.masterNode());
}
return;
}
try {
if (logger.isDebugEnabled()) {
logger.debug("[{}][{}] creating shard", shardRouting.index(), shardId);
}
IndexShard indexShard = indexService.createShard(shardId, shardRouting.primary());
indexShard.updateRoutingEntry(
shardRouting, state.blocks().disableStatePersistence() == false);
indexShard.addFailedEngineListener(failedEngineHandler);
} catch (IndexShardAlreadyExistsException e) {
// ignore this, the method call can happen several times
} catch (Throwable e) {
failAndRemoveShard(shardRouting, indexService, true, "failed to create shard", e);
return;
}
}
final IndexShard indexShard = indexService.shardSafe(shardId);
if (indexShard.ignoreRecoveryAttempt()) {
// we are already recovering (we can get to this state since the cluster event can happen
// several
// times while we recover)
logger.trace(
"ignoring recovery instruction for shard {} (shard state: [{}])",
indexShard.shardId(),
indexShard.state());
return;
}
if (isPeerRecovery(shardRouting)) {
try {
assert sourceNode != null : "peer recovery started but sourceNode is null";
// we don't mark this one as relocated at the end.
// For primaries: requests in any case are routed to both when its relocating and that way
// we handle
// the edge case where its mark as relocated, and we might need to roll it back...
// For replicas: we are recovering a backup from a primary
RecoveryState.Type type =
shardRouting.primary() ? RecoveryState.Type.RELOCATION : RecoveryState.Type.REPLICA;
recoveryTarget.startRecovery(
indexShard,
type,
sourceNode,
new PeerRecoveryListener(shardRouting, indexService, indexMetaData));
} catch (Throwable e) {
indexShard.failShard("corrupted preexisting index", e);
handleRecoveryFailure(indexService, shardRouting, true, e);
}
} else {
final IndexShardRoutingTable indexShardRouting =
routingTable.index(shardRouting.index()).shard(shardRouting.id());
indexService
.shard(shardId)
.recoverFromStore(
indexShardRouting,
new StoreRecoveryService.RecoveryListener() {
@Override
public void onRecoveryDone() {
shardStateAction.shardStarted(
shardRouting, indexMetaData.getIndexUUID(), "after recovery from store");
}
@Override
public void onIgnoreRecovery(String reason) {}
@Override
public void onRecoveryFailed(IndexShardRecoveryException e) {
handleRecoveryFailure(indexService, shardRouting, true, e);
}
});
}
}
private void applyNewOrUpdatedShards(final ClusterChangedEvent event) {
if (!indicesService.changesAllowed()) {
return;
}
RoutingTable routingTable = event.state().routingTable();
RoutingNodes.RoutingNodeIterator routingNode =
event.state().readOnlyRoutingNodes().routingNodeIter(event.state().nodes().localNodeId());
if (routingNode == null) {
failedShards.clear();
return;
}
DiscoveryNodes nodes = event.state().nodes();
for (final ShardRouting shardRouting : routingNode) {
final IndexService indexService = indicesService.indexService(shardRouting.index());
if (indexService == null) {
// got deleted on us, ignore
continue;
}
final IndexMetaData indexMetaData = event.state().metaData().index(shardRouting.index());
if (indexMetaData == null) {
// the index got deleted on the metadata, we will clean it later in the apply deleted method
// call
continue;
}
final int shardId = shardRouting.id();
if (!indexService.hasShard(shardId) && shardRouting.started()) {
if (failedShards.containsKey(shardRouting.shardId())) {
if (nodes.masterNode() != null) {
shardStateAction.resendShardFailed(
shardRouting,
indexMetaData.getIndexUUID(),
"master "
+ nodes.masterNode()
+ " marked shard as started, but shard has previous failed. resending shard failure.",
nodes.masterNode());
}
} else {
// the master thinks we are started, but we don't have this shard at all, mark it as
// failed
sendFailShard(
shardRouting,
indexMetaData.getIndexUUID(),
"master ["
+ nodes.masterNode()
+ "] marked shard as started, but shard has not been created, mark shard as failed",
null);
}
continue;
}
IndexShard indexShard = indexService.shard(shardId);
if (indexShard != null) {
ShardRouting currentRoutingEntry = indexShard.routingEntry();
// if the current and global routing are initializing, but are still not the same, its a
// different "shard" being allocated
// for example: a shard that recovers from one node and now needs to recover to another
// node,
// or a replica allocated and then allocating a primary because the primary
// failed on another node
boolean shardHasBeenRemoved = false;
if (currentRoutingEntry.initializing()
&& shardRouting.initializing()
&& !currentRoutingEntry.equals(shardRouting)) {
logger.debug(
"[{}][{}] removing shard (different instance of it allocated on this node, current [{}], global [{}])",
shardRouting.index(),
shardRouting.id(),
currentRoutingEntry,
shardRouting);
// closing the shard will also cancel any ongoing recovery.
indexService.removeShard(
shardRouting.id(),
"removing shard (different instance of it allocated on this node)");
shardHasBeenRemoved = true;
} else if (isPeerRecovery(shardRouting)) {
final DiscoveryNode sourceNode =
findSourceNodeForPeerRecovery(routingTable, nodes, shardRouting);
// check if there is an existing recovery going, and if so, and the source node is not the
// same, cancel the recovery to restart it
final Predicate<RecoveryStatus> shouldCancel =
new Predicate<RecoveryStatus>() {
@Override
public boolean apply(@Nullable RecoveryStatus status) {
return status.sourceNode().equals(sourceNode) == false;
}
};
if (recoveryTarget.cancelRecoveriesForShard(
indexShard.shardId(), "recovery source node changed", shouldCancel)) {
logger.debug(
"[{}][{}] removing shard (recovery source changed), current [{}], global [{}])",
shardRouting.index(),
shardRouting.id(),
currentRoutingEntry,
shardRouting);
// closing the shard will also cancel any ongoing recovery.
indexService.removeShard(
shardRouting.id(), "removing shard (recovery source node changed)");
shardHasBeenRemoved = true;
}
}
if (shardHasBeenRemoved == false
&& (shardRouting.equals(indexShard.routingEntry()) == false
|| shardRouting.version() > indexShard.routingEntry().version())) {
if (shardRouting.primary()
&& indexShard.routingEntry().primary() == false
&& shardRouting.initializing()
&& indexShard.allowsPrimaryPromotion() == false) {
logger.debug("{} reinitialize shard on primary promotion", indexShard.shardId());
indexService.removeShard(shardId, "promoted to primary");
} else {
// if we happen to remove the shardRouting by id above we don't need to jump in here!
indexShard.updateRoutingEntry(
shardRouting, event.state().blocks().disableStatePersistence() == false);
}
}
}
if (shardRouting.initializing()) {
applyInitializingShard(event.state(), indexMetaData, shardRouting);
}
}
}
public synchronized IndexShard createShard(ShardRouting routing) throws IOException {
final boolean primary = routing.primary();
/*
* TODO: we execute this in parallel but it's a synced method. Yet, we might
* be able to serialize the execution via the cluster state in the future. for now we just
* keep it synced.
*/
if (closed.get()) {
throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed");
}
final Settings indexSettings = this.indexSettings.getSettings();
final ShardId shardId = routing.shardId();
boolean success = false;
Store store = null;
IndexShard indexShard = null;
final ShardLock lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5));
try {
eventListener.beforeIndexShardCreated(shardId, indexSettings);
ShardPath path;
try {
path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
} catch (IllegalStateException ex) {
logger.warn("{} failed to load shard path, trying to remove leftover", shardId);
try {
ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings);
path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
} catch (Throwable t) {
t.addSuppressed(ex);
throw t;
}
}
if (path == null) {
// TODO: we should, instead, hold a "bytes reserved" of how large we anticipate this shard
// will be, e.g. for a shard
// that's being relocated/replicated we know how large it will become once it's done
// copying:
// Count up how many shards are currently on each data path:
Map<Path, Integer> dataPathToShardCount = new HashMap<>();
for (IndexShard shard : this) {
Path dataPath = shard.shardPath().getRootStatePath();
Integer curCount = dataPathToShardCount.get(dataPath);
if (curCount == null) {
curCount = 0;
}
dataPathToShardCount.put(dataPath, curCount + 1);
}
path =
ShardPath.selectNewPathForShard(
nodeEnv,
shardId,
this.indexSettings,
routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE
? getAvgShardSizeInBytes()
: routing.getExpectedShardSize(),
dataPathToShardCount);
logger.debug("{} creating using a new path [{}]", shardId, path);
} else {
logger.debug("{} creating using an existing path [{}]", shardId, path);
}
if (shards.containsKey(shardId.id())) {
throw new IndexShardAlreadyExistsException(shardId + " already exists");
}
logger.debug("creating shard_id {}", shardId);
// if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the
// primary.
final boolean canDeleteShardContent =
IndexMetaData.isOnSharedFilesystem(indexSettings) == false
|| (primary && IndexMetaData.isOnSharedFilesystem(indexSettings));
store =
new Store(
shardId,
this.indexSettings,
indexStore.newDirectoryService(path),
lock,
new StoreCloseListener(
shardId,
canDeleteShardContent,
() -> nodeServicesProvider.getIndicesQueryCache().onClose(shardId)));
if (useShadowEngine(primary, indexSettings)) {
indexShard =
new ShadowIndexShard(
shardId,
this.indexSettings,
path,
store,
indexCache,
mapperService,
similarityService,
indexFieldData,
engineFactory,
eventListener,
searcherWrapper,
nodeServicesProvider); // no indexing listeners - shadow engines don't index
} else {
indexShard =
new IndexShard(
shardId,
this.indexSettings,
path,
store,
indexCache,
mapperService,
similarityService,
indexFieldData,
engineFactory,
eventListener,
searcherWrapper,
nodeServicesProvider,
listeners);
}
eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
eventListener.afterIndexShardCreated(indexShard);
indexShard.updateRoutingEntry(routing, true);
shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
success = true;
return indexShard;
} finally {
if (success == false) {
IOUtils.closeWhileHandlingException(lock);
closeShard("initialization failed", shardId, indexShard, store, eventListener);
}
}
}