private void applyInitializingShard( final ClusterState state, final IndexMetaData indexMetaData, final ShardRouting shardRouting) { final IndexService indexService = indicesService.indexService(shardRouting.index()); if (indexService == null) { // got deleted on us, ignore return; } final RoutingTable routingTable = state.routingTable(); final DiscoveryNodes nodes = state.getNodes(); final int shardId = shardRouting.id(); if (indexService.hasShard(shardId)) { IndexShard indexShard = indexService.shardSafe(shardId); if (indexShard.state() == IndexShardState.STARTED || indexShard.state() == IndexShardState.POST_RECOVERY) { // the master thinks we are initializing, but we are already started or on POST_RECOVERY and // waiting // for master to confirm a shard started message (either master failover, or a cluster event // before // we managed to tell the master we started), mark us as started if (logger.isTraceEnabled()) { logger.trace( "{} master marked shard as initializing, but shard has state [{}], resending shard started to {}", indexShard.shardId(), indexShard.state(), nodes.masterNode()); } if (nodes.masterNode() != null) { shardStateAction.shardStarted( shardRouting, indexMetaData.getIndexUUID(), "master " + nodes.masterNode() + " marked shard as initializing, but shard state is [" + indexShard.state() + "], mark shard as started", nodes.masterNode()); } return; } else { if (indexShard.ignoreRecoveryAttempt()) { logger.trace( "ignoring recovery instruction for an existing shard {} (shard state: [{}])", indexShard.shardId(), indexShard.state()); return; } } } // if we're in peer recovery, try to find out the source node now so in case it fails, we will // not create the index shard DiscoveryNode sourceNode = null; if (isPeerRecovery(shardRouting)) { sourceNode = findSourceNodeForPeerRecovery(routingTable, nodes, shardRouting); if (sourceNode == null) { logger.trace( "ignoring initializing shard {} - no source node can be found.", shardRouting.shardId()); return; } } // if there is no shard, create it if (!indexService.hasShard(shardId)) { if (failedShards.containsKey(shardRouting.shardId())) { if (nodes.masterNode() != null) { shardStateAction.resendShardFailed( shardRouting, indexMetaData.getIndexUUID(), "master " + nodes.masterNode() + " marked shard as initializing, but shard is marked as failed, resend shard failure", nodes.masterNode()); } return; } try { if (logger.isDebugEnabled()) { logger.debug("[{}][{}] creating shard", shardRouting.index(), shardId); } IndexShard indexShard = indexService.createShard(shardId, shardRouting.primary()); indexShard.updateRoutingEntry( shardRouting, state.blocks().disableStatePersistence() == false); indexShard.addFailedEngineListener(failedEngineHandler); } catch (IndexShardAlreadyExistsException e) { // ignore this, the method call can happen several times } catch (Throwable e) { failAndRemoveShard(shardRouting, indexService, true, "failed to create shard", e); return; } } final IndexShard indexShard = indexService.shardSafe(shardId); if (indexShard.ignoreRecoveryAttempt()) { // we are already recovering (we can get to this state since the cluster event can happen // several // times while we recover) logger.trace( "ignoring recovery instruction for shard {} (shard state: [{}])", indexShard.shardId(), indexShard.state()); return; } if (isPeerRecovery(shardRouting)) { try { assert sourceNode != null : "peer recovery started but sourceNode is null"; // we don't mark this one as relocated at the end. // For primaries: requests in any case are routed to both when its relocating and that way // we handle // the edge case where its mark as relocated, and we might need to roll it back... // For replicas: we are recovering a backup from a primary RecoveryState.Type type = shardRouting.primary() ? RecoveryState.Type.RELOCATION : RecoveryState.Type.REPLICA; recoveryTarget.startRecovery( indexShard, type, sourceNode, new PeerRecoveryListener(shardRouting, indexService, indexMetaData)); } catch (Throwable e) { indexShard.failShard("corrupted preexisting index", e); handleRecoveryFailure(indexService, shardRouting, true, e); } } else { final IndexShardRoutingTable indexShardRouting = routingTable.index(shardRouting.index()).shard(shardRouting.id()); indexService .shard(shardId) .recoverFromStore( indexShardRouting, new StoreRecoveryService.RecoveryListener() { @Override public void onRecoveryDone() { shardStateAction.shardStarted( shardRouting, indexMetaData.getIndexUUID(), "after recovery from store"); } @Override public void onIgnoreRecovery(String reason) {} @Override public void onRecoveryFailed(IndexShardRecoveryException e) { handleRecoveryFailure(indexService, shardRouting, true, e); } }); } }