@Test
  public void gatewayRecoveryTest() throws Exception {
    logger.info("--> start nodes");
    String node = cluster().startNode(settingsBuilder().put("gateway.type", "local"));

    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);

    logger.info("--> restarting cluster");
    cluster().fullRestart();
    ensureGreen();

    logger.info("--> request recoveries");
    RecoveryResponse response =
        client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    assertThat(response.shardResponses().size(), equalTo(SHARD_COUNT));
    assertThat(response.shardResponses().get(INDEX_NAME).size(), equalTo(1));

    List<ShardRecoveryResponse> shardResponses = response.shardResponses().get(INDEX_NAME);
    assertThat(shardResponses.size(), equalTo(1));

    ShardRecoveryResponse shardResponse = shardResponses.get(0);
    RecoveryState state = shardResponse.recoveryState();

    assertThat(state.getType(), equalTo(RecoveryState.Type.GATEWAY));
    assertThat(state.getStage(), equalTo(RecoveryState.Stage.DONE));
    assertThat(node, equalTo(state.getSourceNode().getName()));
    assertThat(node, equalTo(state.getTargetNode().getName()));
    assertNull(state.getRestoreSource());
  }
  public void testGatewayRecovery() throws Exception {
    logger.info("--> start nodes");
    String node = internalCluster().startNode();

    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);

    logger.info("--> restarting cluster");
    internalCluster().fullRestart();
    ensureGreen();

    logger.info("--> request recoveries");
    RecoveryResponse response =
        client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    assertThat(response.shardRecoveryStates().size(), equalTo(SHARD_COUNT));
    assertThat(response.shardRecoveryStates().get(INDEX_NAME).size(), equalTo(1));

    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    assertThat(recoveryStates.size(), equalTo(1));

    RecoveryState recoveryState = recoveryStates.get(0);

    assertRecoveryState(
        recoveryState,
        0,
        StoreRecoverySource.EXISTING_STORE_INSTANCE,
        true,
        Stage.DONE,
        null,
        node);

    validateIndexRecoveryState(recoveryState.getIndex());
  }
Example #3
0
 void assertIndexSanity(String indexName, Version indexCreated) {
   GetIndexResponse getIndexResponse =
       client().admin().indices().prepareGetIndex().addIndices(indexName).get();
   assertEquals(1, getIndexResponse.indices().length);
   assertEquals(indexName, getIndexResponse.indices()[0]);
   Version actualVersionCreated =
       Version.indexCreated(getIndexResponse.getSettings().get(indexName));
   assertEquals(indexCreated, actualVersionCreated);
   ensureYellow(indexName);
   RecoveryResponse recoveryResponse =
       client()
           .admin()
           .indices()
           .prepareRecoveries(indexName)
           .setDetailed(true)
           .setActiveOnly(false)
           .get();
   boolean foundTranslog = false;
   for (List<RecoveryState> states : recoveryResponse.shardRecoveryStates().values()) {
     for (RecoveryState state : states) {
       if (state.getStage() == RecoveryState.Stage.DONE
           && state.getPrimary()
           && state.getRecoverySource().getType() == RecoverySource.Type.EXISTING_STORE) {
         assertFalse("more than one primary recoverd?", foundTranslog);
         assertNotEquals(0, state.getTranslog().recoveredOperations());
         foundTranslog = true;
       }
     }
   }
   assertTrue("expected translog but nothing was recovered", foundTranslog);
   IndicesSegmentResponse segmentsResponse =
       client().admin().indices().prepareSegments(indexName).get();
   IndexSegments segments = segmentsResponse.getIndices().get(indexName);
   int numCurrent = 0;
   int numBWC = 0;
   for (IndexShardSegments indexShardSegments : segments) {
     for (ShardSegments shardSegments : indexShardSegments) {
       for (Segment segment : shardSegments) {
         if (indexCreated.luceneVersion.equals(segment.version)) {
           numBWC++;
           if (Version.CURRENT.luceneVersion.equals(segment.version)) {
             numCurrent++;
           }
         } else if (Version.CURRENT.luceneVersion.equals(segment.version)) {
           numCurrent++;
         } else {
           fail("unexpected version " + segment.version);
         }
       }
     }
   }
   assertNotEquals("expected at least 1 current segment after translog recovery", 0, numCurrent);
   assertNotEquals("expected at least 1 old segment", 0, numBWC);
   SearchResponse test = client().prepareSearch(indexName).get();
   assertThat(test.getHits().getTotalHits(), greaterThanOrEqualTo(1L));
 }
  public void testReplicaRecovery() throws Exception {
    logger.info("--> start node A");
    String nodeA = internalCluster().startNode();

    logger.info("--> create index on node: {}", nodeA);
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);

    logger.info("--> start node B");
    String nodeB = internalCluster().startNode();
    ensureGreen();

    // force a shard recovery from nodeA to nodeB
    logger.info("--> bump replica count");
    client()
        .admin()
        .indices()
        .prepareUpdateSettings(INDEX_NAME)
        .setSettings(Settings.builder().put("number_of_replicas", 1))
        .execute()
        .actionGet();
    ensureGreen();

    logger.info("--> request recoveries");
    RecoveryResponse response =
        client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();

    // we should now have two total shards, one primary and one replica
    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    assertThat(recoveryStates.size(), equalTo(2));

    List<RecoveryState> nodeAResponses = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeAResponses.size(), equalTo(1));
    List<RecoveryState> nodeBResponses = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBResponses.size(), equalTo(1));

    // validate node A recovery
    RecoveryState nodeARecoveryState = nodeAResponses.get(0);
    assertRecoveryState(
        nodeARecoveryState,
        0,
        StoreRecoverySource.EMPTY_STORE_INSTANCE,
        true,
        Stage.DONE,
        null,
        nodeA);
    validateIndexRecoveryState(nodeARecoveryState.getIndex());

    // validate node B recovery
    RecoveryState nodeBRecoveryState = nodeBResponses.get(0);
    assertRecoveryState(
        nodeBRecoveryState, 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryState.getIndex());
  }
  @Test
  public void replicaRecoveryTest() throws Exception {
    logger.info("--> start node A");
    String nodeA = cluster().startNode(settingsBuilder().put("gateway.type", "local"));

    logger.info("--> create index on node: {}", nodeA);
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);

    logger.info("--> start node B");
    String nodeB = cluster().startNode(settingsBuilder().put("gateway.type", "local"));
    ensureGreen();

    // force a shard recovery from nodeA to nodeB
    logger.info("--> bump replica count");
    client()
        .admin()
        .indices()
        .prepareUpdateSettings(INDEX_NAME)
        .setSettings(settingsBuilder().put("number_of_replicas", 1))
        .execute()
        .actionGet();
    ensureGreen();

    logger.info("--> request recoveries");
    RecoveryResponse response =
        client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();

    // we should now have two total shards, one primary and one replica
    List<ShardRecoveryResponse> shardResponses = response.shardResponses().get(INDEX_NAME);
    assertThat(shardResponses.size(), equalTo(2));

    List<ShardRecoveryResponse> nodeAResponses = findRecoveriesForTargetNode(nodeA, shardResponses);
    assertThat(nodeAResponses.size(), equalTo(1));
    List<ShardRecoveryResponse> nodeBResponses = findRecoveriesForTargetNode(nodeB, shardResponses);
    assertThat(nodeBResponses.size(), equalTo(1));

    // validate node A recovery
    ShardRecoveryResponse nodeAShardResponse = nodeAResponses.get(0);
    assertThat(nodeAShardResponse.recoveryState().getShardId().id(), equalTo(0));
    assertThat(nodeAShardResponse.recoveryState().getSourceNode().getName(), equalTo(nodeA));
    assertThat(nodeAShardResponse.recoveryState().getTargetNode().getName(), equalTo(nodeA));
    assertThat(nodeAShardResponse.recoveryState().getType(), equalTo(RecoveryState.Type.GATEWAY));
    assertThat(nodeAShardResponse.recoveryState().getStage(), equalTo(RecoveryState.Stage.DONE));

    // validate node B recovery
    ShardRecoveryResponse nodeBShardResponse = nodeBResponses.get(0);
    assertThat(nodeBShardResponse.recoveryState().getShardId().id(), equalTo(0));
    assertThat(nodeBShardResponse.recoveryState().getSourceNode().getName(), equalTo(nodeA));
    assertThat(nodeBShardResponse.recoveryState().getTargetNode().getName(), equalTo(nodeB));
    assertThat(nodeBShardResponse.recoveryState().getType(), equalTo(RecoveryState.Type.REPLICA));
    assertThat(nodeBShardResponse.recoveryState().getStage(), equalTo(RecoveryState.Stage.DONE));
  }
 private void createIndexIfNotExist(Client client) {
   try {
     client.admin().indices().prepareCreate(INDEX_NAME).execute().actionGet();
     RecoveryResponse response =
         client.admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
     int shards = response.getTotalShards();
     client
         .admin()
         .cluster()
         .prepareHealth(INDEX_NAME)
         .setWaitForActiveShards(shards)
         .setWaitForYellowStatus()
         .execute()
         .actionGet();
   } catch (IndexAlreadyExistsException e) {
     // ignore
   }
 }
  @Test
  public void rerouteRecoveryTest() throws Exception {
    logger.info("--> start node A");
    String nodeA = cluster().startNode(settingsBuilder().put("gateway.type", "local"));

    logger.info("--> create index on node: {}", nodeA);
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);

    logger.info("--> start node B");
    String nodeB = cluster().startNode(settingsBuilder().put("gateway.type", "local"));
    ensureGreen();

    logger.info("--> move shard from: {} to: {}", nodeA, nodeB);
    client()
        .admin()
        .cluster()
        .prepareReroute()
        .add(new MoveAllocationCommand(new ShardId(INDEX_NAME, 0), nodeA, nodeB))
        .execute()
        .actionGet()
        .getState();

    ensureGreen();

    logger.info("--> request recoveries");
    RecoveryResponse response =
        client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();

    List<ShardRecoveryResponse> shardResponses = response.shardResponses().get(INDEX_NAME);
    assertThat(shardResponses.size(), equalTo(1));

    ShardRecoveryResponse shardResponse = shardResponses.get(0);
    RecoveryState state = shardResponse.recoveryState();

    assertThat(state.getType(), equalTo(RecoveryState.Type.RELOCATION));
    assertThat(state.getStage(), equalTo(RecoveryState.Stage.DONE));
    assertThat(nodeA, equalTo(state.getSourceNode().getName()));
    assertThat(nodeB, equalTo(state.getTargetNode().getName()));
    assertNull(state.getRestoreSource());
  }
  public void testGatewayRecoveryTestActiveOnly() throws Exception {
    logger.info("--> start nodes");
    internalCluster().startNode();

    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);

    logger.info("--> restarting cluster");
    internalCluster().fullRestart();
    ensureGreen();

    logger.info("--> request recoveries");
    RecoveryResponse response =
        client()
            .admin()
            .indices()
            .prepareRecoveries(INDEX_NAME)
            .setActiveOnly(true)
            .execute()
            .actionGet();

    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    assertThat(recoveryStates.size(), equalTo(0)); // Should not expect any responses back
  }
  public void testSnapshotRecovery() throws Exception {
    logger.info("--> start node A");
    String nodeA = internalCluster().startNode();

    logger.info("--> create repository");
    assertAcked(
        client()
            .admin()
            .cluster()
            .preparePutRepository(REPO_NAME)
            .setType("fs")
            .setSettings(
                Settings.builder().put("location", randomRepoPath()).put("compress", false))
            .get());

    ensureGreen();

    logger.info("--> create index on node: {}", nodeA);
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);

    logger.info("--> snapshot");
    CreateSnapshotResponse createSnapshotResponse =
        client()
            .admin()
            .cluster()
            .prepareCreateSnapshot(REPO_NAME, SNAP_NAME)
            .setWaitForCompletion(true)
            .setIndices(INDEX_NAME)
            .get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(
        createSnapshotResponse.getSnapshotInfo().successfulShards(),
        equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));

    assertThat(
        client()
            .admin()
            .cluster()
            .prepareGetSnapshots(REPO_NAME)
            .setSnapshots(SNAP_NAME)
            .get()
            .getSnapshots()
            .get(0)
            .state(),
        equalTo(SnapshotState.SUCCESS));

    client().admin().indices().prepareClose(INDEX_NAME).execute().actionGet();

    logger.info("--> restore");
    RestoreSnapshotResponse restoreSnapshotResponse =
        client()
            .admin()
            .cluster()
            .prepareRestoreSnapshot(REPO_NAME, SNAP_NAME)
            .setWaitForCompletion(true)
            .execute()
            .actionGet();
    int totalShards = restoreSnapshotResponse.getRestoreInfo().totalShards();
    assertThat(totalShards, greaterThan(0));

    ensureGreen();

    logger.info("--> request recoveries");
    RecoveryResponse response =
        client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();

    for (Map.Entry<String, List<RecoveryState>> indexRecoveryStates :
        response.shardRecoveryStates().entrySet()) {

      assertThat(indexRecoveryStates.getKey(), equalTo(INDEX_NAME));
      List<RecoveryState> recoveryStates = indexRecoveryStates.getValue();
      assertThat(recoveryStates.size(), equalTo(totalShards));

      for (RecoveryState recoveryState : recoveryStates) {
        SnapshotRecoverySource recoverySource =
            new SnapshotRecoverySource(
                new Snapshot(REPO_NAME, createSnapshotResponse.getSnapshotInfo().snapshotId()),
                Version.CURRENT,
                INDEX_NAME);
        assertRecoveryState(recoveryState, 0, recoverySource, true, Stage.DONE, null, nodeA);
        validateIndexRecoveryState(recoveryState.getIndex());
      }
    }
  }
  public void testRerouteRecovery() throws Exception {
    logger.info("--> start node A");
    final String nodeA = internalCluster().startNode();

    logger.info("--> create index on node: {}", nodeA);
    ByteSizeValue shardSize =
        createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT)
            .getShards()[0]
            .getStats()
            .getStore()
            .size();

    logger.info("--> start node B");
    final String nodeB = internalCluster().startNode();

    ensureGreen();

    logger.info("--> slowing down recoveries");
    slowDownRecovery(shardSize);

    logger.info("--> move shard from: {} to: {}", nodeA, nodeB);
    client()
        .admin()
        .cluster()
        .prepareReroute()
        .add(new MoveAllocationCommand(INDEX_NAME, 0, nodeA, nodeB))
        .execute()
        .actionGet()
        .getState();

    logger.info("--> waiting for recovery to start both on source and target");
    final Index index = resolveIndex(INDEX_NAME);
    assertBusy(
        new Runnable() {
          @Override
          public void run() {

            IndicesService indicesService =
                internalCluster().getInstance(IndicesService.class, nodeA);
            assertThat(
                indicesService
                    .indexServiceSafe(index)
                    .getShard(0)
                    .recoveryStats()
                    .currentAsSource(),
                equalTo(1));
            indicesService = internalCluster().getInstance(IndicesService.class, nodeB);
            assertThat(
                indicesService
                    .indexServiceSafe(index)
                    .getShard(0)
                    .recoveryStats()
                    .currentAsTarget(),
                equalTo(1));
          }
        });

    logger.info("--> request recoveries");
    RecoveryResponse response =
        client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();

    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    List<RecoveryState> nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(1));
    List<RecoveryState> nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));

    assertRecoveryState(
        nodeARecoveryStates.get(0),
        0,
        StoreRecoverySource.EMPTY_STORE_INSTANCE,
        true,
        Stage.DONE,
        null,
        nodeA);
    validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());

    assertOnGoingRecoveryState(
        nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());

    logger.info("--> request node recovery stats");
    NodesStatsResponse statsResponse =
        client()
            .admin()
            .cluster()
            .prepareNodesStats()
            .clear()
            .setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery))
            .get();
    long nodeAThrottling = Long.MAX_VALUE;
    long nodeBThrottling = Long.MAX_VALUE;
    for (NodeStats nodeStats : statsResponse.getNodes()) {
      final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
      if (nodeStats.getNode().getName().equals(nodeA)) {
        assertThat(
            "node A should have ongoing recovery as source",
            recoveryStats.currentAsSource(),
            equalTo(1));
        assertThat(
            "node A should not have ongoing recovery as target",
            recoveryStats.currentAsTarget(),
            equalTo(0));
        nodeAThrottling = recoveryStats.throttleTime().millis();
      }
      if (nodeStats.getNode().getName().equals(nodeB)) {
        assertThat(
            "node B should not have ongoing recovery as source",
            recoveryStats.currentAsSource(),
            equalTo(0));
        assertThat(
            "node B should have ongoing recovery as target",
            recoveryStats.currentAsTarget(),
            equalTo(1));
        nodeBThrottling = recoveryStats.throttleTime().millis();
      }
    }

    logger.info("--> checking throttling increases");
    final long finalNodeAThrottling = nodeAThrottling;
    final long finalNodeBThrottling = nodeBThrottling;
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            NodesStatsResponse statsResponse =
                client()
                    .admin()
                    .cluster()
                    .prepareNodesStats()
                    .clear()
                    .setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery))
                    .get();
            assertThat(statsResponse.getNodes(), hasSize(2));
            for (NodeStats nodeStats : statsResponse.getNodes()) {
              final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
              if (nodeStats.getNode().getName().equals(nodeA)) {
                assertThat(
                    "node A throttling should increase",
                    recoveryStats.throttleTime().millis(),
                    greaterThan(finalNodeAThrottling));
              }
              if (nodeStats.getNode().getName().equals(nodeB)) {
                assertThat(
                    "node B throttling should increase",
                    recoveryStats.throttleTime().millis(),
                    greaterThan(finalNodeBThrottling));
              }
            }
          }
        });

    logger.info("--> speeding up recoveries");
    restoreRecoverySpeed();

    // wait for it to be finished
    ensureGreen();

    response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();

    recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    assertThat(recoveryStates.size(), equalTo(1));

    assertRecoveryState(
        recoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(recoveryStates.get(0).getIndex());

    statsResponse =
        client()
            .admin()
            .cluster()
            .prepareNodesStats()
            .clear()
            .setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery))
            .get();
    assertThat(statsResponse.getNodes(), hasSize(2));
    for (NodeStats nodeStats : statsResponse.getNodes()) {
      final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
      assertThat(recoveryStats.currentAsSource(), equalTo(0));
      assertThat(recoveryStats.currentAsTarget(), equalTo(0));
      if (nodeStats.getNode().getName().equals(nodeA)) {
        assertThat(
            "node A throttling should be >0",
            recoveryStats.throttleTime().millis(),
            greaterThan(0L));
      }
      if (nodeStats.getNode().getName().equals(nodeB)) {
        assertThat(
            "node B throttling should be >0 ",
            recoveryStats.throttleTime().millis(),
            greaterThan(0L));
      }
    }

    logger.info("--> bump replica count");
    client()
        .admin()
        .indices()
        .prepareUpdateSettings(INDEX_NAME)
        .setSettings(Settings.builder().put("number_of_replicas", 1))
        .execute()
        .actionGet();
    ensureGreen();

    statsResponse =
        client()
            .admin()
            .cluster()
            .prepareNodesStats()
            .clear()
            .setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery))
            .get();
    assertThat(statsResponse.getNodes(), hasSize(2));
    for (NodeStats nodeStats : statsResponse.getNodes()) {
      final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
      assertThat(recoveryStats.currentAsSource(), equalTo(0));
      assertThat(recoveryStats.currentAsTarget(), equalTo(0));
      if (nodeStats.getNode().getName().equals(nodeA)) {
        assertThat(
            "node A throttling should be >0",
            recoveryStats.throttleTime().millis(),
            greaterThan(0L));
      }
      if (nodeStats.getNode().getName().equals(nodeB)) {
        assertThat(
            "node B throttling should be >0 ",
            recoveryStats.throttleTime().millis(),
            greaterThan(0L));
      }
    }

    logger.info("--> start node C");
    String nodeC = internalCluster().startNode();
    assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("3").get().isTimedOut());

    logger.info("--> slowing down recoveries");
    slowDownRecovery(shardSize);

    logger.info("--> move replica shard from: {} to: {}", nodeA, nodeC);
    client()
        .admin()
        .cluster()
        .prepareReroute()
        .add(new MoveAllocationCommand(INDEX_NAME, 0, nodeA, nodeC))
        .execute()
        .actionGet()
        .getState();

    response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);

    nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(1));
    nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    List<RecoveryState> nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
    assertThat(nodeCRecoveryStates.size(), equalTo(1));

    assertRecoveryState(
        nodeARecoveryStates.get(0),
        0,
        PeerRecoverySource.INSTANCE,
        false,
        Stage.DONE,
        nodeB,
        nodeA);
    validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());

    assertRecoveryState(
        nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());

    // relocations of replicas are marked as REPLICA and the source node is the node holding the
    // primary (B)
    assertOnGoingRecoveryState(
        nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, nodeB, nodeC);
    validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());

    if (randomBoolean()) {
      // shutdown node with relocation source of replica shard and check if recovery continues
      internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA));
      ensureStableCluster(2);

      response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
      recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);

      nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
      assertThat(nodeARecoveryStates.size(), equalTo(0));
      nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
      assertThat(nodeBRecoveryStates.size(), equalTo(1));
      nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
      assertThat(nodeCRecoveryStates.size(), equalTo(1));

      assertRecoveryState(
          nodeBRecoveryStates.get(0),
          0,
          PeerRecoverySource.INSTANCE,
          true,
          Stage.DONE,
          nodeA,
          nodeB);
      validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());

      assertOnGoingRecoveryState(
          nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, nodeB, nodeC);
      validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
    }

    logger.info("--> speeding up recoveries");
    restoreRecoverySpeed();
    ensureGreen();

    response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);

    nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(0));
    nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
    assertThat(nodeCRecoveryStates.size(), equalTo(1));

    assertRecoveryState(
        nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());

    // relocations of replicas are marked as REPLICA and the source node is the node holding the
    // primary (B)
    assertRecoveryState(
        nodeCRecoveryStates.get(0),
        0,
        PeerRecoverySource.INSTANCE,
        false,
        Stage.DONE,
        nodeB,
        nodeC);
    validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
  }
  @Test
  public void snapshotRecoveryTest() throws Exception {
    logger.info("--> start node A");
    String nodeA = cluster().startNode(settingsBuilder().put("gateway.type", "local"));

    logger.info("--> create repository");
    assertAcked(
        client()
            .admin()
            .cluster()
            .preparePutRepository(REPO_NAME)
            .setType("fs")
            .setSettings(
                ImmutableSettings.settingsBuilder()
                    .put("location", newTempDir(LifecycleScope.SUITE))
                    .put("compress", false))
            .get());

    ensureGreen();

    logger.info("--> create index on node: {}", nodeA);
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);

    logger.info("--> snapshot");
    CreateSnapshotResponse createSnapshotResponse =
        client()
            .admin()
            .cluster()
            .prepareCreateSnapshot(REPO_NAME, SNAP_NAME)
            .setWaitForCompletion(true)
            .setIndices(INDEX_NAME)
            .get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(
        createSnapshotResponse.getSnapshotInfo().successfulShards(),
        equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));

    assertThat(
        client()
            .admin()
            .cluster()
            .prepareGetSnapshots(REPO_NAME)
            .setSnapshots(SNAP_NAME)
            .get()
            .getSnapshots()
            .get(0)
            .state(),
        equalTo(SnapshotState.SUCCESS));

    client().admin().indices().prepareClose(INDEX_NAME).execute().actionGet();

    logger.info("--> restore");
    RestoreSnapshotResponse restoreSnapshotResponse =
        client()
            .admin()
            .cluster()
            .prepareRestoreSnapshot(REPO_NAME, SNAP_NAME)
            .setWaitForCompletion(true)
            .execute()
            .actionGet();
    int totalShards = restoreSnapshotResponse.getRestoreInfo().totalShards();
    assertThat(totalShards, greaterThan(0));

    ensureGreen();

    logger.info("--> request recoveries");
    RecoveryResponse response =
        client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();

    for (Map.Entry<String, List<ShardRecoveryResponse>> shardRecoveryResponse :
        response.shardResponses().entrySet()) {

      assertThat(shardRecoveryResponse.getKey(), equalTo(INDEX_NAME));
      List<ShardRecoveryResponse> shardRecoveryResponses = shardRecoveryResponse.getValue();
      assertThat(shardRecoveryResponses.size(), equalTo(totalShards));

      for (ShardRecoveryResponse shardResponse : shardRecoveryResponses) {

        assertThat(shardResponse.recoveryState().getType(), equalTo(RecoveryState.Type.SNAPSHOT));
        assertThat(shardResponse.recoveryState().getStage(), equalTo(RecoveryState.Stage.DONE));
        assertNotNull(shardResponse.recoveryState().getRestoreSource());
        assertThat(shardResponse.recoveryState().getTargetNode().getName(), equalTo(nodeA));
      }
    }
  }