@Test
public void gatewayRecoveryTest() throws Exception {
logger.info("--> start nodes");
String node = cluster().startNode(settingsBuilder().put("gateway.type", "local"));
createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
logger.info("--> restarting cluster");
cluster().fullRestart();
ensureGreen();
logger.info("--> request recoveries");
RecoveryResponse response =
client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
assertThat(response.shardResponses().size(), equalTo(SHARD_COUNT));
assertThat(response.shardResponses().get(INDEX_NAME).size(), equalTo(1));
List<ShardRecoveryResponse> shardResponses = response.shardResponses().get(INDEX_NAME);
assertThat(shardResponses.size(), equalTo(1));
ShardRecoveryResponse shardResponse = shardResponses.get(0);
RecoveryState state = shardResponse.recoveryState();
assertThat(state.getType(), equalTo(RecoveryState.Type.GATEWAY));
assertThat(state.getStage(), equalTo(RecoveryState.Stage.DONE));
assertThat(node, equalTo(state.getSourceNode().getName()));
assertThat(node, equalTo(state.getTargetNode().getName()));
assertNull(state.getRestoreSource());
}
private List<ShardRecoveryResponse> findRecoveriesForTargetNode(
String nodeName, List<ShardRecoveryResponse> responses) {
List<ShardRecoveryResponse> nodeResponses = new ArrayList<>();
for (ShardRecoveryResponse response : responses) {
if (response.recoveryState().getTargetNode().getName().equals(nodeName)) {
nodeResponses.add(response);
}
}
return nodeResponses;
}
@Test
public void replicaRecoveryTest() throws Exception {
logger.info("--> start node A");
String nodeA = cluster().startNode(settingsBuilder().put("gateway.type", "local"));
logger.info("--> create index on node: {}", nodeA);
createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
logger.info("--> start node B");
String nodeB = cluster().startNode(settingsBuilder().put("gateway.type", "local"));
ensureGreen();
// force a shard recovery from nodeA to nodeB
logger.info("--> bump replica count");
client()
.admin()
.indices()
.prepareUpdateSettings(INDEX_NAME)
.setSettings(settingsBuilder().put("number_of_replicas", 1))
.execute()
.actionGet();
ensureGreen();
logger.info("--> request recoveries");
RecoveryResponse response =
client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
// we should now have two total shards, one primary and one replica
List<ShardRecoveryResponse> shardResponses = response.shardResponses().get(INDEX_NAME);
assertThat(shardResponses.size(), equalTo(2));
List<ShardRecoveryResponse> nodeAResponses = findRecoveriesForTargetNode(nodeA, shardResponses);
assertThat(nodeAResponses.size(), equalTo(1));
List<ShardRecoveryResponse> nodeBResponses = findRecoveriesForTargetNode(nodeB, shardResponses);
assertThat(nodeBResponses.size(), equalTo(1));
// validate node A recovery
ShardRecoveryResponse nodeAShardResponse = nodeAResponses.get(0);
assertThat(nodeAShardResponse.recoveryState().getShardId().id(), equalTo(0));
assertThat(nodeAShardResponse.recoveryState().getSourceNode().getName(), equalTo(nodeA));
assertThat(nodeAShardResponse.recoveryState().getTargetNode().getName(), equalTo(nodeA));
assertThat(nodeAShardResponse.recoveryState().getType(), equalTo(RecoveryState.Type.GATEWAY));
assertThat(nodeAShardResponse.recoveryState().getStage(), equalTo(RecoveryState.Stage.DONE));
// validate node B recovery
ShardRecoveryResponse nodeBShardResponse = nodeBResponses.get(0);
assertThat(nodeBShardResponse.recoveryState().getShardId().id(), equalTo(0));
assertThat(nodeBShardResponse.recoveryState().getSourceNode().getName(), equalTo(nodeA));
assertThat(nodeBShardResponse.recoveryState().getTargetNode().getName(), equalTo(nodeB));
assertThat(nodeBShardResponse.recoveryState().getType(), equalTo(RecoveryState.Type.REPLICA));
assertThat(nodeBShardResponse.recoveryState().getStage(), equalTo(RecoveryState.Stage.DONE));
}
@Test
public void rerouteRecoveryTest() throws Exception {
logger.info("--> start node A");
String nodeA = cluster().startNode(settingsBuilder().put("gateway.type", "local"));
logger.info("--> create index on node: {}", nodeA);
createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
logger.info("--> start node B");
String nodeB = cluster().startNode(settingsBuilder().put("gateway.type", "local"));
ensureGreen();
logger.info("--> move shard from: {} to: {}", nodeA, nodeB);
client()
.admin()
.cluster()
.prepareReroute()
.add(new MoveAllocationCommand(new ShardId(INDEX_NAME, 0), nodeA, nodeB))
.execute()
.actionGet()
.getState();
ensureGreen();
logger.info("--> request recoveries");
RecoveryResponse response =
client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
List<ShardRecoveryResponse> shardResponses = response.shardResponses().get(INDEX_NAME);
assertThat(shardResponses.size(), equalTo(1));
ShardRecoveryResponse shardResponse = shardResponses.get(0);
RecoveryState state = shardResponse.recoveryState();
assertThat(state.getType(), equalTo(RecoveryState.Type.RELOCATION));
assertThat(state.getStage(), equalTo(RecoveryState.Stage.DONE));
assertThat(nodeA, equalTo(state.getSourceNode().getName()));
assertThat(nodeB, equalTo(state.getTargetNode().getName()));
assertNull(state.getRestoreSource());
}
@Test
public void snapshotRecoveryTest() throws Exception {
logger.info("--> start node A");
String nodeA = cluster().startNode(settingsBuilder().put("gateway.type", "local"));
logger.info("--> create repository");
assertAcked(
client()
.admin()
.cluster()
.preparePutRepository(REPO_NAME)
.setType("fs")
.setSettings(
ImmutableSettings.settingsBuilder()
.put("location", newTempDir(LifecycleScope.SUITE))
.put("compress", false))
.get());
ensureGreen();
logger.info("--> create index on node: {}", nodeA);
createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
logger.info("--> snapshot");
CreateSnapshotResponse createSnapshotResponse =
client()
.admin()
.cluster()
.prepareCreateSnapshot(REPO_NAME, SNAP_NAME)
.setWaitForCompletion(true)
.setIndices(INDEX_NAME)
.get();
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
assertThat(
createSnapshotResponse.getSnapshotInfo().successfulShards(),
equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
assertThat(
client()
.admin()
.cluster()
.prepareGetSnapshots(REPO_NAME)
.setSnapshots(SNAP_NAME)
.get()
.getSnapshots()
.get(0)
.state(),
equalTo(SnapshotState.SUCCESS));
client().admin().indices().prepareClose(INDEX_NAME).execute().actionGet();
logger.info("--> restore");
RestoreSnapshotResponse restoreSnapshotResponse =
client()
.admin()
.cluster()
.prepareRestoreSnapshot(REPO_NAME, SNAP_NAME)
.setWaitForCompletion(true)
.execute()
.actionGet();
int totalShards = restoreSnapshotResponse.getRestoreInfo().totalShards();
assertThat(totalShards, greaterThan(0));
ensureGreen();
logger.info("--> request recoveries");
RecoveryResponse response =
client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
for (Map.Entry<String, List<ShardRecoveryResponse>> shardRecoveryResponse :
response.shardResponses().entrySet()) {
assertThat(shardRecoveryResponse.getKey(), equalTo(INDEX_NAME));
List<ShardRecoveryResponse> shardRecoveryResponses = shardRecoveryResponse.getValue();
assertThat(shardRecoveryResponses.size(), equalTo(totalShards));
for (ShardRecoveryResponse shardResponse : shardRecoveryResponses) {
assertThat(shardResponse.recoveryState().getType(), equalTo(RecoveryState.Type.SNAPSHOT));
assertThat(shardResponse.recoveryState().getStage(), equalTo(RecoveryState.Stage.DONE));
assertNotNull(shardResponse.recoveryState().getRestoreSource());
assertThat(shardResponse.recoveryState().getTargetNode().getName(), equalTo(nodeA));
}
}
}