public void testIndexCleanup() throws Exception { final String masterNode = internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false)); final String node_1 = internalCluster() .startNode(Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false)); final String node_2 = internalCluster() .startNode(Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false)); logger.info("--> creating index [test] with one shard and on replica"); assertAcked( prepareCreate("test") .setSettings( Settings.builder() .put(indexSettings()) .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1))); ensureGreen("test"); ClusterState state = client().admin().cluster().prepareState().get().getState(); Index index = state.metaData().index("test").getIndex(); logger.info("--> making sure that shard and its replica are allocated on node_1 and node_2"); assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true)); assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(true)); assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true)); assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(true)); logger.info("--> starting node server3"); final String node_3 = internalCluster() .startNode(Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false)); logger.info("--> running cluster_health"); ClusterHealthResponse clusterHealth = client() .admin() .cluster() .prepareHealth() .setWaitForNodes("4") .setWaitForNoRelocatingShards(true) .get(); assertThat(clusterHealth.isTimedOut(), equalTo(false)); assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true)); assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(true)); assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true)); assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(true)); assertThat(Files.exists(shardDirectory(node_3, index, 0)), equalTo(false)); assertThat(Files.exists(indexDirectory(node_3, index)), equalTo(false)); logger.info("--> move shard from node_1 to node_3, and wait for relocation to finish"); if (randomBoolean()) { // sometimes add cluster-state delay to trigger observers in // IndicesStore.ShardActiveRequestHandler SingleNodeDisruption disruption = new BlockClusterStateProcessing(node_3, random()); internalCluster().setDisruptionScheme(disruption); MockTransportService transportServiceNode3 = (MockTransportService) internalCluster().getInstance(TransportService.class, node_3); CountDownLatch beginRelocationLatch = new CountDownLatch(1); CountDownLatch endRelocationLatch = new CountDownLatch(1); transportServiceNode3.addTracer( new ReclocationStartEndTracer(logger, beginRelocationLatch, endRelocationLatch)); internalCluster() .client() .admin() .cluster() .prepareReroute() .add(new MoveAllocationCommand("test", 0, node_1, node_3)) .get(); // wait for relocation to start beginRelocationLatch.await(); disruption.startDisrupting(); // wait for relocation to finish endRelocationLatch.await(); // wait a little so that cluster state observer is registered sleep(50); disruption.stopDisrupting(); } else { internalCluster() .client() .admin() .cluster() .prepareReroute() .add(new MoveAllocationCommand("test", 0, node_1, node_3)) .get(); } clusterHealth = client().admin().cluster().prepareHealth().setWaitForNoRelocatingShards(true).get(); assertThat(clusterHealth.isTimedOut(), equalTo(false)); assertThat(waitForShardDeletion(node_1, index, 0), equalTo(false)); assertThat(waitForIndexDeletion(node_1, index), equalTo(false)); assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true)); assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(true)); assertThat(Files.exists(shardDirectory(node_3, index, 0)), equalTo(true)); assertThat(Files.exists(indexDirectory(node_3, index)), equalTo(true)); }