public void testShardActiveElsewhereDoesNotDeleteAnother() throws Exception {
InternalTestCluster.Async<String> masterFuture =
internalCluster()
.startNodeAsync(
Settings.builder()
.put(
Node.NODE_MASTER_SETTING.getKey(),
true,
Node.NODE_DATA_SETTING.getKey(),
false)
.build());
InternalTestCluster.Async<List<String>> nodesFutures =
internalCluster()
.startNodesAsync(
4,
Settings.builder()
.put(
Node.NODE_MASTER_SETTING.getKey(),
false,
Node.NODE_DATA_SETTING.getKey(),
true)
.build());
final String masterNode = masterFuture.get();
final String node1 = nodesFutures.get().get(0);
final String node2 = nodesFutures.get().get(1);
final String node3 = nodesFutures.get().get(2);
// we will use this later on, handy to start now to make sure it has a different data folder
// that node 1,2 &3
final String node4 = nodesFutures.get().get(3);
assertAcked(
prepareCreate("test")
.setSettings(
Settings.builder()
.put(indexSettings())
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 3)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
.put(
IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + "_name",
node4)));
assertFalse(
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForNoRelocatingShards(true)
.setWaitForGreenStatus()
.setWaitForNodes("5")
.get()
.isTimedOut());
// disable allocation to control the situation more easily
assertAcked(
client()
.admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(
Settings.builder()
.put(
EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(),
"none")));
logger.debug("--> shutting down two random nodes");
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node1, node2, node3));
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node1, node2, node3));
logger.debug("--> verifying index is red");
ClusterHealthResponse health =
client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
if (health.getStatus() != ClusterHealthStatus.RED) {
logClusterState();
fail("cluster didn't become red, despite of shutting 2 of 3 nodes");
}
logger.debug("--> allowing index to be assigned to node [{}]", node4);
assertAcked(
client()
.admin()
.indices()
.prepareUpdateSettings("test")
.setSettings(
Settings.builder()
.put(
IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + "_name",
"NONE")));
assertAcked(
client()
.admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(
Settings.builder()
.put(
EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(),
"all")));
logger.debug("--> waiting for shards to recover on [{}]", node4);
// we have to do this in two steps as we now do async shard fetching before assigning, so the
// change to the
// allocation filtering may not have immediate effect
// TODO: we should add an easier to do this. It's too much of a song and dance..
Index index = resolveIndex("test");
assertBusy(
new Runnable() {
@Override
public void run() {
assertTrue(internalCluster().getInstance(IndicesService.class, node4).hasIndex(index));
}
});
// wait for 4 active shards - we should have lost one shard
assertFalse(
client().admin().cluster().prepareHealth().setWaitForActiveShards(4).get().isTimedOut());
// disable allocation again to control concurrency a bit and allow shard active to kick in
// before allocation
assertAcked(
client()
.admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(
Settings.builder()
.put(
EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(),
"none")));
logger.debug("--> starting the two old nodes back");
internalCluster()
.startNodesAsync(
2,
Settings.builder()
.put(
Node.NODE_MASTER_SETTING.getKey(), false, Node.NODE_DATA_SETTING.getKey(), true)
.build());
assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("5").get().isTimedOut());
assertAcked(
client()
.admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(
Settings.builder()
.put(
EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(),
"all")));
logger.debug("--> waiting for the lost shard to be recovered");
ensureGreen("test");
}
void setupCluster() throws Exception {
InternalTestCluster.Async<List<String>> replicas =
internalCluster().startNodesAsync(1); // for replicas
Path baseTempDir = createTempDir();
// start single data path node
Settings.Builder nodeSettings =
Settings.builder()
.put(
Environment.PATH_DATA_SETTING.getKey(),
baseTempDir.resolve("single-path").toAbsolutePath())
.put(
Node.NODE_MASTER_SETTING.getKey(),
false); // workaround for dangling index loading issue when node is master
InternalTestCluster.Async<String> singleDataPathNode =
internalCluster().startNodeAsync(nodeSettings.build());
// start multi data path node
nodeSettings =
Settings.builder()
.put(
Environment.PATH_DATA_SETTING.getKey(),
baseTempDir.resolve("multi-path1").toAbsolutePath()
+ ","
+ baseTempDir.resolve("multi-path2").toAbsolutePath())
.put(
Node.NODE_MASTER_SETTING.getKey(),
false); // workaround for dangling index loading issue when node is master
InternalTestCluster.Async<String> multiDataPathNode =
internalCluster().startNodeAsync(nodeSettings.build());
// find single data path dir
singleDataPathNodeName = singleDataPathNode.get();
Path[] nodePaths =
internalCluster()
.getInstance(NodeEnvironment.class, singleDataPathNodeName)
.nodeDataPaths();
assertEquals(1, nodePaths.length);
singleDataPath = nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER);
assertFalse(Files.exists(singleDataPath));
Files.createDirectories(singleDataPath);
logger.info("--> Single data path: {}", singleDataPath);
// find multi data path dirs
multiDataPathNodeName = multiDataPathNode.get();
nodePaths =
internalCluster().getInstance(NodeEnvironment.class, multiDataPathNodeName).nodeDataPaths();
assertEquals(2, nodePaths.length);
multiDataPath =
new Path[] {
nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER),
nodePaths[1].resolve(NodeEnvironment.INDICES_FOLDER)
};
assertFalse(Files.exists(multiDataPath[0]));
assertFalse(Files.exists(multiDataPath[1]));
Files.createDirectories(multiDataPath[0]);
Files.createDirectories(multiDataPath[1]);
logger.info("--> Multi data paths: {}, {}", multiDataPath[0], multiDataPath[1]);
replicas.get(); // wait for replicas
}