public void testFailToPublishWithLessThanMinMasterNodes() throws Exception {
final int masterNodes = randomIntBetween(1, 10);
MockNode master = createMockNode("master");
DiscoveryNodes.Builder discoveryNodesBuilder =
DiscoveryNodes.builder().add(master.discoveryNode);
for (int i = 1; i < masterNodes; i++) {
discoveryNodesBuilder.add(createMockNode("node" + i).discoveryNode);
}
final int dataNodes = randomIntBetween(0, 5);
final Settings dataSettings =
Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false).build();
for (int i = 0; i < dataNodes; i++) {
discoveryNodesBuilder.add(createMockNode("data_" + i, dataSettings, null).discoveryNode);
}
discoveryNodesBuilder
.localNodeId(master.discoveryNode.getId())
.masterNodeId(master.discoveryNode.getId());
DiscoveryNodes discoveryNodes = discoveryNodesBuilder.build();
MetaData metaData = MetaData.EMPTY_META_DATA;
ClusterState clusterState =
ClusterState.builder(CLUSTER_NAME).metaData(metaData).nodes(discoveryNodes).build();
ClusterState previousState = master.clusterState;
try {
publishState(
master.action, clusterState, previousState, masterNodes + randomIntBetween(1, 5));
fail("cluster state publishing didn't fail despite of not having enough nodes");
} catch (Discovery.FailedToCommitClusterStateException expected) {
logger.debug("failed to publish as expected", expected);
}
}
public static boolean nodeRequiresLocalStorage(Settings settings) {
boolean localStorageEnable = Node.NODE_LOCAL_STORAGE_SETTING.get(settings);
if (localStorageEnable == false
&& (Node.NODE_DATA_SETTING.get(settings) || Node.NODE_MASTER_SETTING.get(settings))) {
// TODO: make this a proper setting validation logic, requiring multi-settings validation
throw new IllegalArgumentException("storage can not be disabled for master and data nodes");
}
return localStorageEnable;
}
/** Creates a DiscoveryNode representing the local node. */
public static DiscoveryNode createLocal(
Settings settings, TransportAddress publishAddress, String nodeIdSupplier) {
Map<String, String> attributes = new HashMap<>(Node.NODE_ATTRIBUTES.get(settings).getAsMap());
Set<DiscoveryNode.Role> roles = new HashSet<>();
if (Node.NODE_INGEST_SETTING.get(settings)) {
roles.add(DiscoveryNode.Role.INGEST);
}
if (Node.NODE_MASTER_SETTING.get(settings)) {
roles.add(DiscoveryNode.Role.MASTER);
}
if (Node.NODE_DATA_SETTING.get(settings)) {
roles.add(DiscoveryNode.Role.DATA);
}
return new DiscoveryNode(
Node.NODE_NAME_SETTING.get(settings),
nodeIdSupplier,
publishAddress,
attributes,
roles,
Version.CURRENT);
}
public void testPublishingWithSendingErrors() throws Exception {
int goodNodes = randomIntBetween(2, 5);
int errorNodes = randomIntBetween(1, 5);
int timeOutNodes =
randomBoolean()
? 0
: randomIntBetween(1, 5); // adding timeout nodes will force timeout errors
final int numberOfMasterNodes = goodNodes + errorNodes + timeOutNodes + 1; // master
final boolean expectingToCommit = randomBoolean();
Settings.Builder settings = Settings.builder();
// make sure we have a reasonable timeout if we expect to timeout, o.w. one that will make the
// test "hang"
settings
.put(
DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(),
expectingToCommit == false && timeOutNodes > 0 ? "100ms" : "1h")
.put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "5ms"); // test is about committing
MockNode master = createMockNode("master", settings.build(), null);
// randomize things a bit
int[] nodeTypes = new int[goodNodes + errorNodes + timeOutNodes];
for (int i = 0; i < goodNodes; i++) {
nodeTypes[i] = 0;
}
for (int i = goodNodes; i < goodNodes + errorNodes; i++) {
nodeTypes[i] = 1;
}
for (int i = goodNodes + errorNodes; i < nodeTypes.length; i++) {
nodeTypes[i] = 2;
}
Collections.shuffle(Arrays.asList(nodeTypes), random());
DiscoveryNodes.Builder discoveryNodesBuilder =
DiscoveryNodes.builder().add(master.discoveryNode);
for (int i = 0; i < nodeTypes.length; i++) {
final MockNode mockNode = createMockNode("node" + i);
discoveryNodesBuilder.add(mockNode.discoveryNode);
switch (nodeTypes[i]) {
case 1:
mockNode.action.errorOnSend.set(true);
break;
case 2:
mockNode.action.timeoutOnSend.set(true);
break;
}
}
final int dataNodes = randomIntBetween(0, 3); // data nodes don't matter
for (int i = 0; i < dataNodes; i++) {
final MockNode mockNode =
createMockNode(
"data_" + i,
Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false).build(),
null);
discoveryNodesBuilder.add(mockNode.discoveryNode);
if (randomBoolean()) {
// we really don't care - just chaos monkey
mockNode.action.errorOnCommit.set(randomBoolean());
mockNode.action.errorOnSend.set(randomBoolean());
mockNode.action.timeoutOnCommit.set(randomBoolean());
mockNode.action.timeoutOnSend.set(randomBoolean());
}
}
final int minMasterNodes;
final String expectedBehavior;
if (expectingToCommit) {
minMasterNodes = randomIntBetween(0, goodNodes + 1); // count master
expectedBehavior = "succeed";
} else {
minMasterNodes = randomIntBetween(goodNodes + 2, numberOfMasterNodes); // +2 because of master
expectedBehavior = timeOutNodes > 0 ? "timeout" : "fail";
}
logger.info(
"--> expecting commit to {}. good nodes [{}], errors [{}], timeouts [{}]. min_master_nodes [{}]",
expectedBehavior,
goodNodes + 1,
errorNodes,
timeOutNodes,
minMasterNodes);
discoveryNodesBuilder
.localNodeId(master.discoveryNode.getId())
.masterNodeId(master.discoveryNode.getId());
DiscoveryNodes discoveryNodes = discoveryNodesBuilder.build();
MetaData metaData = MetaData.EMPTY_META_DATA;
ClusterState clusterState =
ClusterState.builder(CLUSTER_NAME).metaData(metaData).nodes(discoveryNodes).build();
ClusterState previousState = master.clusterState;
try {
publishState(master.action, clusterState, previousState, minMasterNodes);
if (expectingToCommit == false) {
fail("cluster state publishing didn't fail despite of not have enough nodes");
}
} catch (Discovery.FailedToCommitClusterStateException exception) {
logger.debug("failed to publish as expected", exception);
if (expectingToCommit) {
throw exception;
}
assertThat(exception.getMessage(), containsString(timeOutNodes > 0 ? "timed out" : "failed"));
}
}
/**
* Tests corruption that happens on the network layer and that the primary does not get affected
* by corruption that happens on the way to the replica. The file on disk stays uncorrupted
*/
public void testCorruptionOnNetworkLayer() throws ExecutionException, InterruptedException {
int numDocs = scaledRandomIntBetween(100, 1000);
internalCluster().ensureAtLeastNumDataNodes(2);
if (cluster().numDataNodes() < 3) {
internalCluster()
.startNode(
Settings.builder()
.put(Node.NODE_DATA_SETTING.getKey(), true)
.put(Node.NODE_MASTER_SETTING.getKey(), false));
}
NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
List<NodeStats> dataNodeStats = new ArrayList<>();
for (NodeStats stat : nodeStats.getNodes()) {
if (stat.getNode().isDataNode()) {
dataNodeStats.add(stat);
}
}
assertThat(dataNodeStats.size(), greaterThanOrEqualTo(2));
Collections.shuffle(dataNodeStats, random());
NodeStats primariesNode = dataNodeStats.get(0);
NodeStats unluckyNode = dataNodeStats.get(1);
assertAcked(
prepareCreate("test")
.setSettings(
Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
.put(
IndexMetaData.SETTING_NUMBER_OF_SHARDS,
between(1, 4)) // don't go crazy here it must recovery fast
// This does corrupt files on the replica, so we can't check:
.put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false)
.put(
"index.routing.allocation.include._name", primariesNode.getNode().getName())
.put(
EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(),
EnableAllocationDecider.Rebalance.NONE)));
ensureGreen();
IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
for (int i = 0; i < builders.length; i++) {
builders[i] = client().prepareIndex("test", "type").setSource("field", "value");
}
indexRandom(true, builders);
ensureGreen();
assertAllSuccessful(
client()
.admin()
.indices()
.prepareFlush()
.setForce(true)
.setWaitIfOngoing(true)
.execute()
.actionGet());
// we have to flush at least once here since we don't corrupt the translog
SearchResponse countResponse = client().prepareSearch().setSize(0).get();
assertHitCount(countResponse, numDocs);
final boolean truncate = randomBoolean();
for (NodeStats dataNode : dataNodeStats) {
MockTransportService mockTransportService =
((MockTransportService)
internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
mockTransportService.addDelegate(
internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()),
new MockTransportService.DelegateTransport(mockTransportService.original()) {
@Override
public void sendRequest(
DiscoveryNode node,
long requestId,
String action,
TransportRequest request,
TransportRequestOptions options)
throws IOException, TransportException {
if (action.equals(RecoveryTargetService.Actions.FILE_CHUNK)) {
RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
if (truncate && req.length() > 1) {
BytesRef bytesRef = req.content().toBytesRef();
BytesArray array =
new BytesArray(bytesRef.bytes, bytesRef.offset, (int) req.length() - 1);
request =
new RecoveryFileChunkRequest(
req.recoveryId(),
req.shardId(),
req.metadata(),
req.position(),
array,
req.lastChunk(),
req.totalTranslogOps(),
req.sourceThrottleTimeInNanos());
} else {
assert req.content().toBytesRef().bytes == req.content().toBytesRef().bytes
: "no internal reference!!";
final byte[] array = req.content().toBytesRef().bytes;
int i = randomIntBetween(0, req.content().length() - 1);
array[i] = (byte) ~array[i]; // flip one byte in the content
}
}
super.sendRequest(node, requestId, action, request, options);
}
});
}
Settings build =
Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1")
.put("index.routing.allocation.include._name", "*")
.build();
client().admin().indices().prepareUpdateSettings("test").setSettings(build).get();
client().admin().cluster().prepareReroute().get();
ClusterHealthResponse actionGet =
client()
.admin()
.cluster()
.health(Requests.clusterHealthRequest("test").waitForGreenStatus())
.actionGet();
if (actionGet.isTimedOut()) {
logger.info(
"ensureGreen timed out, cluster state:\n{}\n{}",
client().admin().cluster().prepareState().get().getState().prettyPrint(),
client().admin().cluster().preparePendingClusterTasks().get().prettyPrint());
assertThat("timed out waiting for green state", actionGet.isTimedOut(), equalTo(false));
}
// we are green so primaries got not corrupted.
// ensure that no shard is actually allocated on the unlucky node
ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
for (IndexShardRoutingTable table :
clusterStateResponse.getState().getRoutingTable().index("test")) {
for (ShardRouting routing : table) {
if (unluckyNode.getNode().getId().equals(routing.currentNodeId())) {
assertThat(routing.state(), not(equalTo(ShardRoutingState.STARTED)));
assertThat(routing.state(), not(equalTo(ShardRoutingState.RELOCATING)));
}
}
}
final int numIterations = scaledRandomIntBetween(5, 20);
for (int i = 0; i < numIterations; i++) {
SearchResponse response = client().prepareSearch().setSize(numDocs).get();
assertHitCount(response, numDocs);
}
}
public void testShardActiveElsewhereDoesNotDeleteAnother() throws Exception {
InternalTestCluster.Async<String> masterFuture =
internalCluster()
.startNodeAsync(
Settings.builder()
.put(
Node.NODE_MASTER_SETTING.getKey(),
true,
Node.NODE_DATA_SETTING.getKey(),
false)
.build());
InternalTestCluster.Async<List<String>> nodesFutures =
internalCluster()
.startNodesAsync(
4,
Settings.builder()
.put(
Node.NODE_MASTER_SETTING.getKey(),
false,
Node.NODE_DATA_SETTING.getKey(),
true)
.build());
final String masterNode = masterFuture.get();
final String node1 = nodesFutures.get().get(0);
final String node2 = nodesFutures.get().get(1);
final String node3 = nodesFutures.get().get(2);
// we will use this later on, handy to start now to make sure it has a different data folder
// that node 1,2 &3
final String node4 = nodesFutures.get().get(3);
assertAcked(
prepareCreate("test")
.setSettings(
Settings.builder()
.put(indexSettings())
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 3)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
.put(
IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + "_name",
node4)));
assertFalse(
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForNoRelocatingShards(true)
.setWaitForGreenStatus()
.setWaitForNodes("5")
.get()
.isTimedOut());
// disable allocation to control the situation more easily
assertAcked(
client()
.admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(
Settings.builder()
.put(
EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(),
"none")));
logger.debug("--> shutting down two random nodes");
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node1, node2, node3));
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node1, node2, node3));
logger.debug("--> verifying index is red");
ClusterHealthResponse health =
client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
if (health.getStatus() != ClusterHealthStatus.RED) {
logClusterState();
fail("cluster didn't become red, despite of shutting 2 of 3 nodes");
}
logger.debug("--> allowing index to be assigned to node [{}]", node4);
assertAcked(
client()
.admin()
.indices()
.prepareUpdateSettings("test")
.setSettings(
Settings.builder()
.put(
IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + "_name",
"NONE")));
assertAcked(
client()
.admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(
Settings.builder()
.put(
EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(),
"all")));
logger.debug("--> waiting for shards to recover on [{}]", node4);
// we have to do this in two steps as we now do async shard fetching before assigning, so the
// change to the
// allocation filtering may not have immediate effect
// TODO: we should add an easier to do this. It's too much of a song and dance..
Index index = resolveIndex("test");
assertBusy(
new Runnable() {
@Override
public void run() {
assertTrue(internalCluster().getInstance(IndicesService.class, node4).hasIndex(index));
}
});
// wait for 4 active shards - we should have lost one shard
assertFalse(
client().admin().cluster().prepareHealth().setWaitForActiveShards(4).get().isTimedOut());
// disable allocation again to control concurrency a bit and allow shard active to kick in
// before allocation
assertAcked(
client()
.admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(
Settings.builder()
.put(
EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(),
"none")));
logger.debug("--> starting the two old nodes back");
internalCluster()
.startNodesAsync(
2,
Settings.builder()
.put(
Node.NODE_MASTER_SETTING.getKey(), false, Node.NODE_DATA_SETTING.getKey(), true)
.build());
assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("5").get().isTimedOut());
assertAcked(
client()
.admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(
Settings.builder()
.put(
EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(),
"all")));
logger.debug("--> waiting for the lost shard to be recovered");
ensureGreen("test");
}
public void testIndexCleanup() throws Exception {
final String masterNode =
internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false));
final String node_1 =
internalCluster()
.startNode(Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false));
final String node_2 =
internalCluster()
.startNode(Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false));
logger.info("--> creating index [test] with one shard and on replica");
assertAcked(
prepareCreate("test")
.setSettings(
Settings.builder()
.put(indexSettings())
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)));
ensureGreen("test");
ClusterState state = client().admin().cluster().prepareState().get().getState();
Index index = state.metaData().index("test").getIndex();
logger.info("--> making sure that shard and its replica are allocated on node_1 and node_2");
assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(true));
logger.info("--> starting node server3");
final String node_3 =
internalCluster()
.startNode(Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false));
logger.info("--> running cluster_health");
ClusterHealthResponse clusterHealth =
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForNodes("4")
.setWaitForNoRelocatingShards(true)
.get();
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_3, index, 0)), equalTo(false));
assertThat(Files.exists(indexDirectory(node_3, index)), equalTo(false));
logger.info("--> move shard from node_1 to node_3, and wait for relocation to finish");
if (randomBoolean()) { // sometimes add cluster-state delay to trigger observers in
// IndicesStore.ShardActiveRequestHandler
SingleNodeDisruption disruption = new BlockClusterStateProcessing(node_3, random());
internalCluster().setDisruptionScheme(disruption);
MockTransportService transportServiceNode3 =
(MockTransportService) internalCluster().getInstance(TransportService.class, node_3);
CountDownLatch beginRelocationLatch = new CountDownLatch(1);
CountDownLatch endRelocationLatch = new CountDownLatch(1);
transportServiceNode3.addTracer(
new ReclocationStartEndTracer(logger, beginRelocationLatch, endRelocationLatch));
internalCluster()
.client()
.admin()
.cluster()
.prepareReroute()
.add(new MoveAllocationCommand("test", 0, node_1, node_3))
.get();
// wait for relocation to start
beginRelocationLatch.await();
disruption.startDisrupting();
// wait for relocation to finish
endRelocationLatch.await();
// wait a little so that cluster state observer is registered
sleep(50);
disruption.stopDisrupting();
} else {
internalCluster()
.client()
.admin()
.cluster()
.prepareReroute()
.add(new MoveAllocationCommand("test", 0, node_1, node_3))
.get();
}
clusterHealth =
client().admin().cluster().prepareHealth().setWaitForNoRelocatingShards(true).get();
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(waitForShardDeletion(node_1, index, 0), equalTo(false));
assertThat(waitForIndexDeletion(node_1, index), equalTo(false));
assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_3, index, 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_3, index)), equalTo(true));
}
public static boolean isMasterNode(Settings settings) {
return Node.NODE_MASTER_SETTING.get(settings);
}
void setupCluster() throws Exception {
InternalTestCluster.Async<List<String>> replicas =
internalCluster().startNodesAsync(1); // for replicas
Path baseTempDir = createTempDir();
// start single data path node
Settings.Builder nodeSettings =
Settings.builder()
.put(
Environment.PATH_DATA_SETTING.getKey(),
baseTempDir.resolve("single-path").toAbsolutePath())
.put(
Node.NODE_MASTER_SETTING.getKey(),
false); // workaround for dangling index loading issue when node is master
InternalTestCluster.Async<String> singleDataPathNode =
internalCluster().startNodeAsync(nodeSettings.build());
// start multi data path node
nodeSettings =
Settings.builder()
.put(
Environment.PATH_DATA_SETTING.getKey(),
baseTempDir.resolve("multi-path1").toAbsolutePath()
+ ","
+ baseTempDir.resolve("multi-path2").toAbsolutePath())
.put(
Node.NODE_MASTER_SETTING.getKey(),
false); // workaround for dangling index loading issue when node is master
InternalTestCluster.Async<String> multiDataPathNode =
internalCluster().startNodeAsync(nodeSettings.build());
// find single data path dir
singleDataPathNodeName = singleDataPathNode.get();
Path[] nodePaths =
internalCluster()
.getInstance(NodeEnvironment.class, singleDataPathNodeName)
.nodeDataPaths();
assertEquals(1, nodePaths.length);
singleDataPath = nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER);
assertFalse(Files.exists(singleDataPath));
Files.createDirectories(singleDataPath);
logger.info("--> Single data path: {}", singleDataPath);
// find multi data path dirs
multiDataPathNodeName = multiDataPathNode.get();
nodePaths =
internalCluster().getInstance(NodeEnvironment.class, multiDataPathNodeName).nodeDataPaths();
assertEquals(2, nodePaths.length);
multiDataPath =
new Path[] {
nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER),
nodePaths[1].resolve(NodeEnvironment.INDICES_FOLDER)
};
assertFalse(Files.exists(multiDataPath[0]));
assertFalse(Files.exists(multiDataPath[1]));
Files.createDirectories(multiDataPath[0]);
Files.createDirectories(multiDataPath[1]);
logger.info("--> Multi data paths: {}, {}", multiDataPath[0], multiDataPath[1]);
replicas.get(); // wait for replicas
}