@Test
public void testNetworkPartitionDuringReplicaIndexOp() throws Exception {
final String INDEX = "testidx";
List<String> nodes = internalCluster().startNodesAsync(2, nodeSettings).get();
// Create index test with 1 shard, 1 replica and ensure it is green
createIndex(INDEX);
ensureGreen(INDEX);
// Disable allocation so the replica cannot be reallocated when it fails
Settings s =
ImmutableSettings.builder().put("cluster.routing.allocation.enable", "none").build();
client().admin().cluster().prepareUpdateSettings().setTransientSettings(s).get();
// Determine which node holds the primary shard
ClusterState state = getNodeClusterState(nodes.get(0));
IndexShardRoutingTable shard = state.getRoutingTable().index(INDEX).shard(0);
String primaryNode;
String replicaNode;
if (shard.getShards().get(0).primary()) {
primaryNode = nodes.get(0);
replicaNode = nodes.get(1);
} else {
primaryNode = nodes.get(1);
replicaNode = nodes.get(0);
}
logger.info("--> primary shard is on {}", primaryNode);
// Index a document to make sure everything works well
IndexResponse resp =
internalCluster()
.client(primaryNode)
.prepareIndex(INDEX, "doc")
.setSource("foo", "bar")
.get();
assertThat(
"document exists on primary node",
internalCluster()
.client(primaryNode)
.prepareGet(INDEX, "doc", resp.getId())
.setPreference("_only_local")
.get()
.isExists(),
equalTo(true));
assertThat(
"document exists on replica node",
internalCluster()
.client(replicaNode)
.prepareGet(INDEX, "doc", resp.getId())
.setPreference("_only_local")
.get()
.isExists(),
equalTo(true));
// Disrupt the network so indexing requests fail to replicate
logger.info("--> preventing index/replica operations");
TransportService mockTransportService =
internalCluster().getInstance(TransportService.class, primaryNode);
((MockTransportService) mockTransportService)
.addFailToSendNoConnectRule(
internalCluster().getInstance(Discovery.class, replicaNode).localNode(),
ImmutableSet.of(IndexAction.NAME + "[r]"));
mockTransportService = internalCluster().getInstance(TransportService.class, replicaNode);
((MockTransportService) mockTransportService)
.addFailToSendNoConnectRule(
internalCluster().getInstance(Discovery.class, primaryNode).localNode(),
ImmutableSet.of(IndexAction.NAME + "[r]"));
logger.info("--> indexing into primary");
// the replica shard should now be marked as failed because the replication operation will fail
resp =
internalCluster()
.client(primaryNode)
.prepareIndex(INDEX, "doc")
.setSource("foo", "baz")
.get();
// wait until the cluster reaches an exact yellow state, meaning replica has failed
assertBusy(
new Runnable() {
@Override
public void run() {
assertThat(
client().admin().cluster().prepareHealth().get().getStatus(),
equalTo(ClusterHealthStatus.YELLOW));
}
});
assertThat(
"document should still be indexed and available",
client().prepareGet(INDEX, "doc", resp.getId()).get().isExists(),
equalTo(true));
state = getNodeClusterState(randomFrom(nodes.toArray(Strings.EMPTY_ARRAY)));
RoutingNodes rn = state.routingNodes();
logger.info(
"--> counts: total: {}, unassigned: {}, initializing: {}, relocating: {}, started: {}",
rn.shards(
new Predicate<MutableShardRouting>() {
@Override
public boolean apply(
org.elasticsearch.cluster.routing.MutableShardRouting input) {
return true;
}
})
.size(),
rn.shardsWithState(UNASSIGNED).size(),
rn.shardsWithState(INITIALIZING).size(),
rn.shardsWithState(RELOCATING).size(),
rn.shardsWithState(STARTED).size());
logger.info(
"--> unassigned: {}, initializing: {}, relocating: {}, started: {}",
rn.shardsWithState(UNASSIGNED),
rn.shardsWithState(INITIALIZING),
rn.shardsWithState(RELOCATING),
rn.shardsWithState(STARTED));
assertThat(
"only a single shard is now active (replica should be failed and not reallocated)",
rn.shardsWithState(STARTED).size(),
equalTo(1));
}
@Test
public void testMockFailToSendNoConnectRule() {
serviceA.registerHandler(
"sayHello",
new BaseTransportRequestHandler<StringMessageRequest>() {
@Override
public StringMessageRequest newInstance() {
return new StringMessageRequest();
}
@Override
public String executor() {
return ThreadPool.Names.GENERIC;
}
@Override
public void messageReceived(StringMessageRequest request, TransportChannel channel)
throws Exception {
assertThat("moshe", equalTo(request.message));
throw new RuntimeException("bad message !!!");
}
});
serviceB.addFailToSendNoConnectRule(nodeA);
TransportFuture<StringMessageResponse> res =
serviceB.submitRequest(
nodeA,
"sayHello",
new StringMessageRequest("moshe"),
new BaseTransportResponseHandler<StringMessageResponse>() {
@Override
public StringMessageResponse newInstance() {
return new StringMessageResponse();
}
@Override
public String executor() {
return ThreadPool.Names.GENERIC;
}
@Override
public void handleResponse(StringMessageResponse response) {
assertThat("got response instead of exception", false, equalTo(true));
}
@Override
public void handleException(TransportException exp) {
assertThat(exp.getCause().getMessage(), endsWith("DISCONNECT: simulated"));
}
});
try {
res.txGet();
assertThat("exception should be thrown", false, equalTo(true));
} catch (Exception e) {
assertThat(e.getCause().getMessage(), endsWith("DISCONNECT: simulated"));
}
try {
serviceB.connectToNode(nodeA);
assertThat("exception should be thrown", false, equalTo(true));
} catch (ConnectTransportException e) {
// all is well
}
try {
serviceB.connectToNodeLight(nodeA);
assertThat("exception should be thrown", false, equalTo(true));
} catch (ConnectTransportException e) {
// all is well
}
serviceA.removeHandler("sayHello");
}
