@Test
@TestLogging(value = "cluster.service:TRACE")
public void testDeleteCreateInOneBulk() throws Exception {
internalCluster()
.startNodesAsync(
2, Settings.builder().put(DiscoveryModule.DISCOVERY_TYPE_KEY, "zen").build())
.get();
assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());
prepareCreate("test")
.setSettings(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, true)
.addMapping("type")
.get();
ensureGreen("test");
// now that the cluster is stable, remove publishing timeout
assertAcked(
client()
.admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(Settings.builder().put(DiscoverySettings.PUBLISH_TIMEOUT, "0")));
Set<String> nodes = new HashSet<>(Arrays.asList(internalCluster().getNodeNames()));
nodes.remove(internalCluster().getMasterName());
// block none master node.
BlockClusterStateProcessing disruption =
new BlockClusterStateProcessing(nodes.iterator().next(), getRandom());
internalCluster().setDisruptionScheme(disruption);
logger.info("--> indexing a doc");
index("test", "type", "1");
refresh();
disruption.startDisrupting();
logger.info("--> delete index and recreate it");
assertFalse(
client()
.admin()
.indices()
.prepareDelete("test")
.setTimeout("200ms")
.get()
.isAcknowledged());
assertFalse(
prepareCreate("test")
.setTimeout("200ms")
.setSettings(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, true)
.get()
.isAcknowledged());
logger.info("--> letting cluster proceed");
disruption.stopDisrupting();
ensureGreen(TimeValue.timeValueMinutes(30), "test");
assertHitCount(client().prepareSearch("test").get(), 0);
}
public void testDelayedMappingPropagationOnReplica() throws Exception {
// This is essentially the same thing as testDelayedMappingPropagationOnPrimary
// but for replicas
// Here we want to test that everything goes well if the mappings that
// are needed for a document are not available on the replica at the
// time of indexing it
final List<String> nodeNames = internalCluster().startNodesAsync(2).get();
assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());
final String master = internalCluster().getMasterName();
assertThat(nodeNames, hasItem(master));
String otherNode = null;
for (String node : nodeNames) {
if (node.equals(master) == false) {
otherNode = node;
break;
}
}
assertNotNull(otherNode);
// Force allocation of the primary on the master node by first only allocating on the master
// and then allowing all nodes so that the replica gets allocated on the other node
assertAcked(
prepareCreate("index")
.setSettings(
Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
.put("index.routing.allocation.include._name", master))
.get());
assertAcked(
client()
.admin()
.indices()
.prepareUpdateSettings("index")
.setSettings(Settings.builder().put("index.routing.allocation.include._name", ""))
.get());
ensureGreen();
// Check routing tables
ClusterState state = client().admin().cluster().prepareState().get().getState();
assertEquals(master, state.nodes().masterNode().name());
List<ShardRouting> shards = state.routingTable().allShards("index");
assertThat(shards, hasSize(2));
for (ShardRouting shard : shards) {
if (shard.primary()) {
// primary must be on the master
assertEquals(state.nodes().masterNodeId(), shard.currentNodeId());
} else {
assertTrue(shard.active());
}
}
// Block cluster state processing on the replica
BlockClusterStateProcessing disruption =
new BlockClusterStateProcessing(otherNode, getRandom());
internalCluster().setDisruptionScheme(disruption);
disruption.startDisrupting();
final AtomicReference<Object> putMappingResponse = new AtomicReference<>();
client()
.admin()
.indices()
.preparePutMapping("index")
.setType("type")
.setSource("field", "type=long")
.execute(
new ActionListener<PutMappingResponse>() {
@Override
public void onResponse(PutMappingResponse response) {
putMappingResponse.set(response);
}
@Override
public void onFailure(Throwable e) {
putMappingResponse.set(e);
}
});
// Wait for mappings to be available on master
assertBusy(
new Runnable() {
@Override
public void run() {
final IndicesService indicesService =
internalCluster().getInstance(IndicesService.class, master);
final IndexService indexService = indicesService.indexServiceSafe("index");
assertNotNull(indexService);
final MapperService mapperService = indexService.mapperService();
DocumentMapper mapper = mapperService.documentMapper("type");
assertNotNull(mapper);
assertNotNull(mapper.mappers().getMapper("field"));
}
});
final AtomicReference<Object> docIndexResponse = new AtomicReference<>();
client()
.prepareIndex("index", "type", "1")
.setSource("field", 42)
.execute(
new ActionListener<IndexResponse>() {
@Override
public void onResponse(IndexResponse response) {
docIndexResponse.set(response);
}
@Override
public void onFailure(Throwable e) {
docIndexResponse.set(e);
}
});
// Wait for document to be indexed on primary
assertBusy(
new Runnable() {
@Override
public void run() {
assertTrue(
client()
.prepareGet("index", "type", "1")
.setPreference("_primary")
.get()
.isExists());
}
});
// The mappings have not been propagated to the replica yet as a consequence the document count
// not be indexed
// We wait on purpose to make sure that the document is not indexed because the shard operation
// is stalled
// and not just because it takes time to replicate the indexing request to the replica
Thread.sleep(100);
assertThat(putMappingResponse.get(), equalTo(null));
assertThat(docIndexResponse.get(), equalTo(null));
// Now make sure the indexing request finishes successfully
disruption.stopDisrupting();
assertBusy(
new Runnable() {
@Override
public void run() {
assertThat(putMappingResponse.get(), instanceOf(PutMappingResponse.class));
PutMappingResponse resp = (PutMappingResponse) putMappingResponse.get();
assertTrue(resp.isAcknowledged());
assertThat(docIndexResponse.get(), instanceOf(IndexResponse.class));
IndexResponse docResp = (IndexResponse) docIndexResponse.get();
assertEquals(
Arrays.toString(docResp.getShardInfo().getFailures()),
2,
docResp.getShardInfo().getTotal()); // both shards should have succeeded
}
});
}
public void testDelayedMappingPropagationOnPrimary() throws Exception {
// Here we want to test that things go well if there is a first request
// that adds mappings but before mappings are propagated to all nodes
// another index request introduces the same mapping. The master node
// will reply immediately since it did not change the cluster state
// but the change might not be on the node that performed the indexing
// operation yet
Settings settings = Settings.builder().put(DiscoverySettings.PUBLISH_TIMEOUT, "0ms").build();
final List<String> nodeNames = internalCluster().startNodesAsync(2, settings).get();
assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());
final String master = internalCluster().getMasterName();
assertThat(nodeNames, hasItem(master));
String otherNode = null;
for (String node : nodeNames) {
if (node.equals(master) == false) {
otherNode = node;
break;
}
}
assertNotNull(otherNode);
// Don't allocate the shard on the master node
assertAcked(
prepareCreate("index")
.setSettings(
Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put("index.routing.allocation.exclude._name", master))
.get());
ensureGreen();
// Check routing tables
ClusterState state = client().admin().cluster().prepareState().get().getState();
assertEquals(master, state.nodes().masterNode().name());
List<ShardRouting> shards = state.routingTable().allShards("index");
assertThat(shards, hasSize(1));
for (ShardRouting shard : shards) {
if (shard.primary()) {
// primary must not be on the master node
assertFalse(state.nodes().masterNodeId().equals(shard.currentNodeId()));
} else {
fail(); // only primaries
}
}
// Block cluster state processing where our shard is
BlockClusterStateProcessing disruption =
new BlockClusterStateProcessing(otherNode, getRandom());
internalCluster().setDisruptionScheme(disruption);
disruption.startDisrupting();
// Add a new mapping...
final AtomicReference<Object> putMappingResponse = new AtomicReference<>();
client()
.admin()
.indices()
.preparePutMapping("index")
.setType("type")
.setSource("field", "type=long")
.execute(
new ActionListener<PutMappingResponse>() {
@Override
public void onResponse(PutMappingResponse response) {
putMappingResponse.set(response);
}
@Override
public void onFailure(Throwable e) {
putMappingResponse.set(e);
}
});
// ...and wait for mappings to be available on master
assertBusy(
new Runnable() {
@Override
public void run() {
ImmutableOpenMap<String, MappingMetaData> indexMappings =
client()
.admin()
.indices()
.prepareGetMappings("index")
.get()
.getMappings()
.get("index");
assertNotNull(indexMappings);
MappingMetaData typeMappings = indexMappings.get("type");
assertNotNull(typeMappings);
Object properties;
try {
properties = typeMappings.getSourceAsMap().get("properties");
} catch (IOException e) {
throw new AssertionError(e);
}
assertNotNull(properties);
Object fieldMapping = ((Map<String, Object>) properties).get("field");
assertNotNull(fieldMapping);
}
});
final AtomicReference<Object> docIndexResponse = new AtomicReference<>();
client()
.prepareIndex("index", "type", "1")
.setSource("field", 42)
.execute(
new ActionListener<IndexResponse>() {
@Override
public void onResponse(IndexResponse response) {
docIndexResponse.set(response);
}
@Override
public void onFailure(Throwable e) {
docIndexResponse.set(e);
}
});
// Wait a bit to make sure that the reason why we did not get a response
// is that cluster state processing is blocked and not just that it takes
// time to process the indexing request
Thread.sleep(100);
assertThat(putMappingResponse.get(), equalTo(null));
assertThat(docIndexResponse.get(), equalTo(null));
// Now make sure the indexing request finishes successfully
disruption.stopDisrupting();
assertBusy(
new Runnable() {
@Override
public void run() {
assertThat(putMappingResponse.get(), instanceOf(PutMappingResponse.class));
PutMappingResponse resp = (PutMappingResponse) putMappingResponse.get();
assertTrue(resp.isAcknowledged());
assertThat(docIndexResponse.get(), instanceOf(IndexResponse.class));
IndexResponse docResp = (IndexResponse) docIndexResponse.get();
assertEquals(
Arrays.toString(docResp.getShardInfo().getFailures()),
1,
docResp.getShardInfo().getTotal());
}
});
}
