private void ensureActiveShardCopies(final int shardId, final int copyCount) throws Exception {
assertBusy(
new Runnable() {
@Override
public void run() {
ClusterState state = client().admin().cluster().prepareState().get().getState();
assertThat(state.routingTable().index("idx"), not(nullValue()));
assertThat(state.routingTable().index("idx").shard(shardId), not(nullValue()));
assertThat(
state.routingTable().index("idx").shard(shardId).activeShards().size(),
equalTo(copyCount));
ClusterHealthResponse healthResponse =
client().admin().cluster().prepareHealth("idx").setWaitForRelocatingShards(0).get();
assertThat(healthResponse.isTimedOut(), equalTo(false));
RecoveryResponse recoveryResponse =
client().admin().indices().prepareRecoveries("idx").setActiveOnly(true).get();
assertThat(recoveryResponse.shardResponses().get("idx").size(), equalTo(0));
}
});
}
public void testDelayedMappingPropagationOnReplica() throws Exception {
// This is essentially the same thing as testDelayedMappingPropagationOnPrimary
// but for replicas
// Here we want to test that everything goes well if the mappings that
// are needed for a document are not available on the replica at the
// time of indexing it
final List<String> nodeNames = internalCluster().startNodesAsync(2).get();
assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());
final String master = internalCluster().getMasterName();
assertThat(nodeNames, hasItem(master));
String otherNode = null;
for (String node : nodeNames) {
if (node.equals(master) == false) {
otherNode = node;
break;
}
}
assertNotNull(otherNode);
// Force allocation of the primary on the master node by first only allocating on the master
// and then allowing all nodes so that the replica gets allocated on the other node
assertAcked(
prepareCreate("index")
.setSettings(
Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
.put("index.routing.allocation.include._name", master))
.get());
assertAcked(
client()
.admin()
.indices()
.prepareUpdateSettings("index")
.setSettings(Settings.builder().put("index.routing.allocation.include._name", ""))
.get());
ensureGreen();
// Check routing tables
ClusterState state = client().admin().cluster().prepareState().get().getState();
assertEquals(master, state.nodes().masterNode().name());
List<ShardRouting> shards = state.routingTable().allShards("index");
assertThat(shards, hasSize(2));
for (ShardRouting shard : shards) {
if (shard.primary()) {
// primary must be on the master
assertEquals(state.nodes().masterNodeId(), shard.currentNodeId());
} else {
assertTrue(shard.active());
}
}
// Block cluster state processing on the replica
BlockClusterStateProcessing disruption =
new BlockClusterStateProcessing(otherNode, getRandom());
internalCluster().setDisruptionScheme(disruption);
disruption.startDisrupting();
final AtomicReference<Object> putMappingResponse = new AtomicReference<>();
client()
.admin()
.indices()
.preparePutMapping("index")
.setType("type")
.setSource("field", "type=long")
.execute(
new ActionListener<PutMappingResponse>() {
@Override
public void onResponse(PutMappingResponse response) {
putMappingResponse.set(response);
}
@Override
public void onFailure(Throwable e) {
putMappingResponse.set(e);
}
});
// Wait for mappings to be available on master
assertBusy(
new Runnable() {
@Override
public void run() {
final IndicesService indicesService =
internalCluster().getInstance(IndicesService.class, master);
final IndexService indexService = indicesService.indexServiceSafe("index");
assertNotNull(indexService);
final MapperService mapperService = indexService.mapperService();
DocumentMapper mapper = mapperService.documentMapper("type");
assertNotNull(mapper);
assertNotNull(mapper.mappers().getMapper("field"));
}
});
final AtomicReference<Object> docIndexResponse = new AtomicReference<>();
client()
.prepareIndex("index", "type", "1")
.setSource("field", 42)
.execute(
new ActionListener<IndexResponse>() {
@Override
public void onResponse(IndexResponse response) {
docIndexResponse.set(response);
}
@Override
public void onFailure(Throwable e) {
docIndexResponse.set(e);
}
});
// Wait for document to be indexed on primary
assertBusy(
new Runnable() {
@Override
public void run() {
assertTrue(
client()
.prepareGet("index", "type", "1")
.setPreference("_primary")
.get()
.isExists());
}
});
// The mappings have not been propagated to the replica yet as a consequence the document count
// not be indexed
// We wait on purpose to make sure that the document is not indexed because the shard operation
// is stalled
// and not just because it takes time to replicate the indexing request to the replica
Thread.sleep(100);
assertThat(putMappingResponse.get(), equalTo(null));
assertThat(docIndexResponse.get(), equalTo(null));
// Now make sure the indexing request finishes successfully
disruption.stopDisrupting();
assertBusy(
new Runnable() {
@Override
public void run() {
assertThat(putMappingResponse.get(), instanceOf(PutMappingResponse.class));
PutMappingResponse resp = (PutMappingResponse) putMappingResponse.get();
assertTrue(resp.isAcknowledged());
assertThat(docIndexResponse.get(), instanceOf(IndexResponse.class));
IndexResponse docResp = (IndexResponse) docIndexResponse.get();
assertEquals(
Arrays.toString(docResp.getShardInfo().getFailures()),
2,
docResp.getShardInfo().getTotal()); // both shards should have succeeded
}
});
}
public void testDelayedMappingPropagationOnPrimary() throws Exception {
// Here we want to test that things go well if there is a first request
// that adds mappings but before mappings are propagated to all nodes
// another index request introduces the same mapping. The master node
// will reply immediately since it did not change the cluster state
// but the change might not be on the node that performed the indexing
// operation yet
Settings settings = Settings.builder().put(DiscoverySettings.PUBLISH_TIMEOUT, "0ms").build();
final List<String> nodeNames = internalCluster().startNodesAsync(2, settings).get();
assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());
final String master = internalCluster().getMasterName();
assertThat(nodeNames, hasItem(master));
String otherNode = null;
for (String node : nodeNames) {
if (node.equals(master) == false) {
otherNode = node;
break;
}
}
assertNotNull(otherNode);
// Don't allocate the shard on the master node
assertAcked(
prepareCreate("index")
.setSettings(
Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put("index.routing.allocation.exclude._name", master))
.get());
ensureGreen();
// Check routing tables
ClusterState state = client().admin().cluster().prepareState().get().getState();
assertEquals(master, state.nodes().masterNode().name());
List<ShardRouting> shards = state.routingTable().allShards("index");
assertThat(shards, hasSize(1));
for (ShardRouting shard : shards) {
if (shard.primary()) {
// primary must not be on the master node
assertFalse(state.nodes().masterNodeId().equals(shard.currentNodeId()));
} else {
fail(); // only primaries
}
}
// Block cluster state processing where our shard is
BlockClusterStateProcessing disruption =
new BlockClusterStateProcessing(otherNode, getRandom());
internalCluster().setDisruptionScheme(disruption);
disruption.startDisrupting();
// Add a new mapping...
final AtomicReference<Object> putMappingResponse = new AtomicReference<>();
client()
.admin()
.indices()
.preparePutMapping("index")
.setType("type")
.setSource("field", "type=long")
.execute(
new ActionListener<PutMappingResponse>() {
@Override
public void onResponse(PutMappingResponse response) {
putMappingResponse.set(response);
}
@Override
public void onFailure(Throwable e) {
putMappingResponse.set(e);
}
});
// ...and wait for mappings to be available on master
assertBusy(
new Runnable() {
@Override
public void run() {
ImmutableOpenMap<String, MappingMetaData> indexMappings =
client()
.admin()
.indices()
.prepareGetMappings("index")
.get()
.getMappings()
.get("index");
assertNotNull(indexMappings);
MappingMetaData typeMappings = indexMappings.get("type");
assertNotNull(typeMappings);
Object properties;
try {
properties = typeMappings.getSourceAsMap().get("properties");
} catch (IOException e) {
throw new AssertionError(e);
}
assertNotNull(properties);
Object fieldMapping = ((Map<String, Object>) properties).get("field");
assertNotNull(fieldMapping);
}
});
final AtomicReference<Object> docIndexResponse = new AtomicReference<>();
client()
.prepareIndex("index", "type", "1")
.setSource("field", 42)
.execute(
new ActionListener<IndexResponse>() {
@Override
public void onResponse(IndexResponse response) {
docIndexResponse.set(response);
}
@Override
public void onFailure(Throwable e) {
docIndexResponse.set(e);
}
});
// Wait a bit to make sure that the reason why we did not get a response
// is that cluster state processing is blocked and not just that it takes
// time to process the indexing request
Thread.sleep(100);
assertThat(putMappingResponse.get(), equalTo(null));
assertThat(docIndexResponse.get(), equalTo(null));
// Now make sure the indexing request finishes successfully
disruption.stopDisrupting();
assertBusy(
new Runnable() {
@Override
public void run() {
assertThat(putMappingResponse.get(), instanceOf(PutMappingResponse.class));
PutMappingResponse resp = (PutMappingResponse) putMappingResponse.get();
assertTrue(resp.isAcknowledged());
assertThat(docIndexResponse.get(), instanceOf(IndexResponse.class));
IndexResponse docResp = (IndexResponse) docIndexResponse.get();
assertEquals(
Arrays.toString(docResp.getShardInfo().getFailures()),
1,
docResp.getShardInfo().getTotal());
}
});
}
@Test
public void testCancellationCleansTempFiles() throws Exception {
final String indexName = "test";
final String p_node = internalCluster().startNode();
client()
.admin()
.indices()
.prepareCreate(indexName)
.setSettings(
Settings.builder()
.put(
IndexMetaData.SETTING_NUMBER_OF_SHARDS,
1,
IndexMetaData.SETTING_NUMBER_OF_REPLICAS,
0))
.get();
internalCluster().startNodesAsync(2).get();
List<IndexRequestBuilder> requests = new ArrayList<>();
int numDocs = scaledRandomIntBetween(25, 250);
for (int i = 0; i < numDocs; i++) {
requests.add(client().prepareIndex(indexName, "type").setCreate(true).setSource("{}"));
}
indexRandom(true, requests);
assertFalse(
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForNodes("3")
.setWaitForGreenStatus()
.get()
.isTimedOut());
flush();
int allowedFailures = randomIntBetween(3, 10);
logger.info("--> blocking recoveries from primary (allowed failures: [{}])", allowedFailures);
CountDownLatch corruptionCount = new CountDownLatch(allowedFailures);
ClusterService clusterService = internalCluster().getInstance(ClusterService.class, p_node);
MockTransportService mockTransportService =
(MockTransportService) internalCluster().getInstance(TransportService.class, p_node);
for (DiscoveryNode node : clusterService.state().nodes()) {
if (!node.equals(clusterService.localNode())) {
mockTransportService.addDelegate(
node, new RecoveryCorruption(mockTransportService.original(), corruptionCount));
}
}
client()
.admin()
.indices()
.prepareUpdateSettings(indexName)
.setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1))
.get();
corruptionCount.await();
logger.info("--> stopping replica assignment");
assertAcked(
client()
.admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(
Settings.builder()
.put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "none")));
logger.info("--> wait for all replica shards to be removed, on all nodes");
assertBusy(
new Runnable() {
@Override
public void run() {
for (String node : internalCluster().getNodeNames()) {
if (node.equals(p_node)) {
continue;
}
ClusterState state =
client(node).admin().cluster().prepareState().setLocal(true).get().getState();
assertThat(
node + " indicates assigned replicas",
state
.getRoutingTable()
.index(indexName)
.shardsWithState(ShardRoutingState.UNASSIGNED)
.size(),
equalTo(1));
}
}
});
logger.info("--> verifying no temporary recoveries are left");
for (String node : internalCluster().getNodeNames()) {
NodeEnvironment nodeEnvironment = internalCluster().getInstance(NodeEnvironment.class, node);
for (final Path shardLoc : nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))) {
if (Files.exists(shardLoc)) {
assertBusy(
new Runnable() {
@Override
public void run() {
try {
Files.walkFileTree(
shardLoc,
new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
assertThat(
"found a temporary recovery file: " + file,
file.getFileName().toString(),
not(startsWith("recovery.")));
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
throw new AssertionError(
"failed to walk file tree starting at [" + shardLoc + "]", e);
}
}
});
}
}
}
}
@Test
public void testNoMasterActions() throws Exception {
// note, sometimes, we want to check with the fact that an index gets created, sometimes not...
boolean autoCreateIndex = randomBoolean();
logger.info("auto_create_index set to {}", autoCreateIndex);
Settings settings =
settingsBuilder()
.put("discovery.type", "zen")
.put("action.auto_create_index", autoCreateIndex)
.put("discovery.zen.minimum_master_nodes", 2)
.put("discovery.zen.ping_timeout", "200ms")
.put("discovery.initial_state_timeout", "500ms")
.put(DiscoverySettings.NO_MASTER_BLOCK, "all")
.build();
TimeValue timeout = TimeValue.timeValueMillis(200);
internalCluster().startNode(settings);
// start a second node, create an index, and then shut it down so we have no master block
internalCluster().startNode(settings);
createIndex("test");
client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();
internalCluster().stopRandomDataNode();
assertBusy(
new Runnable() {
@Override
public void run() {
ClusterState state =
client()
.admin()
.cluster()
.prepareState()
.setLocal(true)
.execute()
.actionGet()
.getState();
assertTrue(state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID));
}
});
assertThrows(
client().prepareGet("test", "type1", "1"),
ClusterBlockException.class,
RestStatus.SERVICE_UNAVAILABLE);
assertThrows(
client().prepareGet("no_index", "type1", "1"),
ClusterBlockException.class,
RestStatus.SERVICE_UNAVAILABLE);
assertThrows(
client().prepareMultiGet().add("test", "type1", "1"),
ClusterBlockException.class,
RestStatus.SERVICE_UNAVAILABLE);
assertThrows(
client().prepareMultiGet().add("no_index", "type1", "1"),
ClusterBlockException.class,
RestStatus.SERVICE_UNAVAILABLE);
PercolateSourceBuilder percolateSource = new PercolateSourceBuilder();
percolateSource.setDoc(docBuilder().setDoc(new HashMap()));
assertThrows(
client()
.preparePercolate()
.setIndices("test")
.setDocumentType("type1")
.setSource(percolateSource),
ClusterBlockException.class,
RestStatus.SERVICE_UNAVAILABLE);
percolateSource = new PercolateSourceBuilder();
percolateSource.setDoc(docBuilder().setDoc(new HashMap()));
assertThrows(
client()
.preparePercolate()
.setIndices("no_index")
.setDocumentType("type1")
.setSource(percolateSource),
ClusterBlockException.class,
RestStatus.SERVICE_UNAVAILABLE);
assertThrows(
client().admin().indices().prepareAnalyze("test", "this is a test"),
ClusterBlockException.class,
RestStatus.SERVICE_UNAVAILABLE);
assertThrows(
client().admin().indices().prepareAnalyze("no_index", "this is a test"),
ClusterBlockException.class,
RestStatus.SERVICE_UNAVAILABLE);
assertThrows(
client().prepareCount("test"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE);
assertThrows(
client().prepareCount("no_index"),
ClusterBlockException.class,
RestStatus.SERVICE_UNAVAILABLE);
checkWriteAction(
false,
timeout,
client()
.prepareUpdate("test", "type1", "1")
.setScript("test script", ScriptService.ScriptType.INLINE)
.setTimeout(timeout));
checkWriteAction(
autoCreateIndex,
timeout,
client()
.prepareUpdate("no_index", "type1", "1")
.setScript("test script", ScriptService.ScriptType.INLINE)
.setTimeout(timeout));
checkWriteAction(
false,
timeout,
client()
.prepareIndex("test", "type1", "1")
.setSource(XContentFactory.jsonBuilder().startObject().endObject())
.setTimeout(timeout));
checkWriteAction(
autoCreateIndex,
timeout,
client()
.prepareIndex("no_index", "type1", "1")
.setSource(XContentFactory.jsonBuilder().startObject().endObject())
.setTimeout(timeout));
BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
bulkRequestBuilder.add(
client()
.prepareIndex("test", "type1", "1")
.setSource(XContentFactory.jsonBuilder().startObject().endObject()));
bulkRequestBuilder.add(
client()
.prepareIndex("test", "type1", "2")
.setSource(XContentFactory.jsonBuilder().startObject().endObject()));
checkBulkAction(false, bulkRequestBuilder);
bulkRequestBuilder = client().prepareBulk();
bulkRequestBuilder.add(
client()
.prepareIndex("no_index", "type1", "1")
.setSource(XContentFactory.jsonBuilder().startObject().endObject()));
bulkRequestBuilder.add(
client()
.prepareIndex("no_index", "type1", "2")
.setSource(XContentFactory.jsonBuilder().startObject().endObject()));
checkBulkAction(autoCreateIndex, bulkRequestBuilder);
internalCluster().startNode(settings);
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForGreenStatus()
.setWaitForNodes("2")
.execute()
.actionGet();
}
