private void ensureActiveShardCopies(final int shardId, final int copyCount) throws Exception { assertBusy( new Runnable() { @Override public void run() { ClusterState state = client().admin().cluster().prepareState().get().getState(); assertThat(state.routingTable().index("idx"), not(nullValue())); assertThat(state.routingTable().index("idx").shard(shardId), not(nullValue())); assertThat( state.routingTable().index("idx").shard(shardId).activeShards().size(), equalTo(copyCount)); ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth("idx").setWaitForRelocatingShards(0).get(); assertThat(healthResponse.isTimedOut(), equalTo(false)); RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("idx").setActiveOnly(true).get(); assertThat(recoveryResponse.shardResponses().get("idx").size(), equalTo(0)); } }); }
public void testDelayedMappingPropagationOnReplica() throws Exception { // This is essentially the same thing as testDelayedMappingPropagationOnPrimary // but for replicas // Here we want to test that everything goes well if the mappings that // are needed for a document are not available on the replica at the // time of indexing it final List<String> nodeNames = internalCluster().startNodesAsync(2).get(); assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut()); final String master = internalCluster().getMasterName(); assertThat(nodeNames, hasItem(master)); String otherNode = null; for (String node : nodeNames) { if (node.equals(master) == false) { otherNode = node; break; } } assertNotNull(otherNode); // Force allocation of the primary on the master node by first only allocating on the master // and then allowing all nodes so that the replica gets allocated on the other node assertAcked( prepareCreate("index") .setSettings( Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) .put("index.routing.allocation.include._name", master)) .get()); assertAcked( client() .admin() .indices() .prepareUpdateSettings("index") .setSettings(Settings.builder().put("index.routing.allocation.include._name", "")) .get()); ensureGreen(); // Check routing tables ClusterState state = client().admin().cluster().prepareState().get().getState(); assertEquals(master, state.nodes().masterNode().name()); List<ShardRouting> shards = state.routingTable().allShards("index"); assertThat(shards, hasSize(2)); for (ShardRouting shard : shards) { if (shard.primary()) { // primary must be on the master assertEquals(state.nodes().masterNodeId(), shard.currentNodeId()); } else { assertTrue(shard.active()); } } // Block cluster state processing on the replica BlockClusterStateProcessing disruption = new BlockClusterStateProcessing(otherNode, getRandom()); internalCluster().setDisruptionScheme(disruption); disruption.startDisrupting(); final AtomicReference<Object> putMappingResponse = new AtomicReference<>(); client() .admin() .indices() .preparePutMapping("index") .setType("type") .setSource("field", "type=long") .execute( new ActionListener<PutMappingResponse>() { @Override public void onResponse(PutMappingResponse response) { putMappingResponse.set(response); } @Override public void onFailure(Throwable e) { putMappingResponse.set(e); } }); // Wait for mappings to be available on master assertBusy( new Runnable() { @Override public void run() { final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, master); final IndexService indexService = indicesService.indexServiceSafe("index"); assertNotNull(indexService); final MapperService mapperService = indexService.mapperService(); DocumentMapper mapper = mapperService.documentMapper("type"); assertNotNull(mapper); assertNotNull(mapper.mappers().getMapper("field")); } }); final AtomicReference<Object> docIndexResponse = new AtomicReference<>(); client() .prepareIndex("index", "type", "1") .setSource("field", 42) .execute( new ActionListener<IndexResponse>() { @Override public void onResponse(IndexResponse response) { docIndexResponse.set(response); } @Override public void onFailure(Throwable e) { docIndexResponse.set(e); } }); // Wait for document to be indexed on primary assertBusy( new Runnable() { @Override public void run() { assertTrue( client() .prepareGet("index", "type", "1") .setPreference("_primary") .get() .isExists()); } }); // The mappings have not been propagated to the replica yet as a consequence the document count // not be indexed // We wait on purpose to make sure that the document is not indexed because the shard operation // is stalled // and not just because it takes time to replicate the indexing request to the replica Thread.sleep(100); assertThat(putMappingResponse.get(), equalTo(null)); assertThat(docIndexResponse.get(), equalTo(null)); // Now make sure the indexing request finishes successfully disruption.stopDisrupting(); assertBusy( new Runnable() { @Override public void run() { assertThat(putMappingResponse.get(), instanceOf(PutMappingResponse.class)); PutMappingResponse resp = (PutMappingResponse) putMappingResponse.get(); assertTrue(resp.isAcknowledged()); assertThat(docIndexResponse.get(), instanceOf(IndexResponse.class)); IndexResponse docResp = (IndexResponse) docIndexResponse.get(); assertEquals( Arrays.toString(docResp.getShardInfo().getFailures()), 2, docResp.getShardInfo().getTotal()); // both shards should have succeeded } }); }
public void testDelayedMappingPropagationOnPrimary() throws Exception { // Here we want to test that things go well if there is a first request // that adds mappings but before mappings are propagated to all nodes // another index request introduces the same mapping. The master node // will reply immediately since it did not change the cluster state // but the change might not be on the node that performed the indexing // operation yet Settings settings = Settings.builder().put(DiscoverySettings.PUBLISH_TIMEOUT, "0ms").build(); final List<String> nodeNames = internalCluster().startNodesAsync(2, settings).get(); assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut()); final String master = internalCluster().getMasterName(); assertThat(nodeNames, hasItem(master)); String otherNode = null; for (String node : nodeNames) { if (node.equals(master) == false) { otherNode = node; break; } } assertNotNull(otherNode); // Don't allocate the shard on the master node assertAcked( prepareCreate("index") .setSettings( Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .put("index.routing.allocation.exclude._name", master)) .get()); ensureGreen(); // Check routing tables ClusterState state = client().admin().cluster().prepareState().get().getState(); assertEquals(master, state.nodes().masterNode().name()); List<ShardRouting> shards = state.routingTable().allShards("index"); assertThat(shards, hasSize(1)); for (ShardRouting shard : shards) { if (shard.primary()) { // primary must not be on the master node assertFalse(state.nodes().masterNodeId().equals(shard.currentNodeId())); } else { fail(); // only primaries } } // Block cluster state processing where our shard is BlockClusterStateProcessing disruption = new BlockClusterStateProcessing(otherNode, getRandom()); internalCluster().setDisruptionScheme(disruption); disruption.startDisrupting(); // Add a new mapping... final AtomicReference<Object> putMappingResponse = new AtomicReference<>(); client() .admin() .indices() .preparePutMapping("index") .setType("type") .setSource("field", "type=long") .execute( new ActionListener<PutMappingResponse>() { @Override public void onResponse(PutMappingResponse response) { putMappingResponse.set(response); } @Override public void onFailure(Throwable e) { putMappingResponse.set(e); } }); // ...and wait for mappings to be available on master assertBusy( new Runnable() { @Override public void run() { ImmutableOpenMap<String, MappingMetaData> indexMappings = client() .admin() .indices() .prepareGetMappings("index") .get() .getMappings() .get("index"); assertNotNull(indexMappings); MappingMetaData typeMappings = indexMappings.get("type"); assertNotNull(typeMappings); Object properties; try { properties = typeMappings.getSourceAsMap().get("properties"); } catch (IOException e) { throw new AssertionError(e); } assertNotNull(properties); Object fieldMapping = ((Map<String, Object>) properties).get("field"); assertNotNull(fieldMapping); } }); final AtomicReference<Object> docIndexResponse = new AtomicReference<>(); client() .prepareIndex("index", "type", "1") .setSource("field", 42) .execute( new ActionListener<IndexResponse>() { @Override public void onResponse(IndexResponse response) { docIndexResponse.set(response); } @Override public void onFailure(Throwable e) { docIndexResponse.set(e); } }); // Wait a bit to make sure that the reason why we did not get a response // is that cluster state processing is blocked and not just that it takes // time to process the indexing request Thread.sleep(100); assertThat(putMappingResponse.get(), equalTo(null)); assertThat(docIndexResponse.get(), equalTo(null)); // Now make sure the indexing request finishes successfully disruption.stopDisrupting(); assertBusy( new Runnable() { @Override public void run() { assertThat(putMappingResponse.get(), instanceOf(PutMappingResponse.class)); PutMappingResponse resp = (PutMappingResponse) putMappingResponse.get(); assertTrue(resp.isAcknowledged()); assertThat(docIndexResponse.get(), instanceOf(IndexResponse.class)); IndexResponse docResp = (IndexResponse) docIndexResponse.get(); assertEquals( Arrays.toString(docResp.getShardInfo().getFailures()), 1, docResp.getShardInfo().getTotal()); } }); }
@Test public void testCancellationCleansTempFiles() throws Exception { final String indexName = "test"; final String p_node = internalCluster().startNode(); client() .admin() .indices() .prepareCreate(indexName) .setSettings( Settings.builder() .put( IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)) .get(); internalCluster().startNodesAsync(2).get(); List<IndexRequestBuilder> requests = new ArrayList<>(); int numDocs = scaledRandomIntBetween(25, 250); for (int i = 0; i < numDocs; i++) { requests.add(client().prepareIndex(indexName, "type").setCreate(true).setSource("{}")); } indexRandom(true, requests); assertFalse( client() .admin() .cluster() .prepareHealth() .setWaitForNodes("3") .setWaitForGreenStatus() .get() .isTimedOut()); flush(); int allowedFailures = randomIntBetween(3, 10); logger.info("--> blocking recoveries from primary (allowed failures: [{}])", allowedFailures); CountDownLatch corruptionCount = new CountDownLatch(allowedFailures); ClusterService clusterService = internalCluster().getInstance(ClusterService.class, p_node); MockTransportService mockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, p_node); for (DiscoveryNode node : clusterService.state().nodes()) { if (!node.equals(clusterService.localNode())) { mockTransportService.addDelegate( node, new RecoveryCorruption(mockTransportService.original(), corruptionCount)); } } client() .admin() .indices() .prepareUpdateSettings(indexName) .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)) .get(); corruptionCount.await(); logger.info("--> stopping replica assignment"); assertAcked( client() .admin() .cluster() .prepareUpdateSettings() .setTransientSettings( Settings.builder() .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "none"))); logger.info("--> wait for all replica shards to be removed, on all nodes"); assertBusy( new Runnable() { @Override public void run() { for (String node : internalCluster().getNodeNames()) { if (node.equals(p_node)) { continue; } ClusterState state = client(node).admin().cluster().prepareState().setLocal(true).get().getState(); assertThat( node + " indicates assigned replicas", state .getRoutingTable() .index(indexName) .shardsWithState(ShardRoutingState.UNASSIGNED) .size(), equalTo(1)); } } }); logger.info("--> verifying no temporary recoveries are left"); for (String node : internalCluster().getNodeNames()) { NodeEnvironment nodeEnvironment = internalCluster().getInstance(NodeEnvironment.class, node); for (final Path shardLoc : nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))) { if (Files.exists(shardLoc)) { assertBusy( new Runnable() { @Override public void run() { try { Files.walkFileTree( shardLoc, new SimpleFileVisitor<Path>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { assertThat( "found a temporary recovery file: " + file, file.getFileName().toString(), not(startsWith("recovery."))); return FileVisitResult.CONTINUE; } }); } catch (IOException e) { throw new AssertionError( "failed to walk file tree starting at [" + shardLoc + "]", e); } } }); } } } }
@Test public void testNoMasterActions() throws Exception { // note, sometimes, we want to check with the fact that an index gets created, sometimes not... boolean autoCreateIndex = randomBoolean(); logger.info("auto_create_index set to {}", autoCreateIndex); Settings settings = settingsBuilder() .put("discovery.type", "zen") .put("action.auto_create_index", autoCreateIndex) .put("discovery.zen.minimum_master_nodes", 2) .put("discovery.zen.ping_timeout", "200ms") .put("discovery.initial_state_timeout", "500ms") .put(DiscoverySettings.NO_MASTER_BLOCK, "all") .build(); TimeValue timeout = TimeValue.timeValueMillis(200); internalCluster().startNode(settings); // start a second node, create an index, and then shut it down so we have no master block internalCluster().startNode(settings); createIndex("test"); client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet(); internalCluster().stopRandomDataNode(); assertBusy( new Runnable() { @Override public void run() { ClusterState state = client() .admin() .cluster() .prepareState() .setLocal(true) .execute() .actionGet() .getState(); assertTrue(state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID)); } }); assertThrows( client().prepareGet("test", "type1", "1"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE); assertThrows( client().prepareGet("no_index", "type1", "1"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE); assertThrows( client().prepareMultiGet().add("test", "type1", "1"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE); assertThrows( client().prepareMultiGet().add("no_index", "type1", "1"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE); PercolateSourceBuilder percolateSource = new PercolateSourceBuilder(); percolateSource.setDoc(docBuilder().setDoc(new HashMap())); assertThrows( client() .preparePercolate() .setIndices("test") .setDocumentType("type1") .setSource(percolateSource), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE); percolateSource = new PercolateSourceBuilder(); percolateSource.setDoc(docBuilder().setDoc(new HashMap())); assertThrows( client() .preparePercolate() .setIndices("no_index") .setDocumentType("type1") .setSource(percolateSource), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE); assertThrows( client().admin().indices().prepareAnalyze("test", "this is a test"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE); assertThrows( client().admin().indices().prepareAnalyze("no_index", "this is a test"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE); assertThrows( client().prepareCount("test"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE); assertThrows( client().prepareCount("no_index"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE); checkWriteAction( false, timeout, client() .prepareUpdate("test", "type1", "1") .setScript("test script", ScriptService.ScriptType.INLINE) .setTimeout(timeout)); checkWriteAction( autoCreateIndex, timeout, client() .prepareUpdate("no_index", "type1", "1") .setScript("test script", ScriptService.ScriptType.INLINE) .setTimeout(timeout)); checkWriteAction( false, timeout, client() .prepareIndex("test", "type1", "1") .setSource(XContentFactory.jsonBuilder().startObject().endObject()) .setTimeout(timeout)); checkWriteAction( autoCreateIndex, timeout, client() .prepareIndex("no_index", "type1", "1") .setSource(XContentFactory.jsonBuilder().startObject().endObject()) .setTimeout(timeout)); BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); bulkRequestBuilder.add( client() .prepareIndex("test", "type1", "1") .setSource(XContentFactory.jsonBuilder().startObject().endObject())); bulkRequestBuilder.add( client() .prepareIndex("test", "type1", "2") .setSource(XContentFactory.jsonBuilder().startObject().endObject())); checkBulkAction(false, bulkRequestBuilder); bulkRequestBuilder = client().prepareBulk(); bulkRequestBuilder.add( client() .prepareIndex("no_index", "type1", "1") .setSource(XContentFactory.jsonBuilder().startObject().endObject())); bulkRequestBuilder.add( client() .prepareIndex("no_index", "type1", "2") .setSource(XContentFactory.jsonBuilder().startObject().endObject())); checkBulkAction(autoCreateIndex, bulkRequestBuilder); internalCluster().startNode(settings); client() .admin() .cluster() .prepareHealth() .setWaitForGreenStatus() .setWaitForNodes("2") .execute() .actionGet(); }