示例#1
0
  private void ensureActiveShardCopies(final int shardId, final int copyCount) throws Exception {
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            ClusterState state = client().admin().cluster().prepareState().get().getState();
            assertThat(state.routingTable().index("idx"), not(nullValue()));
            assertThat(state.routingTable().index("idx").shard(shardId), not(nullValue()));
            assertThat(
                state.routingTable().index("idx").shard(shardId).activeShards().size(),
                equalTo(copyCount));

            ClusterHealthResponse healthResponse =
                client().admin().cluster().prepareHealth("idx").setWaitForRelocatingShards(0).get();
            assertThat(healthResponse.isTimedOut(), equalTo(false));

            RecoveryResponse recoveryResponse =
                client().admin().indices().prepareRecoveries("idx").setActiveOnly(true).get();
            assertThat(recoveryResponse.shardResponses().get("idx").size(), equalTo(0));
          }
        });
  }
  public void testDelayedMappingPropagationOnReplica() throws Exception {
    // This is essentially the same thing as testDelayedMappingPropagationOnPrimary
    // but for replicas
    // Here we want to test that everything goes well if the mappings that
    // are needed for a document are not available on the replica at the
    // time of indexing it
    final List<String> nodeNames = internalCluster().startNodesAsync(2).get();
    assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());

    final String master = internalCluster().getMasterName();
    assertThat(nodeNames, hasItem(master));
    String otherNode = null;
    for (String node : nodeNames) {
      if (node.equals(master) == false) {
        otherNode = node;
        break;
      }
    }
    assertNotNull(otherNode);

    // Force allocation of the primary on the master node by first only allocating on the master
    // and then allowing all nodes so that the replica gets allocated on the other node
    assertAcked(
        prepareCreate("index")
            .setSettings(
                Settings.builder()
                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
                    .put("index.routing.allocation.include._name", master))
            .get());
    assertAcked(
        client()
            .admin()
            .indices()
            .prepareUpdateSettings("index")
            .setSettings(Settings.builder().put("index.routing.allocation.include._name", ""))
            .get());
    ensureGreen();

    // Check routing tables
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    assertEquals(master, state.nodes().masterNode().name());
    List<ShardRouting> shards = state.routingTable().allShards("index");
    assertThat(shards, hasSize(2));
    for (ShardRouting shard : shards) {
      if (shard.primary()) {
        // primary must be on the master
        assertEquals(state.nodes().masterNodeId(), shard.currentNodeId());
      } else {
        assertTrue(shard.active());
      }
    }

    // Block cluster state processing on the replica
    BlockClusterStateProcessing disruption =
        new BlockClusterStateProcessing(otherNode, getRandom());
    internalCluster().setDisruptionScheme(disruption);
    disruption.startDisrupting();
    final AtomicReference<Object> putMappingResponse = new AtomicReference<>();
    client()
        .admin()
        .indices()
        .preparePutMapping("index")
        .setType("type")
        .setSource("field", "type=long")
        .execute(
            new ActionListener<PutMappingResponse>() {
              @Override
              public void onResponse(PutMappingResponse response) {
                putMappingResponse.set(response);
              }

              @Override
              public void onFailure(Throwable e) {
                putMappingResponse.set(e);
              }
            });
    // Wait for mappings to be available on master
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            final IndicesService indicesService =
                internalCluster().getInstance(IndicesService.class, master);
            final IndexService indexService = indicesService.indexServiceSafe("index");
            assertNotNull(indexService);
            final MapperService mapperService = indexService.mapperService();
            DocumentMapper mapper = mapperService.documentMapper("type");
            assertNotNull(mapper);
            assertNotNull(mapper.mappers().getMapper("field"));
          }
        });

    final AtomicReference<Object> docIndexResponse = new AtomicReference<>();
    client()
        .prepareIndex("index", "type", "1")
        .setSource("field", 42)
        .execute(
            new ActionListener<IndexResponse>() {
              @Override
              public void onResponse(IndexResponse response) {
                docIndexResponse.set(response);
              }

              @Override
              public void onFailure(Throwable e) {
                docIndexResponse.set(e);
              }
            });

    // Wait for document to be indexed on primary
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            assertTrue(
                client()
                    .prepareGet("index", "type", "1")
                    .setPreference("_primary")
                    .get()
                    .isExists());
          }
        });

    // The mappings have not been propagated to the replica yet as a consequence the document count
    // not be indexed
    // We wait on purpose to make sure that the document is not indexed because the shard operation
    // is stalled
    // and not just because it takes time to replicate the indexing request to the replica
    Thread.sleep(100);
    assertThat(putMappingResponse.get(), equalTo(null));
    assertThat(docIndexResponse.get(), equalTo(null));

    // Now make sure the indexing request finishes successfully
    disruption.stopDisrupting();
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            assertThat(putMappingResponse.get(), instanceOf(PutMappingResponse.class));
            PutMappingResponse resp = (PutMappingResponse) putMappingResponse.get();
            assertTrue(resp.isAcknowledged());
            assertThat(docIndexResponse.get(), instanceOf(IndexResponse.class));
            IndexResponse docResp = (IndexResponse) docIndexResponse.get();
            assertEquals(
                Arrays.toString(docResp.getShardInfo().getFailures()),
                2,
                docResp.getShardInfo().getTotal()); // both shards should have succeeded
          }
        });
  }
  public void testDelayedMappingPropagationOnPrimary() throws Exception {
    // Here we want to test that things go well if there is a first request
    // that adds mappings but before mappings are propagated to all nodes
    // another index request introduces the same mapping. The master node
    // will reply immediately since it did not change the cluster state
    // but the change might not be on the node that performed the indexing
    // operation yet

    Settings settings = Settings.builder().put(DiscoverySettings.PUBLISH_TIMEOUT, "0ms").build();
    final List<String> nodeNames = internalCluster().startNodesAsync(2, settings).get();
    assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());

    final String master = internalCluster().getMasterName();
    assertThat(nodeNames, hasItem(master));
    String otherNode = null;
    for (String node : nodeNames) {
      if (node.equals(master) == false) {
        otherNode = node;
        break;
      }
    }
    assertNotNull(otherNode);

    // Don't allocate the shard on the master node
    assertAcked(
        prepareCreate("index")
            .setSettings(
                Settings.builder()
                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                    .put("index.routing.allocation.exclude._name", master))
            .get());
    ensureGreen();

    // Check routing tables
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    assertEquals(master, state.nodes().masterNode().name());
    List<ShardRouting> shards = state.routingTable().allShards("index");
    assertThat(shards, hasSize(1));
    for (ShardRouting shard : shards) {
      if (shard.primary()) {
        // primary must not be on the master node
        assertFalse(state.nodes().masterNodeId().equals(shard.currentNodeId()));
      } else {
        fail(); // only primaries
      }
    }

    // Block cluster state processing where our shard is
    BlockClusterStateProcessing disruption =
        new BlockClusterStateProcessing(otherNode, getRandom());
    internalCluster().setDisruptionScheme(disruption);
    disruption.startDisrupting();

    // Add a new mapping...
    final AtomicReference<Object> putMappingResponse = new AtomicReference<>();
    client()
        .admin()
        .indices()
        .preparePutMapping("index")
        .setType("type")
        .setSource("field", "type=long")
        .execute(
            new ActionListener<PutMappingResponse>() {
              @Override
              public void onResponse(PutMappingResponse response) {
                putMappingResponse.set(response);
              }

              @Override
              public void onFailure(Throwable e) {
                putMappingResponse.set(e);
              }
            });
    // ...and wait for mappings to be available on master
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            ImmutableOpenMap<String, MappingMetaData> indexMappings =
                client()
                    .admin()
                    .indices()
                    .prepareGetMappings("index")
                    .get()
                    .getMappings()
                    .get("index");
            assertNotNull(indexMappings);
            MappingMetaData typeMappings = indexMappings.get("type");
            assertNotNull(typeMappings);
            Object properties;
            try {
              properties = typeMappings.getSourceAsMap().get("properties");
            } catch (IOException e) {
              throw new AssertionError(e);
            }
            assertNotNull(properties);
            Object fieldMapping = ((Map<String, Object>) properties).get("field");
            assertNotNull(fieldMapping);
          }
        });

    final AtomicReference<Object> docIndexResponse = new AtomicReference<>();
    client()
        .prepareIndex("index", "type", "1")
        .setSource("field", 42)
        .execute(
            new ActionListener<IndexResponse>() {
              @Override
              public void onResponse(IndexResponse response) {
                docIndexResponse.set(response);
              }

              @Override
              public void onFailure(Throwable e) {
                docIndexResponse.set(e);
              }
            });

    // Wait a bit to make sure that the reason why we did not get a response
    // is that cluster state processing is blocked and not just that it takes
    // time to process the indexing request
    Thread.sleep(100);
    assertThat(putMappingResponse.get(), equalTo(null));
    assertThat(docIndexResponse.get(), equalTo(null));

    // Now make sure the indexing request finishes successfully
    disruption.stopDisrupting();
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            assertThat(putMappingResponse.get(), instanceOf(PutMappingResponse.class));
            PutMappingResponse resp = (PutMappingResponse) putMappingResponse.get();
            assertTrue(resp.isAcknowledged());
            assertThat(docIndexResponse.get(), instanceOf(IndexResponse.class));
            IndexResponse docResp = (IndexResponse) docIndexResponse.get();
            assertEquals(
                Arrays.toString(docResp.getShardInfo().getFailures()),
                1,
                docResp.getShardInfo().getTotal());
          }
        });
  }
  @Test
  public void testCancellationCleansTempFiles() throws Exception {
    final String indexName = "test";

    final String p_node = internalCluster().startNode();

    client()
        .admin()
        .indices()
        .prepareCreate(indexName)
        .setSettings(
            Settings.builder()
                .put(
                    IndexMetaData.SETTING_NUMBER_OF_SHARDS,
                    1,
                    IndexMetaData.SETTING_NUMBER_OF_REPLICAS,
                    0))
        .get();

    internalCluster().startNodesAsync(2).get();

    List<IndexRequestBuilder> requests = new ArrayList<>();
    int numDocs = scaledRandomIntBetween(25, 250);
    for (int i = 0; i < numDocs; i++) {
      requests.add(client().prepareIndex(indexName, "type").setCreate(true).setSource("{}"));
    }
    indexRandom(true, requests);
    assertFalse(
        client()
            .admin()
            .cluster()
            .prepareHealth()
            .setWaitForNodes("3")
            .setWaitForGreenStatus()
            .get()
            .isTimedOut());
    flush();

    int allowedFailures = randomIntBetween(3, 10);
    logger.info("--> blocking recoveries from primary (allowed failures: [{}])", allowedFailures);
    CountDownLatch corruptionCount = new CountDownLatch(allowedFailures);
    ClusterService clusterService = internalCluster().getInstance(ClusterService.class, p_node);
    MockTransportService mockTransportService =
        (MockTransportService) internalCluster().getInstance(TransportService.class, p_node);
    for (DiscoveryNode node : clusterService.state().nodes()) {
      if (!node.equals(clusterService.localNode())) {
        mockTransportService.addDelegate(
            node, new RecoveryCorruption(mockTransportService.original(), corruptionCount));
      }
    }

    client()
        .admin()
        .indices()
        .prepareUpdateSettings(indexName)
        .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1))
        .get();

    corruptionCount.await();

    logger.info("--> stopping replica assignment");
    assertAcked(
        client()
            .admin()
            .cluster()
            .prepareUpdateSettings()
            .setTransientSettings(
                Settings.builder()
                    .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "none")));

    logger.info("--> wait for all replica shards to be removed, on all nodes");
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            for (String node : internalCluster().getNodeNames()) {
              if (node.equals(p_node)) {
                continue;
              }
              ClusterState state =
                  client(node).admin().cluster().prepareState().setLocal(true).get().getState();
              assertThat(
                  node + " indicates assigned replicas",
                  state
                      .getRoutingTable()
                      .index(indexName)
                      .shardsWithState(ShardRoutingState.UNASSIGNED)
                      .size(),
                  equalTo(1));
            }
          }
        });

    logger.info("--> verifying no temporary recoveries are left");
    for (String node : internalCluster().getNodeNames()) {
      NodeEnvironment nodeEnvironment = internalCluster().getInstance(NodeEnvironment.class, node);
      for (final Path shardLoc : nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))) {
        if (Files.exists(shardLoc)) {
          assertBusy(
              new Runnable() {
                @Override
                public void run() {
                  try {
                    Files.walkFileTree(
                        shardLoc,
                        new SimpleFileVisitor<Path>() {
                          @Override
                          public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
                              throws IOException {
                            assertThat(
                                "found a temporary recovery file: " + file,
                                file.getFileName().toString(),
                                not(startsWith("recovery.")));
                            return FileVisitResult.CONTINUE;
                          }
                        });
                  } catch (IOException e) {
                    throw new AssertionError(
                        "failed to walk file tree starting at [" + shardLoc + "]", e);
                  }
                }
              });
        }
      }
    }
  }
  @Test
  public void testNoMasterActions() throws Exception {
    // note, sometimes, we want to check with the fact that an index gets created, sometimes not...
    boolean autoCreateIndex = randomBoolean();
    logger.info("auto_create_index set to {}", autoCreateIndex);

    Settings settings =
        settingsBuilder()
            .put("discovery.type", "zen")
            .put("action.auto_create_index", autoCreateIndex)
            .put("discovery.zen.minimum_master_nodes", 2)
            .put("discovery.zen.ping_timeout", "200ms")
            .put("discovery.initial_state_timeout", "500ms")
            .put(DiscoverySettings.NO_MASTER_BLOCK, "all")
            .build();

    TimeValue timeout = TimeValue.timeValueMillis(200);

    internalCluster().startNode(settings);
    // start a second node, create an index, and then shut it down so we have no master block
    internalCluster().startNode(settings);
    createIndex("test");
    client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();
    internalCluster().stopRandomDataNode();
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            ClusterState state =
                client()
                    .admin()
                    .cluster()
                    .prepareState()
                    .setLocal(true)
                    .execute()
                    .actionGet()
                    .getState();
            assertTrue(state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID));
          }
        });

    assertThrows(
        client().prepareGet("test", "type1", "1"),
        ClusterBlockException.class,
        RestStatus.SERVICE_UNAVAILABLE);

    assertThrows(
        client().prepareGet("no_index", "type1", "1"),
        ClusterBlockException.class,
        RestStatus.SERVICE_UNAVAILABLE);

    assertThrows(
        client().prepareMultiGet().add("test", "type1", "1"),
        ClusterBlockException.class,
        RestStatus.SERVICE_UNAVAILABLE);

    assertThrows(
        client().prepareMultiGet().add("no_index", "type1", "1"),
        ClusterBlockException.class,
        RestStatus.SERVICE_UNAVAILABLE);

    PercolateSourceBuilder percolateSource = new PercolateSourceBuilder();
    percolateSource.setDoc(docBuilder().setDoc(new HashMap()));
    assertThrows(
        client()
            .preparePercolate()
            .setIndices("test")
            .setDocumentType("type1")
            .setSource(percolateSource),
        ClusterBlockException.class,
        RestStatus.SERVICE_UNAVAILABLE);

    percolateSource = new PercolateSourceBuilder();
    percolateSource.setDoc(docBuilder().setDoc(new HashMap()));
    assertThrows(
        client()
            .preparePercolate()
            .setIndices("no_index")
            .setDocumentType("type1")
            .setSource(percolateSource),
        ClusterBlockException.class,
        RestStatus.SERVICE_UNAVAILABLE);

    assertThrows(
        client().admin().indices().prepareAnalyze("test", "this is a test"),
        ClusterBlockException.class,
        RestStatus.SERVICE_UNAVAILABLE);

    assertThrows(
        client().admin().indices().prepareAnalyze("no_index", "this is a test"),
        ClusterBlockException.class,
        RestStatus.SERVICE_UNAVAILABLE);

    assertThrows(
        client().prepareCount("test"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE);

    assertThrows(
        client().prepareCount("no_index"),
        ClusterBlockException.class,
        RestStatus.SERVICE_UNAVAILABLE);

    checkWriteAction(
        false,
        timeout,
        client()
            .prepareUpdate("test", "type1", "1")
            .setScript("test script", ScriptService.ScriptType.INLINE)
            .setTimeout(timeout));

    checkWriteAction(
        autoCreateIndex,
        timeout,
        client()
            .prepareUpdate("no_index", "type1", "1")
            .setScript("test script", ScriptService.ScriptType.INLINE)
            .setTimeout(timeout));

    checkWriteAction(
        false,
        timeout,
        client()
            .prepareIndex("test", "type1", "1")
            .setSource(XContentFactory.jsonBuilder().startObject().endObject())
            .setTimeout(timeout));

    checkWriteAction(
        autoCreateIndex,
        timeout,
        client()
            .prepareIndex("no_index", "type1", "1")
            .setSource(XContentFactory.jsonBuilder().startObject().endObject())
            .setTimeout(timeout));

    BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
    bulkRequestBuilder.add(
        client()
            .prepareIndex("test", "type1", "1")
            .setSource(XContentFactory.jsonBuilder().startObject().endObject()));
    bulkRequestBuilder.add(
        client()
            .prepareIndex("test", "type1", "2")
            .setSource(XContentFactory.jsonBuilder().startObject().endObject()));
    checkBulkAction(false, bulkRequestBuilder);

    bulkRequestBuilder = client().prepareBulk();
    bulkRequestBuilder.add(
        client()
            .prepareIndex("no_index", "type1", "1")
            .setSource(XContentFactory.jsonBuilder().startObject().endObject()));
    bulkRequestBuilder.add(
        client()
            .prepareIndex("no_index", "type1", "2")
            .setSource(XContentFactory.jsonBuilder().startObject().endObject()));
    checkBulkAction(autoCreateIndex, bulkRequestBuilder);

    internalCluster().startNode(settings);
    client()
        .admin()
        .cluster()
        .prepareHealth()
        .setWaitForGreenStatus()
        .setWaitForNodes("2")
        .execute()
        .actionGet();
  }