public void testShardActiveElsewhereDoesNotDeleteAnother() throws Exception {
    InternalTestCluster.Async<String> masterFuture =
        internalCluster()
            .startNodeAsync(
                Settings.builder()
                    .put(
                        Node.NODE_MASTER_SETTING.getKey(),
                        true,
                        Node.NODE_DATA_SETTING.getKey(),
                        false)
                    .build());
    InternalTestCluster.Async<List<String>> nodesFutures =
        internalCluster()
            .startNodesAsync(
                4,
                Settings.builder()
                    .put(
                        Node.NODE_MASTER_SETTING.getKey(),
                        false,
                        Node.NODE_DATA_SETTING.getKey(),
                        true)
                    .build());

    final String masterNode = masterFuture.get();
    final String node1 = nodesFutures.get().get(0);
    final String node2 = nodesFutures.get().get(1);
    final String node3 = nodesFutures.get().get(2);
    // we will use this later on, handy to start now to make sure it has a different data folder
    // that node 1,2 &3
    final String node4 = nodesFutures.get().get(3);

    assertAcked(
        prepareCreate("test")
            .setSettings(
                Settings.builder()
                    .put(indexSettings())
                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 3)
                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
                    .put(
                        IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + "_name",
                        node4)));
    assertFalse(
        client()
            .admin()
            .cluster()
            .prepareHealth()
            .setWaitForNoRelocatingShards(true)
            .setWaitForGreenStatus()
            .setWaitForNodes("5")
            .get()
            .isTimedOut());

    // disable allocation to control the situation more easily
    assertAcked(
        client()
            .admin()
            .cluster()
            .prepareUpdateSettings()
            .setTransientSettings(
                Settings.builder()
                    .put(
                        EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(),
                        "none")));

    logger.debug("--> shutting down two random nodes");
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node1, node2, node3));
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node1, node2, node3));

    logger.debug("--> verifying index is red");
    ClusterHealthResponse health =
        client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
    if (health.getStatus() != ClusterHealthStatus.RED) {
      logClusterState();
      fail("cluster didn't become red, despite of shutting 2 of 3 nodes");
    }

    logger.debug("--> allowing index to be assigned to node [{}]", node4);
    assertAcked(
        client()
            .admin()
            .indices()
            .prepareUpdateSettings("test")
            .setSettings(
                Settings.builder()
                    .put(
                        IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey() + "_name",
                        "NONE")));

    assertAcked(
        client()
            .admin()
            .cluster()
            .prepareUpdateSettings()
            .setTransientSettings(
                Settings.builder()
                    .put(
                        EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(),
                        "all")));

    logger.debug("--> waiting for shards to recover on [{}]", node4);
    // we have to do this in two steps as we now do async shard fetching before assigning, so the
    // change to the
    // allocation filtering may not have immediate effect
    // TODO: we should add an easier to do this. It's too much of a song and dance..
    Index index = resolveIndex("test");
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            assertTrue(internalCluster().getInstance(IndicesService.class, node4).hasIndex(index));
          }
        });

    // wait for 4 active shards - we should have lost one shard
    assertFalse(
        client().admin().cluster().prepareHealth().setWaitForActiveShards(4).get().isTimedOut());

    // disable allocation again to control concurrency a bit and allow shard active to kick in
    // before allocation
    assertAcked(
        client()
            .admin()
            .cluster()
            .prepareUpdateSettings()
            .setTransientSettings(
                Settings.builder()
                    .put(
                        EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(),
                        "none")));

    logger.debug("--> starting the two old nodes back");

    internalCluster()
        .startNodesAsync(
            2,
            Settings.builder()
                .put(
                    Node.NODE_MASTER_SETTING.getKey(), false, Node.NODE_DATA_SETTING.getKey(), true)
                .build());

    assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("5").get().isTimedOut());

    assertAcked(
        client()
            .admin()
            .cluster()
            .prepareUpdateSettings()
            .setTransientSettings(
                Settings.builder()
                    .put(
                        EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(),
                        "all")));

    logger.debug("--> waiting for the lost shard to be recovered");

    ensureGreen("test");
  }
Exemple #2
0
  void setupCluster() throws Exception {
    InternalTestCluster.Async<List<String>> replicas =
        internalCluster().startNodesAsync(1); // for replicas

    Path baseTempDir = createTempDir();
    // start single data path node
    Settings.Builder nodeSettings =
        Settings.builder()
            .put(
                Environment.PATH_DATA_SETTING.getKey(),
                baseTempDir.resolve("single-path").toAbsolutePath())
            .put(
                Node.NODE_MASTER_SETTING.getKey(),
                false); // workaround for dangling index loading issue when node is master
    InternalTestCluster.Async<String> singleDataPathNode =
        internalCluster().startNodeAsync(nodeSettings.build());

    // start multi data path node
    nodeSettings =
        Settings.builder()
            .put(
                Environment.PATH_DATA_SETTING.getKey(),
                baseTempDir.resolve("multi-path1").toAbsolutePath()
                    + ","
                    + baseTempDir.resolve("multi-path2").toAbsolutePath())
            .put(
                Node.NODE_MASTER_SETTING.getKey(),
                false); // workaround for dangling index loading issue when node is master
    InternalTestCluster.Async<String> multiDataPathNode =
        internalCluster().startNodeAsync(nodeSettings.build());

    // find single data path dir
    singleDataPathNodeName = singleDataPathNode.get();
    Path[] nodePaths =
        internalCluster()
            .getInstance(NodeEnvironment.class, singleDataPathNodeName)
            .nodeDataPaths();
    assertEquals(1, nodePaths.length);
    singleDataPath = nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER);
    assertFalse(Files.exists(singleDataPath));
    Files.createDirectories(singleDataPath);
    logger.info("--> Single data path: {}", singleDataPath);

    // find multi data path dirs
    multiDataPathNodeName = multiDataPathNode.get();
    nodePaths =
        internalCluster().getInstance(NodeEnvironment.class, multiDataPathNodeName).nodeDataPaths();
    assertEquals(2, nodePaths.length);
    multiDataPath =
        new Path[] {
          nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER),
          nodePaths[1].resolve(NodeEnvironment.INDICES_FOLDER)
        };
    assertFalse(Files.exists(multiDataPath[0]));
    assertFalse(Files.exists(multiDataPath[1]));
    Files.createDirectories(multiDataPath[0]);
    Files.createDirectories(multiDataPath[1]);
    logger.info("--> Multi data paths: {}, {}", multiDataPath[0], multiDataPath[1]);

    replicas.get(); // wait for replicas
  }