@Test public void simpleFlagTests() { AllocationService allocation = createAllocationService( settingsBuilder().put("cluster.routing.allocation.concurrent_recoveries", 10).build()); logger.info("creating an index with 1 shard, no replica"); MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").numberOfShards(1).numberOfReplicas(0)) .build(); RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build(); ClusterState clusterState = ClusterState.builder().metaData(metaData).routingTable(routingTable).build(); assertThat( clusterState.routingTable().index("test").shard(0).primaryAllocatedPostApi(), equalTo(false)); logger.info("adding two nodes and performing rerouting"); clusterState = ClusterState.builder(clusterState) .nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))) .build(); RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); assertThat( clusterState.routingTable().index("test").shard(0).primaryAllocatedPostApi(), equalTo(false)); logger.info("start primary shard"); rerouteResult = allocation.applyStartedShards( clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); assertThat( clusterState.routingTable().index("test").shard(0).primaryAllocatedPostApi(), equalTo(true)); }
/** * Tests that higher prioritized primaries and replicas are allocated first even on the balanced * shard allocator See https://github.com/elastic/elasticsearch/issues/13249 for details */ public void testPrioritizedIndicesAllocatedFirst() { AllocationService allocation = createAllocationService( Settings.builder() .put( ThrottlingAllocationDecider .CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getKey(), 1) .put( ThrottlingAllocationDecider .CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING .getKey(), 10) .put( ThrottlingAllocationDecider .CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING .getKey(), 1) .put( ThrottlingAllocationDecider .CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING .getKey(), 1) .build()); final String highPriorityName; final String lowPriorityName; final int priorityFirst; final int prioritySecond; if (randomBoolean()) { highPriorityName = "first"; lowPriorityName = "second"; prioritySecond = 1; priorityFirst = 100; } else { lowPriorityName = "first"; highPriorityName = "second"; prioritySecond = 100; priorityFirst = 1; } MetaData metaData = MetaData.builder() .put( IndexMetaData.builder("first") .settings( settings(Version.CURRENT) .put(IndexMetaData.SETTING_PRIORITY, priorityFirst)) .numberOfShards(2) .numberOfReplicas(1)) .put( IndexMetaData.builder("second") .settings( settings(Version.CURRENT) .put(IndexMetaData.SETTING_PRIORITY, prioritySecond)) .numberOfShards(2) .numberOfReplicas(1)) .build(); RoutingTable routingTable = RoutingTable.builder() .addAsNew(metaData.index("first")) .addAsNew(metaData.index("second")) .build(); ClusterState clusterState = ClusterState.builder( org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault( Settings.EMPTY)) .metaData(metaData) .routingTable(routingTable) .build(); clusterState = ClusterState.builder(clusterState) .nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))) .build(); RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute"); clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); routingTable = allocation.reroute(clusterState, "reroute").routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertEquals(2, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size()); assertEquals( highPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(0).getIndexName()); assertEquals( highPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(1).getIndexName()); routingTable = allocation .applyStartedShards( clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)) .routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertEquals(2, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size()); assertEquals( lowPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(0).getIndexName()); assertEquals( lowPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(1).getIndexName()); routingTable = allocation .applyStartedShards( clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)) .routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertEquals( clusterState.getRoutingNodes().shardsWithState(INITIALIZING).toString(), 2, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size()); assertEquals( highPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(0).getIndexName()); assertEquals( highPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(1).getIndexName()); routingTable = allocation .applyStartedShards( clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)) .routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertEquals(2, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size()); assertEquals( lowPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(0).getIndexName()); assertEquals( lowPriorityName, clusterState.getRoutingNodes().shardsWithState(INITIALIZING).get(1).getIndexName()); }
@Test public void tesStartedShardsMatching() { AllocationService allocation = createAllocationService(); logger.info("--> building initial cluster state"); final IndexMetaData indexMetaData = IndexMetaData.builder("test").numberOfShards(3).numberOfReplicas(0).build(); ClusterState.Builder stateBuilder = ClusterState.builder(ClusterName.DEFAULT) .nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))) .metaData(MetaData.builder().put(indexMetaData, false)); final ImmutableShardRouting initShard = new ImmutableShardRouting( "test", 0, "node1", randomBoolean(), ShardRoutingState.INITIALIZING, 1); final ImmutableShardRouting startedShard = new ImmutableShardRouting( "test", 1, "node2", randomBoolean(), ShardRoutingState.STARTED, 1); final ImmutableShardRouting relocatingShard = new ImmutableShardRouting( "test", 2, "node1", "node2", randomBoolean(), ShardRoutingState.RELOCATING, 1); stateBuilder.routingTable( RoutingTable.builder() .add( IndexRoutingTable.builder("test") .addIndexShard( new IndexShardRoutingTable.Builder(initShard.shardId(), true) .addShard(initShard) .build()) .addIndexShard( new IndexShardRoutingTable.Builder(startedShard.shardId(), true) .addShard(startedShard) .build()) .addIndexShard( new IndexShardRoutingTable.Builder(relocatingShard.shardId(), true) .addShard(relocatingShard) .build()))); ClusterState state = stateBuilder.build(); logger.info("--> test starting of shard"); RoutingAllocation.Result result = allocation.applyStartedShards( state, Arrays.asList( new ImmutableShardRouting( initShard.index(), initShard.id(), initShard.currentNodeId(), initShard.relocatingNodeId(), initShard.primary(), ShardRoutingState.INITIALIZING, randomInt())), false); assertTrue( "failed to start " + initShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); assertTrue( initShard + "isn't started \ncurrent routing table:" + result.routingTable().prettyPrint(), result .routingTable() .index("test") .shard(initShard.id()) .countWithState(ShardRoutingState.STARTED) == 1); logger.info("--> testing shard variants that shouldn't match the started shard"); result = allocation.applyStartedShards( state, Arrays.asList( new ImmutableShardRouting( initShard.index(), initShard.id(), initShard.currentNodeId(), initShard.relocatingNodeId(), !initShard.primary(), ShardRoutingState.INITIALIZING, 1)), false); assertFalse( "wrong primary flag shouldn't start shard " + initShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); result = allocation.applyStartedShards( state, Arrays.asList( new ImmutableShardRouting( initShard.index(), initShard.id(), "some_node", initShard.currentNodeId(), initShard.primary(), ShardRoutingState.INITIALIZING, 1)), false); assertFalse( "relocating shard from node shouldn't start shard " + initShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); result = allocation.applyStartedShards( state, Arrays.asList( new ImmutableShardRouting( initShard.index(), initShard.id(), initShard.currentNodeId(), "some_node", initShard.primary(), ShardRoutingState.INITIALIZING, 1)), false); assertFalse( "relocating shard to node shouldn't start shard " + initShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); logger.info("--> testing double starting"); result = allocation.applyStartedShards( state, Arrays.asList( new ImmutableShardRouting( startedShard.index(), startedShard.id(), startedShard.currentNodeId(), startedShard.relocatingNodeId(), startedShard.primary(), ShardRoutingState.INITIALIZING, 1)), false); assertFalse( "duplicate starting of the same shard should be ignored \ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); logger.info("--> testing starting of relocating shards"); result = allocation.applyStartedShards( state, Arrays.asList( new ImmutableShardRouting( relocatingShard.index(), relocatingShard.id(), relocatingShard.relocatingNodeId(), relocatingShard.currentNodeId(), relocatingShard.primary(), ShardRoutingState.INITIALIZING, randomInt())), false); assertTrue( "failed to start " + relocatingShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); ShardRouting shardRouting = result.routingTable().index("test").shard(relocatingShard.id()).getShards().get(0); assertThat(shardRouting.state(), equalTo(ShardRoutingState.STARTED)); assertThat(shardRouting.currentNodeId(), equalTo("node2")); assertThat(shardRouting.relocatingNodeId(), nullValue()); logger.info("--> testing shard variants that shouldn't match the relocating shard"); result = allocation.applyStartedShards( state, Arrays.asList( new ImmutableShardRouting( relocatingShard.index(), relocatingShard.id(), relocatingShard.relocatingNodeId(), relocatingShard.currentNodeId(), !relocatingShard.primary(), ShardRoutingState.INITIALIZING, 1)), false); assertFalse( "wrong primary flag shouldn't start shard " + relocatingShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); result = allocation.applyStartedShards( state, Arrays.asList( new ImmutableShardRouting( relocatingShard.index(), relocatingShard.id(), "some_node", relocatingShard.currentNodeId(), relocatingShard.primary(), ShardRoutingState.INITIALIZING, 1)), false); assertFalse( "relocating shard to a different node shouldn't start shard " + relocatingShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); result = allocation.applyStartedShards( state, Arrays.asList( new ImmutableShardRouting( relocatingShard.index(), relocatingShard.id(), relocatingShard.relocatingNodeId(), "some_node", relocatingShard.primary(), ShardRoutingState.INITIALIZING, 1)), false); assertFalse( "relocating shard from a different node shouldn't start shard " + relocatingShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); result = allocation.applyStartedShards( state, Arrays.asList( new ImmutableShardRouting( relocatingShard.index(), relocatingShard.id(), relocatingShard.relocatingNodeId(), relocatingShard.primary(), ShardRoutingState.INITIALIZING, 1)), false); assertFalse( "non-relocating shard shouldn't start shard" + relocatingShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); }
public void testSingleRetryOnIgnore() { ClusterState clusterState = createInitialClusterState(); RoutingTable routingTable = clusterState.routingTable(); final int retries = MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY.get(Settings.EMPTY); // now fail it N-1 times for (int i = 0; i < retries - 1; i++) { List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList( new FailedRerouteAllocation.FailedShard( routingTable.index("idx").shard(0).shards().get(0), "boom" + i, new UnsupportedOperationException())); RoutingAllocation.Result result = strategy.applyFailedShards(clusterState, failedShards); assertTrue(result.changed()); routingTable = result.routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertEquals(routingTable.index("idx").shards().size(), 1); assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING); assertEquals( routingTable .index("idx") .shard(0) .shards() .get(0) .unassignedInfo() .getNumFailedAllocations(), i + 1); assertEquals( routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getMessage(), "boom" + i); } // now we go and check that we are actually stick to unassigned on the next failure List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList( new FailedRerouteAllocation.FailedShard( routingTable.index("idx").shard(0).shards().get(0), "boom", new UnsupportedOperationException())); RoutingAllocation.Result result = strategy.applyFailedShards(clusterState, failedShards); assertTrue(result.changed()); routingTable = result.routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertEquals(routingTable.index("idx").shards().size(), 1); assertEquals( routingTable .index("idx") .shard(0) .shards() .get(0) .unassignedInfo() .getNumFailedAllocations(), retries); assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED); assertEquals( routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getMessage(), "boom"); result = strategy.reroute( clusterState, new AllocationCommands(), false, true); // manual reroute should retry once assertTrue(result.changed()); routingTable = result.routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertEquals(routingTable.index("idx").shards().size(), 1); assertEquals( routingTable .index("idx") .shard(0) .shards() .get(0) .unassignedInfo() .getNumFailedAllocations(), retries); assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING); assertEquals( routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getMessage(), "boom"); // now we go and check that we are actually stick to unassigned on the next failure ie. no retry failedShards = Collections.singletonList( new FailedRerouteAllocation.FailedShard( routingTable.index("idx").shard(0).shards().get(0), "boom", new UnsupportedOperationException())); result = strategy.applyFailedShards(clusterState, failedShards); assertTrue(result.changed()); routingTable = result.routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertEquals(routingTable.index("idx").shards().size(), 1); assertEquals( routingTable .index("idx") .shard(0) .shards() .get(0) .unassignedInfo() .getNumFailedAllocations(), retries + 1); assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED); assertEquals( routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getMessage(), "boom"); }
public void testFailedAllocation() { ClusterState clusterState = createInitialClusterState(); RoutingTable routingTable = clusterState.routingTable(); final int retries = MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY.get(Settings.EMPTY); // now fail it N-1 times for (int i = 0; i < retries - 1; i++) { List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList( new FailedRerouteAllocation.FailedShard( routingTable.index("idx").shard(0).shards().get(0), "boom" + i, new UnsupportedOperationException())); RoutingAllocation.Result result = strategy.applyFailedShards(clusterState, failedShards); assertTrue(result.changed()); routingTable = result.routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertEquals(routingTable.index("idx").shards().size(), 1); ShardRouting unassignedPrimary = routingTable.index("idx").shard(0).shards().get(0); assertEquals(unassignedPrimary.state(), INITIALIZING); assertEquals(unassignedPrimary.unassignedInfo().getNumFailedAllocations(), i + 1); assertEquals(unassignedPrimary.unassignedInfo().getMessage(), "boom" + i); // MaxRetryAllocationDecider#canForceAllocatePrimary should return YES decisions because // canAllocate returns YES here assertEquals( Decision.YES, new MaxRetryAllocationDecider(Settings.EMPTY) .canForceAllocatePrimary( unassignedPrimary, null, new RoutingAllocation(null, null, clusterState, null, 0, false))); } // now we go and check that we are actually stick to unassigned on the next failure { List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList( new FailedRerouteAllocation.FailedShard( routingTable.index("idx").shard(0).shards().get(0), "boom", new UnsupportedOperationException())); RoutingAllocation.Result result = strategy.applyFailedShards(clusterState, failedShards); assertTrue(result.changed()); routingTable = result.routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertEquals(routingTable.index("idx").shards().size(), 1); ShardRouting unassignedPrimary = routingTable.index("idx").shard(0).shards().get(0); assertEquals(unassignedPrimary.unassignedInfo().getNumFailedAllocations(), retries); assertEquals(unassignedPrimary.state(), UNASSIGNED); assertEquals(unassignedPrimary.unassignedInfo().getMessage(), "boom"); // MaxRetryAllocationDecider#canForceAllocatePrimary should return a NO decision because // canAllocate returns NO here assertEquals( Decision.NO, new MaxRetryAllocationDecider(Settings.EMPTY) .canForceAllocatePrimary( unassignedPrimary, null, new RoutingAllocation(null, null, clusterState, null, 0, false))); } // change the settings and ensure we can do another round of allocation for that index. clusterState = ClusterState.builder(clusterState) .routingTable(routingTable) .metaData( MetaData.builder(clusterState.metaData()) .put( IndexMetaData.builder(clusterState.metaData().index("idx")) .settings( Settings.builder() .put(clusterState.metaData().index("idx").getSettings()) .put("index.allocation.max_retries", retries + 1) .build()) .build(), true) .build()) .build(); RoutingAllocation.Result result = strategy.reroute(clusterState, "settings changed", false); assertTrue(result.changed()); routingTable = result.routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); // good we are initializing and we are maintaining failure information assertEquals(routingTable.index("idx").shards().size(), 1); ShardRouting unassignedPrimary = routingTable.index("idx").shard(0).shards().get(0); assertEquals(unassignedPrimary.unassignedInfo().getNumFailedAllocations(), retries); assertEquals(unassignedPrimary.state(), INITIALIZING); assertEquals(unassignedPrimary.unassignedInfo().getMessage(), "boom"); // bumped up the max retry count, so canForceAllocatePrimary should return a YES decision assertEquals( Decision.YES, new MaxRetryAllocationDecider(Settings.EMPTY) .canForceAllocatePrimary( routingTable.index("idx").shard(0).shards().get(0), null, new RoutingAllocation(null, null, clusterState, null, 0, false))); // now we start the shard routingTable = strategy .applyStartedShards( clusterState, Collections.singletonList(routingTable.index("idx").shard(0).shards().get(0))) .routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); // all counters have been reset to 0 ie. no unassigned info assertEquals(routingTable.index("idx").shards().size(), 1); assertNull(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo()); assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), STARTED); // now fail again and see if it has a new counter List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList( new FailedRerouteAllocation.FailedShard( routingTable.index("idx").shard(0).shards().get(0), "ZOOOMG", new UnsupportedOperationException())); result = strategy.applyFailedShards(clusterState, failedShards); assertTrue(result.changed()); routingTable = result.routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); assertEquals(routingTable.index("idx").shards().size(), 1); unassignedPrimary = routingTable.index("idx").shard(0).shards().get(0); assertEquals(unassignedPrimary.unassignedInfo().getNumFailedAllocations(), 1); assertEquals(unassignedPrimary.state(), INITIALIZING); assertEquals(unassignedPrimary.unassignedInfo().getMessage(), "ZOOOMG"); // Counter reset, so MaxRetryAllocationDecider#canForceAllocatePrimary should return a YES // decision assertEquals( Decision.YES, new MaxRetryAllocationDecider(Settings.EMPTY) .canForceAllocatePrimary( unassignedPrimary, null, new RoutingAllocation(null, null, clusterState, null, 0, false))); }