/**
* Returns the shards to purge, i.e. the local started primary shards that have ttl enabled and
* disable_purge to false
*/
private List<IndexShard> getShardsToPurge() {
List<IndexShard> shardsToPurge = new ArrayList<>();
MetaData metaData = clusterService.state().metaData();
for (IndexService indexService : indicesService) {
// check the value of disable_purge for this index
IndexMetaData indexMetaData = metaData.index(indexService.index());
if (indexMetaData == null) {
continue;
}
if (indexService.getIndexSettings().isTTLPurgeDisabled()) {
continue;
}
// check if ttl is enabled for at least one type of this index
boolean hasTTLEnabled = false;
for (String type : indexService.mapperService().types()) {
DocumentMapper documentType = indexService.mapperService().documentMapper(type);
if (documentType.TTLFieldMapper().enabled()) {
hasTTLEnabled = true;
break;
}
}
if (hasTTLEnabled) {
for (IndexShard indexShard : indexService) {
if (indexShard.state() == IndexShardState.STARTED
&& indexShard.routingEntry().primary()
&& indexShard.routingEntry().started()) {
shardsToPurge.add(indexShard);
}
}
}
}
return shardsToPurge;
}
private ClusterState createInitialClusterState(AllocationService service) {
MetaData.Builder metaBuilder = MetaData.builder();
metaBuilder.put(
IndexMetaData.builder("idx")
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(0));
MetaData metaData = metaBuilder.build();
RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
routingTableBuilder.addAsNew(metaData.index("idx"));
RoutingTable routingTable = routingTableBuilder.build();
ClusterState clusterState =
ClusterState.builder(
org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(
Settings.EMPTY))
.metaData(metaData)
.routingTable(routingTable)
.build();
clusterState =
ClusterState.builder(clusterState)
.nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")))
.build();
RoutingTable prevRoutingTable = routingTable;
routingTable = service.reroute(clusterState, "reroute").routingTable();
clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
assertEquals(prevRoutingTable.index("idx").shards().size(), 1);
assertEquals(prevRoutingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED);
assertEquals(routingTable.index("idx").shards().size(), 1);
assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
return clusterState;
}
@Test
public void testRoutingTableSerialization() throws Exception {
MetaData metaData =
MetaData.builder()
.put(IndexMetaData.builder("test").numberOfShards(10).numberOfReplicas(1))
.build();
RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
DiscoveryNodes nodes =
DiscoveryNodes.builder()
.put(newNode("node1"))
.put(newNode("node2"))
.put(newNode("node3"))
.build();
ClusterState clusterState =
ClusterState.builder().nodes(nodes).metaData(metaData).routingTable(routingTable).build();
AllocationService strategy = createAllocationService();
RoutingTable source = strategy.reroute(clusterState).routingTable();
BytesStreamOutput outStream = new BytesStreamOutput();
RoutingTable.Builder.writeTo(source, outStream);
BytesStreamInput inStream = new BytesStreamInput(outStream.bytes().toBytes(), false);
RoutingTable target = RoutingTable.Builder.readFrom(inStream);
assertThat(target.prettyPrint(), equalTo(source.prettyPrint()));
}
@Test
public void testClusterStateSerialization() throws Exception {
MetaData metaData =
MetaData.builder()
.put(IndexMetaData.builder("test").numberOfShards(10).numberOfReplicas(1))
.build();
RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
DiscoveryNodes nodes =
DiscoveryNodes.builder()
.put(newNode("node1"))
.put(newNode("node2"))
.put(newNode("node3"))
.localNodeId("node1")
.masterNodeId("node2")
.build();
ClusterState clusterState =
ClusterState.builder().nodes(nodes).metaData(metaData).routingTable(routingTable).build();
AllocationService strategy = createAllocationService();
clusterState =
ClusterState.builder(clusterState)
.routingTable(strategy.reroute(clusterState).routingTable())
.build();
ClusterState serializedClusterState =
ClusterState.Builder.fromBytes(
ClusterState.Builder.toBytes(clusterState), newNode("node1"));
assertThat(
serializedClusterState.routingTable().prettyPrint(),
equalTo(clusterState.routingTable().prettyPrint()));
}
@Test
public void testSingleIndexShardFailed() {
AllocationService strategy =
createAllocationService(
settingsBuilder().put("cluster.routing.allocation.concurrent_recoveries", 10).build());
logger.info("Building initial routing table");
MetaData metaData =
MetaData.builder()
.put(IndexMetaData.builder("test").numberOfShards(1).numberOfReplicas(0))
.build();
RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
ClusterState clusterState =
ClusterState.builder().metaData(metaData).routingTable(routingTable).build();
assertThat(routingTable.index("test").shards().size(), equalTo(1));
assertThat(routingTable.index("test").shard(0).size(), equalTo(1));
assertThat(routingTable.index("test").shard(0).shards().size(), equalTo(1));
assertThat(routingTable.index("test").shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
assertThat(routingTable.index("test").shard(0).shards().get(0).currentNodeId(), nullValue());
logger.info("Adding one node and rerouting");
clusterState =
ClusterState.builder(clusterState)
.nodes(DiscoveryNodes.builder().put(newNode("node1")))
.build();
RoutingTable prevRoutingTable = routingTable;
routingTable = strategy.reroute(clusterState).routingTable();
clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
assertThat(prevRoutingTable != routingTable, equalTo(true));
assertThat(routingTable.index("test").shards().size(), equalTo(1));
assertThat(routingTable.index("test").shard(0).size(), equalTo(1));
assertThat(routingTable.index("test").shard(0).shards().size(), equalTo(1));
assertThat(routingTable.index("test").shard(0).shards().get(0).unassigned(), equalTo(false));
assertThat(routingTable.index("test").shard(0).shards().get(0).state(), equalTo(INITIALIZING));
assertThat(
routingTable.index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
logger.info("Marking the shard as failed");
RoutingNodes routingNodes = clusterState.routingNodes();
prevRoutingTable = routingTable;
routingTable =
strategy
.applyFailedShard(
clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING).get(0))
.routingTable();
clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
assertThat(prevRoutingTable != routingTable, equalTo(true));
assertThat(routingTable.index("test").shards().size(), equalTo(1));
assertThat(routingTable.index("test").shard(0).size(), equalTo(1));
assertThat(routingTable.index("test").shard(0).shards().size(), equalTo(1));
assertThat(routingTable.index("test").shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
assertThat(routingTable.index("test").shard(0).shards().get(0).currentNodeId(), nullValue());
}
private void assertProperMetaDataForVersion(MetaData metaData, long version) {
for (long i = 1; i <= version; i++) {
assertThat(metaData.index("test" + i), notNullValue());
assertThat(metaData.index("test" + i).getNumberOfShards(), equalTo((int) i));
}
assertThat(metaData.index("test" + (version + 1)), nullValue());
assertThat(metaData.transientSettings().get("test"), equalTo(Long.toString(version)));
}
private void writeGlobalState(
String reason, MetaData metaData, @Nullable MetaData previousMetaData) throws Exception {
logger.trace("[_global] writing state, reason [{}]", reason);
// create metadata to write with just the global state
MetaData globalMetaData = MetaData.builder().metaData(metaData).removeAllIndices().build();
XContentBuilder builder = XContentFactory.contentBuilder(format);
builder.startObject();
MetaData.Builder.toXContent(globalMetaData, builder, formatParams);
builder.endObject();
builder.flush();
String globalFileName = "global-" + globalMetaData.version();
Exception lastFailure = null;
boolean wroteAtLeastOnce = false;
for (File dataLocation : nodeEnv.nodeDataLocations()) {
File stateLocation = new File(dataLocation, "_state");
FileSystemUtils.mkdirs(stateLocation);
File stateFile = new File(stateLocation, globalFileName);
FileOutputStream fos = null;
try {
fos = new FileOutputStream(stateFile);
BytesReference bytes = builder.bytes();
fos.write(bytes.array(), bytes.arrayOffset(), bytes.length());
fos.getChannel().force(true);
fos.close();
wroteAtLeastOnce = true;
} catch (Exception e) {
lastFailure = e;
} finally {
IOUtils.closeWhileHandlingException(fos);
}
}
if (!wroteAtLeastOnce) {
logger.warn("[_global]: failed to write global state", lastFailure);
throw new IOException("failed to write global state", lastFailure);
}
// delete the old files
for (File dataLocation : nodeEnv.nodeDataLocations()) {
File[] files = new File(dataLocation, "_state").listFiles();
if (files == null) {
continue;
}
for (File file : files) {
if (!file.getName().startsWith("global-")) {
continue;
}
if (file.getName().equals(globalFileName)) {
continue;
}
file.delete();
}
}
}
@Override
protected boolean resolveRequest(
ClusterState state, UpdateRequest request, ActionListener<UpdateResponse> listener) {
MetaData metaData = clusterService.state().metaData();
String aliasOrIndex = request.index();
request.routing((metaData.resolveIndexRouting(request.routing(), aliasOrIndex)));
request.index(metaData.concreteIndex(request.index()));
return true;
}
public static void main(String[] args) {
final int numberOfRuns = 1;
final int numIndices = 5 * 365; // five years
final int numShards = 6;
final int numReplicas = 2;
final int numberOfNodes = 30;
final int numberOfTags = 2;
AllocationService strategy =
ElasticsearchAllocationTestCase.createAllocationService(
ImmutableSettings.EMPTY, new Random(1));
MetaData.Builder mb = MetaData.builder();
for (int i = 1; i <= numIndices; i++) {
mb.put(
IndexMetaData.builder("test_" + i)
.numberOfShards(numShards)
.numberOfReplicas(numReplicas));
}
MetaData metaData = mb.build();
RoutingTable.Builder rb = RoutingTable.builder();
for (int i = 1; i <= numIndices; i++) {
rb.addAsNew(metaData.index("test_" + i));
}
RoutingTable routingTable = rb.build();
DiscoveryNodes.Builder nb = DiscoveryNodes.builder();
for (int i = 1; i <= numberOfNodes; i++) {
nb.put(newNode("node" + i, ImmutableMap.of("tag", "tag_" + (i % numberOfTags))));
}
ClusterState initialClusterState =
ClusterState.builder().metaData(metaData).routingTable(routingTable).nodes(nb).build();
long start = System.currentTimeMillis();
for (int i = 0; i < numberOfRuns; i++) {
logger.info("[{}] starting... ", i);
long runStart = System.currentTimeMillis();
ClusterState clusterState = initialClusterState;
while (clusterState.readOnlyRoutingNodes().hasUnassignedShards()) {
logger.info(
"[{}] remaining unassigned {}",
i,
clusterState.readOnlyRoutingNodes().unassigned().size());
RoutingAllocation.Result result =
strategy.applyStartedShards(
clusterState, clusterState.readOnlyRoutingNodes().shardsWithState(INITIALIZING));
clusterState = ClusterState.builder(clusterState).routingResult(result).build();
result = strategy.reroute(clusterState);
clusterState = ClusterState.builder(clusterState).routingResult(result).build();
}
logger.info(
"[{}] took {}", i, TimeValue.timeValueMillis(System.currentTimeMillis() - runStart));
}
long took = System.currentTimeMillis() - start;
logger.info(
"total took {}, AVG {}",
TimeValue.timeValueMillis(took),
TimeValue.timeValueMillis(took / numberOfRuns));
}
public void validate(RoutingTableValidation validation, MetaData metaData) {
if (!metaData.hasIndex(index())) {
validation.addIndexFailure(index(), "Exists in routing does not exists in metadata");
return;
}
IndexMetaData indexMetaData = metaData.index(index());
for (String failure : validate(indexMetaData)) {
validation.addIndexFailure(index, failure);
}
}
@Override
protected void masterOperation(
final ClusterStateRequest request,
final ClusterState state,
ActionListener<ClusterStateResponse> listener)
throws ElasticsearchException {
ClusterState currentState = clusterService.state();
logger.trace("Serving cluster state request using version {}", currentState.version());
ClusterState.Builder builder = ClusterState.builder(currentState.getClusterName());
builder.version(currentState.version());
if (request.nodes()) {
builder.nodes(currentState.nodes());
}
if (request.routingTable()) {
if (request.indices().length > 0) {
RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
for (String filteredIndex : request.indices()) {
if (currentState.routingTable().getIndicesRouting().containsKey(filteredIndex)) {
routingTableBuilder.add(
currentState.routingTable().getIndicesRouting().get(filteredIndex));
}
}
builder.routingTable(routingTableBuilder);
} else {
builder.routingTable(currentState.routingTable());
}
}
if (request.blocks()) {
builder.blocks(currentState.blocks());
}
if (request.metaData()) {
MetaData.Builder mdBuilder;
if (request.indices().length == 0) {
mdBuilder = MetaData.builder(currentState.metaData());
} else {
mdBuilder = MetaData.builder();
}
if (request.indices().length > 0) {
String[] indices =
currentState
.metaData()
.concreteIndices(IndicesOptions.lenientExpandOpen(), request.indices());
for (String filteredIndex : indices) {
IndexMetaData indexMetaData = currentState.metaData().index(filteredIndex);
if (indexMetaData != null) {
mdBuilder.put(indexMetaData, false);
}
}
}
builder.metaData(mdBuilder);
}
listener.onResponse(new ClusterStateResponse(clusterName, builder.build()));
}
private boolean metaStateExists(String nodeName, String indexName) throws Exception {
GatewayMetaState nodeMetaState =
((InternalTestCluster) cluster()).getInstance(GatewayMetaState.class, nodeName);
MetaData nodeMetaData = null;
nodeMetaData = nodeMetaState.loadMetaState();
ImmutableOpenMap<String, IndexMetaData> indices = nodeMetaData.getIndices();
boolean inMetaSate = false;
for (ObjectObjectCursor<String, IndexMetaData> index : indices) {
inMetaSate = inMetaSate || index.key.equals(indexName);
}
return inMetaSate;
}
@Override
protected void resolveRequest(
ClusterState state,
InternalRequest request,
ActionListener<IndexResponse> indexResponseActionListener) {
MetaData metaData = clusterService.state().metaData();
MappingMetaData mappingMd = null;
if (metaData.hasIndex(request.concreteIndex())) {
mappingMd =
metaData.index(request.concreteIndex()).mappingOrDefault(request.request().type());
}
request.request().process(metaData, mappingMd, allowIdGeneration, request.concreteIndex());
}
private ClusterState rejoin(ClusterState clusterState, String reason) {
logger.warn(reason + ", current nodes: {}", clusterState.nodes());
nodesFD.stop();
masterFD.stop(reason);
master = false;
ClusterBlocks clusterBlocks =
ClusterBlocks.builder()
.blocks(clusterState.blocks())
.addGlobalBlock(NO_MASTER_BLOCK)
.addGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)
.build();
// clear the routing table, we have no master, so we need to recreate the routing when we reform
// the cluster
RoutingTable routingTable = RoutingTable.builder().build();
// we also clean the metadata, since we are going to recover it if we become master
MetaData metaData = MetaData.builder().build();
// clean the nodes, we are now not connected to anybody, since we try and reform the cluster
latestDiscoNodes =
new DiscoveryNodes.Builder().put(localNode).localNodeId(localNode.id()).build();
asyncJoinCluster();
return ClusterState.builder(clusterState)
.blocks(clusterBlocks)
.nodes(latestDiscoNodes)
.routingTable(routingTable)
.metaData(metaData)
.build();
}
@Override
protected boolean resolveRequest(
ClusterState state, UpdateRequest request, ActionListener<UpdateResponse> listener) {
MetaData metaData = clusterService.state().metaData();
String aliasOrIndex = request.index();
request.routing((metaData.resolveIndexRouting(request.routing(), aliasOrIndex)));
request.index(metaData.concreteSingleIndex(request.index(), request.indicesOptions()));
// Fail fast on the node that received the request, rather than failing when translating on the
// index or delete request.
if (request.routing() == null
&& state.getMetaData().routingRequired(request.index(), request.type())) {
throw new RoutingMissingException(request.index(), request.type(), request.id());
}
return true;
}
private MetaData buildMetaDataForVersion(MetaData metaData, long version) {
ImmutableOpenMap.Builder<String, IndexMetaData> indices =
ImmutableOpenMap.builder(metaData.indices());
indices.put(
"test" + version,
IndexMetaData.builder("test" + version)
.settings(
Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
.numberOfShards((int) version)
.numberOfReplicas(0)
.build());
return MetaData.builder(metaData)
.transientSettings(Settings.builder().put("test", version).build())
.indices(indices.build())
.build();
}
@Test
public void simpleFlagTests() {
AllocationService allocation =
new AllocationService(
settingsBuilder().put("cluster.routing.allocation.concurrent_recoveries", 10).build());
logger.info("creating an index with 1 shard, no replica");
MetaData metaData =
newMetaDataBuilder()
.put(newIndexMetaDataBuilder("test").numberOfShards(1).numberOfReplicas(0))
.build();
RoutingTable routingTable = routingTable().addAsNew(metaData.index("test")).build();
ClusterState clusterState =
newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build();
assertThat(
clusterState.routingTable().index("test").shard(0).primaryAllocatedPostApi(),
equalTo(false));
logger.info("adding two nodes and performing rerouting");
clusterState =
newClusterStateBuilder()
.state(clusterState)
.nodes(newNodesBuilder().put(newNode("node1")).put(newNode("node2")))
.build();
RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState);
clusterState =
newClusterStateBuilder()
.state(clusterState)
.routingTable(rerouteResult.routingTable())
.build();
assertThat(
clusterState.routingTable().index("test").shard(0).primaryAllocatedPostApi(),
equalTo(false));
logger.info("start primary shard");
rerouteResult =
allocation.applyStartedShards(
clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING));
clusterState =
newClusterStateBuilder()
.state(clusterState)
.routingTable(rerouteResult.routingTable())
.build();
assertThat(
clusterState.routingTable().index("test").shard(0).primaryAllocatedPostApi(),
equalTo(true));
}
private DocIndexMetaData buildDocIndexMetaData(String index) {
DocIndexMetaData docIndexMetaData;
try {
docIndexMetaData = new DocIndexMetaData(functions, metaData.index(index), ident);
} catch (IOException e) {
throw new UnhandledServerException("Unable to build DocIndexMetaData", e);
}
return docIndexMetaData.build();
}
/** Cleans dangling indices if they are already allocated on the provided meta data. */
void cleanupAllocatedDangledIndices(MetaData metaData) {
for (String danglingIndex : danglingIndices.keySet()) {
if (metaData.hasIndex(danglingIndex)) {
logger.debug(
"[{}] no longer dangling (created), removing from dangling list", danglingIndex);
danglingIndices.remove(danglingIndex);
}
}
}
private boolean addFailureIfIndexIsUnavailable(
DocumentRequest request,
BulkRequest bulkRequest,
AtomicArray<BulkItemResponse> responses,
int idx,
final ConcreteIndices concreteIndices,
final MetaData metaData) {
String concreteIndex = concreteIndices.getConcreteIndex(request.index());
Exception unavailableException = null;
if (concreteIndex == null) {
try {
concreteIndex = concreteIndices.resolveIfAbsent(request);
} catch (IndexClosedException | IndexNotFoundException ex) {
// Fix for issue where bulk request references an index that
// cannot be auto-created see issue #8125
unavailableException = ex;
}
}
if (unavailableException == null) {
IndexMetaData indexMetaData = metaData.index(concreteIndex);
if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
unavailableException = new IndexClosedException(metaData.index(request.index()).getIndex());
}
}
if (unavailableException != null) {
BulkItemResponse.Failure failure =
new BulkItemResponse.Failure(
request.index(), request.type(), request.id(), unavailableException);
String operationType = "unknown";
if (request instanceof IndexRequest) {
operationType = "index";
} else if (request instanceof DeleteRequest) {
operationType = "delete";
} else if (request instanceof UpdateRequest) {
operationType = "update";
}
BulkItemResponse bulkItemResponse = new BulkItemResponse(idx, operationType, failure);
responses.set(idx, bulkItemResponse);
// make sure the request gets never processed again
bulkRequest.requests.set(idx, null);
return true;
}
return false;
}
private MetaData loadState() throws Exception {
MetaData globalMetaData = loadGlobalState();
MetaData.Builder metaDataBuilder;
if (globalMetaData != null) {
metaDataBuilder = MetaData.builder(globalMetaData);
} else {
metaDataBuilder = MetaData.builder();
}
Set<String> indices = nodeEnv.findAllIndices();
for (String index : indices) {
IndexMetaData indexMetaData = loadIndex(index);
if (indexMetaData == null) {
logger.debug("[{}] failed to find metadata for existing index location", index);
} else {
metaDataBuilder.put(indexMetaData, false);
}
}
return metaDataBuilder.build();
}
/** Creates a cluster state with no index */
public static ClusterState stateWithNoShard() {
int numberOfNodes = 2;
DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
discoBuilder.localNodeId(newNode(0).getId());
discoBuilder.masterNodeId(newNode(1).getId());
ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
state.nodes(discoBuilder);
state.metaData(MetaData.builder().generateClusterUuidIfNeeded());
state.routingTable(RoutingTable.builder().build());
return state.build();
}
public static void toXContent(
MetaData metaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, "API"));
builder.startObject("meta-data");
builder.field("version", metaData.version());
builder.field("cluster_uuid", metaData.clusterUUID);
if (!metaData.persistentSettings().getAsMap().isEmpty()) {
builder.startObject("settings");
for (Map.Entry<String, String> entry :
metaData.persistentSettings().getAsMap().entrySet()) {
builder.field(entry.getKey(), entry.getValue());
}
builder.endObject();
}
if (context == XContentContext.API && !metaData.transientSettings().getAsMap().isEmpty()) {
builder.startObject("transient_settings");
for (Map.Entry<String, String> entry : metaData.transientSettings().getAsMap().entrySet()) {
builder.field(entry.getKey(), entry.getValue());
}
builder.endObject();
}
builder.startObject("templates");
for (ObjectCursor<IndexTemplateMetaData> cursor : metaData.templates().values()) {
IndexTemplateMetaData.Builder.toXContent(cursor.value, builder, params);
}
builder.endObject();
if (context == XContentContext.API && !metaData.indices().isEmpty()) {
builder.startObject("indices");
for (IndexMetaData indexMetaData : metaData) {
IndexMetaData.Builder.toXContent(indexMetaData, builder, params);
}
builder.endObject();
}
for (ObjectObjectCursor<String, Custom> cursor : metaData.customs()) {
Custom proto = lookupPrototypeSafe(cursor.key);
if (proto.context().contains(context)) {
builder.startObject(cursor.key);
cursor.value.toXContent(builder, params);
builder.endObject();
}
}
builder.endObject();
}
private DocIndexMetaData docIndexMetaData() {
DocIndexMetaData docIndexMetaData;
String templateName = PartitionName.templateName(ident.schema(), ident.name());
boolean createdFromTemplate = false;
if (metaData.getTemplates().containsKey(templateName)) {
docIndexMetaData = buildDocIndexMetaDataFromTemplate(ident.indexName(), templateName);
createdFromTemplate = true;
concreteIndices =
metaData.concreteIndices(IndicesOptions.lenientExpandOpen(), ident.indexName());
} else {
try {
concreteIndices =
metaData.concreteIndices(IndicesOptions.strictExpandOpen(), ident.indexName());
if (concreteIndices.length == 0) {
// no matching index found
throw new TableUnknownException(ident);
}
docIndexMetaData = buildDocIndexMetaData(concreteIndices[0]);
} catch (IndexMissingException ex) {
throw new TableUnknownException(ident.fqn(), ex);
}
}
if ((!createdFromTemplate && concreteIndices.length == 1) || !checkAliasSchema) {
return docIndexMetaData;
}
for (int i = 0; i < concreteIndices.length; i++) {
try {
docIndexMetaData =
docIndexMetaData.merge(
buildDocIndexMetaData(concreteIndices[i]),
transportPutIndexTemplateAction,
createdFromTemplate);
} catch (IOException e) {
throw new UnhandledServerException("Unable to merge/build new DocIndexMetaData", e);
}
}
return docIndexMetaData;
}
public boolean equalsAliases(MetaData other) {
for (ObjectCursor<IndexMetaData> cursor : other.indices().values()) {
IndexMetaData otherIndex = cursor.value;
IndexMetaData thisIndex = index(otherIndex.getIndex());
if (thisIndex == null) {
return false;
}
if (otherIndex.getAliases().equals(thisIndex.getAliases()) == false) {
return false;
}
}
return true;
}
@Override
public void clusterChanged(ClusterChangedEvent event) {
if (event.state().blocks().disableStatePersistence()) {
// reset the current metadata, we need to start fresh...
this.currentMetaData = null;
return;
}
MetaData newMetaData = event.state().metaData();
// delete indices that were there before, but are deleted now
// we need to do it so they won't be detected as dangling
if (currentMetaData != null) {
// only delete indices when we already received a state (currentMetaData != null)
for (IndexMetaData current : currentMetaData) {
if (!newMetaData.hasIndex(current.index())) {
logger.debug(
"[{}] deleting index that is no longer part of the metadata (indices: [{}])",
current.index(),
newMetaData.indices().keys());
if (nodeEnv.hasNodeFile()) {
FileSystemUtils.deleteRecursively(nodeEnv.indexLocations(new Index(current.index())));
}
try {
nodeIndexDeletedAction.nodeIndexStoreDeleted(
event.state(), current.index(), event.state().nodes().localNodeId());
} catch (Exception e) {
logger.debug(
"[{}] failed to notify master on local index store deletion", e, current.index());
}
}
}
}
currentMetaData = newMetaData;
}
public static boolean isGlobalStateEquals(MetaData metaData1, MetaData metaData2) {
if (!metaData1.persistentSettings.equals(metaData2.persistentSettings)) {
return false;
}
if (!metaData1.templates.equals(metaData2.templates())) {
return false;
}
// Check if any persistent metadata needs to be saved
int customCount1 = 0;
for (ObjectObjectCursor<String, Custom> cursor : metaData1.customs) {
if (customPrototypes.get(cursor.key).context().contains(XContentContext.GATEWAY)) {
if (!cursor.value.equals(metaData2.custom(cursor.key))) return false;
customCount1++;
}
}
int customCount2 = 0;
for (ObjectObjectCursor<String, Custom> cursor : metaData2.customs) {
if (customPrototypes.get(cursor.key).context().contains(XContentContext.GATEWAY)) {
customCount2++;
}
}
if (customCount1 != customCount2) return false;
return true;
}
/**
* Creates a cluster state where local node and master node can be specified
*
* @param localNode node in allNodes that is the local node
* @param masterNode node in allNodes that is the master node. Can be null if no master exists
* @param allNodes all nodes in the cluster
* @return cluster state
*/
public static ClusterState state(
DiscoveryNode localNode, DiscoveryNode masterNode, DiscoveryNode... allNodes) {
DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
for (DiscoveryNode node : allNodes) {
discoBuilder.put(node);
}
if (masterNode != null) {
discoBuilder.masterNodeId(masterNode.getId());
}
discoBuilder.localNodeId(localNode.getId());
ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
state.nodes(discoBuilder);
state.metaData(MetaData.builder().generateClusterUuidIfNeeded());
return state.build();
}
private DocIndexMetaData buildDocIndexMetaDataFromTemplate(String index, String templateName) {
IndexTemplateMetaData indexTemplateMetaData = metaData.getTemplates().get(templateName);
DocIndexMetaData docIndexMetaData;
try {
IndexMetaData.Builder builder = new IndexMetaData.Builder(index);
builder.putMapping(
Constants.DEFAULT_MAPPING_TYPE,
indexTemplateMetaData.getMappings().get(Constants.DEFAULT_MAPPING_TYPE).toString());
Settings settings = indexTemplateMetaData.settings();
builder.settings(settings);
// default values
builder.numberOfShards(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, 5));
builder.numberOfReplicas(settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, 1));
docIndexMetaData = new DocIndexMetaData(functions, builder.build(), ident);
} catch (IOException e) {
throw new UnhandledServerException("Unable to build DocIndexMetaData from template", e);
}
return docIndexMetaData.build();
}
/** Creates cluster state with several shards and one replica and all shards STARTED. */
public static ClusterState stateWithAssignedPrimariesAndOneReplica(
String index, int numberOfShards) {
int numberOfNodes = 2; // we need a non-local master to test shard failures
DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
for (int i = 0; i < numberOfNodes + 1; i++) {
final DiscoveryNode node = newNode(i);
discoBuilder = discoBuilder.put(node);
}
discoBuilder.localNodeId(newNode(0).getId());
discoBuilder.masterNodeId(
newNode(1).getId()); // we need a non-local master to test shard failures
IndexMetaData indexMetaData =
IndexMetaData.builder(index)
.settings(
Settings.builder()
.put(SETTING_VERSION_CREATED, Version.CURRENT)
.put(SETTING_NUMBER_OF_SHARDS, numberOfShards)
.put(SETTING_NUMBER_OF_REPLICAS, 1)
.put(SETTING_CREATION_DATE, System.currentTimeMillis()))
.build();
ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
state.nodes(discoBuilder);
state.metaData(MetaData.builder().put(indexMetaData, false).generateClusterUuidIfNeeded());
IndexRoutingTable.Builder indexRoutingTableBuilder =
IndexRoutingTable.builder(indexMetaData.getIndex());
for (int i = 0; i < numberOfShards; i++) {
RoutingTable.Builder routing = new RoutingTable.Builder();
routing.addAsNew(indexMetaData);
final ShardId shardId = new ShardId(index, "_na_", i);
IndexShardRoutingTable.Builder indexShardRoutingBuilder =
new IndexShardRoutingTable.Builder(shardId);
indexShardRoutingBuilder.addShard(
TestShardRouting.newShardRouting(
index, i, newNode(0).getId(), null, null, true, ShardRoutingState.STARTED, null));
indexShardRoutingBuilder.addShard(
TestShardRouting.newShardRouting(
index, i, newNode(1).getId(), null, null, false, ShardRoutingState.STARTED, null));
indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder.build());
}
state.routingTable(RoutingTable.builder().add(indexRoutingTableBuilder.build()).build());
return state.build();
}