public Builder put(IndexMetaData.Builder indexMetaDataBuilder) {
// we know its a new one, increment the version and store
indexMetaDataBuilder.version(indexMetaDataBuilder.version() + 1);
IndexMetaData indexMetaData = indexMetaDataBuilder.build();
indices.put(indexMetaData.getIndex().getName(), indexMetaData);
return this;
}
// TODO: This can be moved to IndexNameExpressionResolver too, but this means that we will support
// wildcards and other expressions
// in the index,bulk,update and delete apis.
public String resolveIndexRouting(
@Nullable String parent, @Nullable String routing, String aliasOrIndex) {
if (aliasOrIndex == null) {
if (routing == null) {
return parent;
}
return routing;
}
AliasOrIndex result = getAliasAndIndexLookup().get(aliasOrIndex);
if (result == null || result.isAlias() == false) {
if (routing == null) {
return parent;
}
return routing;
}
AliasOrIndex.Alias alias = (AliasOrIndex.Alias) result;
if (result.getIndices().size() > 1) {
String[] indexNames = new String[result.getIndices().size()];
int i = 0;
for (IndexMetaData indexMetaData : result.getIndices()) {
indexNames[i++] = indexMetaData.getIndex().getName();
}
throw new IllegalArgumentException(
"Alias ["
+ aliasOrIndex
+ "] has more than one index associated with it ["
+ Arrays.toString(indexNames)
+ "], can't execute a single index op");
}
AliasMetaData aliasMd = alias.getFirstAliasMetaData();
if (aliasMd.indexRouting() != null) {
if (aliasMd.indexRouting().indexOf(',') != -1) {
throw new IllegalArgumentException(
"index/alias ["
+ aliasOrIndex
+ "] provided with routing value ["
+ aliasMd.getIndexRouting()
+ "] that resolved to several routing values, rejecting operation");
}
if (routing != null) {
if (!routing.equals(aliasMd.indexRouting())) {
throw new IllegalArgumentException(
"Alias ["
+ aliasOrIndex
+ "] has index routing associated with it ["
+ aliasMd.indexRouting()
+ "], and was provided with routing value ["
+ routing
+ "], rejecting operation");
}
}
// Alias routing overrides the parent routing (if any).
return aliasMd.indexRouting();
}
if (routing == null) {
return parent;
}
return routing;
}
@Override
protected Tuple<IndexResponse, IndexRequest> shardOperationOnPrimary(
ClusterState clusterState, PrimaryOperationRequest shardRequest) throws Throwable {
final IndexRequest request = shardRequest.request;
// validate, if routing is required, that we got routing
IndexMetaData indexMetaData = clusterState.metaData().index(shardRequest.shardId.getIndex());
MappingMetaData mappingMd = indexMetaData.mappingOrDefault(request.type());
if (mappingMd != null && mappingMd.routing().required()) {
if (request.routing() == null) {
throw new RoutingMissingException(
shardRequest.shardId.getIndex(), request.type(), request.id());
}
}
IndexService indexService = indicesService.indexServiceSafe(shardRequest.shardId.getIndex());
IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id());
final WriteResult<IndexResponse> result =
executeIndexRequestOnPrimary(null, request, indexShard);
final IndexResponse response = result.response;
final Translog.Location location = result.location;
processAfter(request.refresh(), indexShard, location);
return new Tuple<>(response, shardRequest.request);
}
/** Checks the mappings for compatibility with the current version */
private void checkMappingsCompatibility(IndexMetaData indexMetaData) {
Index index = new Index(indexMetaData.getIndex());
Settings settings = indexMetaData.getSettings();
try {
SimilarityService similarityService = new SimilarityService(index, settings);
// We cannot instantiate real analysis server at this point because the node might not have
// been started yet. However, we don't really need real analyzers at this stage - so we can
// fake it
try (AnalysisService analysisService = new FakeAnalysisService(index, settings)) {
try (MapperService mapperService =
new MapperService(index, settings, analysisService, similarityService, scriptService)) {
for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
MappingMetaData mappingMetaData = cursor.value;
mapperService.merge(mappingMetaData.type(), mappingMetaData.source(), false, false);
}
}
}
} catch (Exception ex) {
// Wrap the inner exception so we have the index name in the exception message
throw new IllegalStateException(
"unable to upgrade the mappings for the index ["
+ indexMetaData.getIndex()
+ "], reason: ["
+ ex.getMessage()
+ "]",
ex);
}
}
/*
* Finds all mappings for types and concrete indices. Types are expanded to
* include all types that match the glob patterns in the types array. Empty
* types array, null or {"_all"} will be expanded to all types available for
* the given indices.
*/
public ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> findMappings(
String[] concreteIndices, final String[] types) {
assert types != null;
assert concreteIndices != null;
if (concreteIndices.length == 0) {
return ImmutableOpenMap.of();
}
ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder =
ImmutableOpenMap.builder();
Iterable<String> intersection =
HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys());
for (String index : intersection) {
IndexMetaData indexMetaData = indices.get(index);
ImmutableOpenMap.Builder<String, MappingMetaData> filteredMappings;
if (isAllTypes(types)) {
indexMapBuilder.put(
index, indexMetaData.getMappings()); // No types specified means get it all
} else {
filteredMappings = ImmutableOpenMap.builder();
for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.getMappings()) {
if (Regex.simpleMatch(types, cursor.key)) {
filteredMappings.put(cursor.key, cursor.value);
}
}
if (!filteredMappings.isEmpty()) {
indexMapBuilder.put(index, filteredMappings.build());
}
}
}
return indexMapBuilder.build();
}
private void applyCleanedIndices(final ClusterChangedEvent event) {
// handle closed indices, since they are not allocated on a node once they are closed
// so applyDeletedIndices might not take them into account
for (IndexService indexService : indicesService) {
String index = indexService.index().getName();
IndexMetaData indexMetaData = event.state().metaData().index(index);
if (indexMetaData != null && indexMetaData.state() == IndexMetaData.State.CLOSE) {
for (Integer shardId : indexService.shardIds()) {
logger.debug("[{}][{}] removing shard (index is closed)", index, shardId);
try {
indexService.removeShard(shardId, "removing shard (index is closed)");
} catch (Throwable e) {
logger.warn("[{}] failed to remove shard (index is closed)", e, index);
}
}
}
}
for (IndexService indexService : indicesService) {
String index = indexService.index().getName();
if (indexService.shardIds().isEmpty()) {
if (logger.isDebugEnabled()) {
logger.debug("[{}] cleaning index (no shards allocated)", index);
}
// clean the index
removeIndex(index, "removing index (no shards allocated)");
}
}
}
@Test
@SuppressWarnings("squid:S2925")
public void testThatMappingFromTemplateIsApplied() throws Exception {
registry.counter(name("test", "cache-evictions")).inc();
reportAndRefresh();
// somehow the cluster state is not immediately updated... need to check
Thread.sleep(200);
ClusterStateResponse clusterStateResponse =
client()
.admin()
.cluster()
.prepareState()
.setRoutingTable(false)
.setLocal(false)
.setNodes(true)
.setIndices(indexWithDate)
.execute()
.actionGet();
org.assertj.core.api.Assertions.assertThat(
clusterStateResponse.getState().getMetaData().getIndices().containsKey(indexWithDate))
.isTrue();
IndexMetaData indexMetaData =
clusterStateResponse.getState().getMetaData().getIndices().get(indexWithDate);
org.assertj.core.api.Assertions.assertThat(indexMetaData.getMappings().containsKey("counter"))
.isTrue();
Map<String, Object> properties =
getAsMap(indexMetaData.mapping("counter").sourceAsMap(), "properties");
Map<String, Object> mapping = getAsMap(properties, "name");
org.assertj.core.api.Assertions.assertThat(mapping).containsKey("index");
org.assertj.core.api.Assertions.assertThat(mapping.get("index").toString())
.isEqualTo("not_analyzed");
}
/** Initializes an index, to be restored from snapshot */
private Builder initializeAsRestore(
IndexMetaData indexMetaData, RestoreSource restoreSource, boolean asNew) {
if (!shards.isEmpty()) {
throw new ElasticSearchIllegalStateException(
"trying to initialize an index with fresh shards, but already has shards created");
}
for (int shardId = 0; shardId < indexMetaData.numberOfShards(); shardId++) {
IndexShardRoutingTable.Builder indexShardRoutingBuilder =
new IndexShardRoutingTable.Builder(
new ShardId(indexMetaData.index(), shardId), asNew ? false : true);
for (int i = 0; i <= indexMetaData.numberOfReplicas(); i++) {
indexShardRoutingBuilder.addShard(
new ImmutableShardRouting(
index,
shardId,
null,
null,
i == 0 ? restoreSource : null,
i == 0,
ShardRoutingState.UNASSIGNED,
0));
}
shards.put(shardId, indexShardRoutingBuilder.build());
}
return this;
}
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
if (cancellableThreads.isCancelled()
== false) { // no need to run this if recovery is canceled
IndexMetaData indexMetaData =
clusterService.state().metaData().getIndices().get(indexService.index().getName());
ImmutableOpenMap<String, MappingMetaData> metaDataMappings = null;
if (indexMetaData != null) {
metaDataMappings = indexMetaData.getMappings();
}
// default mapping should not be sent back, it can only be updated by put mapping API, and
// its
// a full in place replace, we don't want to override a potential update coming into it
for (DocumentMapper documentMapper : indexService.mapperService().docMappers(false)) {
MappingMetaData mappingMetaData =
metaDataMappings == null ? null : metaDataMappings.get(documentMapper.type());
if (mappingMetaData == null
|| !documentMapper.refreshSource().equals(mappingMetaData.source())) {
// not on master yet in the right form
documentMappersToUpdate.add(documentMapper);
}
}
}
return currentState;
}
/** Marks index as upgraded so we don't have to test it again */
private IndexMetaData markAsUpgraded(IndexMetaData indexMetaData) {
Settings settings =
Settings.builder()
.put(indexMetaData.getSettings())
.put(IndexMetaData.SETTING_VERSION_UPGRADED, Version.CURRENT)
.build();
return IndexMetaData.builder(indexMetaData).settings(settings).build();
}
public Builder updateBlocks(IndexMetaData indexMetaData) {
removeIndexBlock(
indexMetaData.getIndex().getName(), MetaDataIndexStateService.INDEX_CLOSED_BLOCK);
removeIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_READ_ONLY_BLOCK);
removeIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_READ_BLOCK);
removeIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_WRITE_BLOCK);
removeIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_METADATA_BLOCK);
return addBlocks(indexMetaData);
}
private void removeIndex(
ClusterBlocks.Builder blocks,
MetaData.Builder metaData,
RoutingTable.Builder routingTable,
IndexMetaData index) {
metaData.remove(index.getIndex());
routingTable.remove(index.getIndex());
blocks.removeIndexBlocks(index.getIndex());
}
public Builder(IndexMetaData indexMetaData) {
this(indexMetaData.index());
settings(indexMetaData.settings());
mappings.putAll(indexMetaData.mappings);
aliases.putAll(indexMetaData.aliases);
customs.putAll(indexMetaData.customs);
this.state = indexMetaData.state;
this.version = indexMetaData.version;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
fromNode.writeTo(out);
out.writeVInt(indices.length);
for (IndexMetaData indexMetaData : indices) {
indexMetaData.writeTo(out);
}
}
/**
* @param concreteIndex The concrete index to check if routing is required
* @param type The type to check if routing is required
* @return Whether routing is required according to the mapping for the specified index and type
*/
public boolean routingRequired(String concreteIndex, String type) {
IndexMetaData indexMetaData = indices.get(concreteIndex);
if (indexMetaData != null) {
MappingMetaData mappingMetaData = indexMetaData.getMappings().get(type);
if (mappingMetaData != null) {
return mappingMetaData.routing().required();
}
}
return false;
}
public Builder(IndexMetaData indexMetaData) {
this.index = indexMetaData.getIndex().getName();
this.state = indexMetaData.state;
this.version = indexMetaData.version;
this.settings = indexMetaData.getSettings();
this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings);
this.aliases = ImmutableOpenMap.builder(indexMetaData.aliases);
this.customs = ImmutableOpenMap.builder(indexMetaData.customs);
this.activeAllocationIds = ImmutableOpenIntMap.builder(indexMetaData.activeAllocationIds);
}
private Decision shouldIndexFilter(
IndexMetaData indexMd, RoutingNode node, RoutingAllocation allocation) {
if (indexMd.requireFilters() != null) {
if (!indexMd.requireFilters().match(node.node())) {
return allocation.decision(
Decision.NO,
NAME,
"node does not match index required filters [%s]",
indexMd.requireFilters());
}
}
if (indexMd.includeFilters() != null) {
if (!indexMd.includeFilters().match(node.node())) {
return allocation.decision(
Decision.NO,
NAME,
"node does not match index include filters [%s]",
indexMd.includeFilters());
}
}
if (indexMd.excludeFilters() != null) {
if (indexMd.excludeFilters().match(node.node())) {
return allocation.decision(
Decision.NO, NAME, "node matches index exclude filters [%s]", indexMd.excludeFilters());
}
}
return null;
}
private Decision shouldIndexFilter(
IndexMetaData indexMd, RoutingNode node, RoutingAllocation allocation) {
if (indexMd.requireFilters() != null) {
if (!indexMd.requireFilters().match(node.node())) {
return allocation.decision(
Decision.NO,
NAME,
"node does not match index setting [%s] filters [%s]",
IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX,
indexMd.requireFilters());
}
}
if (indexMd.includeFilters() != null) {
if (!indexMd.includeFilters().match(node.node())) {
return allocation.decision(
Decision.NO,
NAME,
"node does not match index setting [%s] filters [%s]",
IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_PREFIX,
indexMd.includeFilters());
}
}
if (indexMd.excludeFilters() != null) {
if (indexMd.excludeFilters().match(node.node())) {
return allocation.decision(
Decision.NO,
NAME,
"node matches index setting [%s] filters [%s]",
IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey(),
indexMd.excludeFilters());
}
}
return null;
}
public void testCloseIndexAcknowledgement() {
createIndex("test");
ensureGreen();
assertAcked(client().admin().indices().prepareClose("test"));
for (Client client : clients()) {
IndexMetaData indexMetaData = getLocalClusterState(client).metaData().indices().get("test");
assertThat(indexMetaData.getState(), equalTo(State.CLOSE));
}
}
/**
* Elasticsearch 3.0 no longer supports indices with pre Lucene v5.0 (Elasticsearch v2.0.0.beta1)
* segments. All indices that were created before Elasticsearch v2.0.0.beta1 should be upgraded
* using upgrade API before they can be open by this version of elasticsearch.
*/
private void checkSupportedVersion(IndexMetaData indexMetaData) {
if (indexMetaData.getState() == IndexMetaData.State.OPEN
&& isSupportedVersion(indexMetaData) == false) {
throw new IllegalStateException(
"The index ["
+ indexMetaData.getIndex()
+ "] was created before v2.0.0.beta1 and wasn't upgraded."
+ " This index should be open using a version before "
+ Version.CURRENT.minimumCompatibilityVersion()
+ " and upgraded using the upgrade API.");
}
}
public Builder put(IndexMetaData indexMetaData, boolean incrementVersion) {
if (indices.get(indexMetaData.getIndex().getName()) == indexMetaData) {
return this;
}
// if we put a new index metadata, increment its version
if (incrementVersion) {
indexMetaData =
IndexMetaData.builder(indexMetaData).version(indexMetaData.getVersion() + 1).build();
}
indices.put(indexMetaData.getIndex().getName(), indexMetaData);
return this;
}
private void updateMappingOnMaster(final String index, final String type) {
try {
MapperService mapperService = indicesService.indexServiceSafe(index).mapperService();
final DocumentMapper documentMapper = mapperService.documentMapper(type);
if (documentMapper == null) { // should not happen
return;
}
IndexMetaData metaData = clusterService.state().metaData().index(index);
if (metaData == null) {
return;
}
long orderId = mappingUpdatedAction.generateNextMappingUpdateOrder();
documentMapper.refreshSource();
DiscoveryNode node = clusterService.localNode();
final MappingUpdatedAction.MappingUpdatedRequest request =
new MappingUpdatedAction.MappingUpdatedRequest(
index,
metaData.uuid(),
type,
documentMapper.mappingSource(),
orderId,
node != null ? node.id() : null);
mappingUpdatedAction.execute(
request,
new ActionListener<MappingUpdatedAction.MappingUpdatedResponse>() {
@Override
public void onResponse(
MappingUpdatedAction.MappingUpdatedResponse mappingUpdatedResponse) {
// all is well
}
@Override
public void onFailure(Throwable e) {
try {
logger.warn(
"failed to update master on updated mapping for index [{}], type [{}] and source [{}]",
e,
index,
type,
documentMapper.mappingSource().string());
} catch (IOException e1) {
// ignore
}
}
});
} catch (Exception e) {
logger.warn(
"failed to update master on updated mapping for index [{}], type [{}]", e, index, type);
}
}
public boolean equalsAliases(MetaData other) {
for (ObjectCursor<IndexMetaData> cursor : other.indices().values()) {
IndexMetaData otherIndex = cursor.value;
IndexMetaData thisIndex = index(otherIndex.getIndex());
if (thisIndex == null) {
return false;
}
if (otherIndex.getAliases().equals(thisIndex.getAliases()) == false) {
return false;
}
}
return true;
}
/*
* Returns true if this index can be supported by the current version of elasticsearch
*/
private static boolean isSupportedVersion(IndexMetaData indexMetaData) {
if (indexMetaData.getCreationVersion().onOrAfter(Version.V_2_0_0_beta1)) {
// The index was created with elasticsearch that was using Lucene 5.2.1
return true;
}
if (indexMetaData.getMinimumCompatibleVersion() != null
&& indexMetaData
.getMinimumCompatibleVersion()
.onOrAfter(org.apache.lucene.util.Version.LUCENE_5_0_0)) {
// The index was upgraded we can work with it
return true;
}
return false;
}
private void writeIndex(
String reason, IndexMetaData indexMetaData, @Nullable IndexMetaData previousIndexMetaData)
throws Exception {
logger.trace("[{}] writing state, reason [{}]", indexMetaData.index(), reason);
XContentBuilder builder = XContentFactory.contentBuilder(format, new BytesStreamOutput());
builder.startObject();
IndexMetaData.Builder.toXContent(indexMetaData, builder, formatParams);
builder.endObject();
builder.flush();
String stateFileName = "state-" + indexMetaData.version();
Exception lastFailure = null;
boolean wroteAtLeastOnce = false;
for (File indexLocation : nodeEnv.indexLocations(new Index(indexMetaData.index()))) {
File stateLocation = new File(indexLocation, "_state");
FileSystemUtils.mkdirs(stateLocation);
File stateFile = new File(stateLocation, stateFileName);
FileOutputStream fos = null;
try {
fos = new FileOutputStream(stateFile);
BytesReference bytes = builder.bytes();
fos.write(bytes.array(), bytes.arrayOffset(), bytes.length());
fos.getChannel().force(true);
fos.close();
wroteAtLeastOnce = true;
} catch (Exception e) {
lastFailure = e;
} finally {
IOUtils.closeWhileHandlingException(fos);
}
}
if (!wroteAtLeastOnce) {
logger.warn("[{}]: failed to state", lastFailure, indexMetaData.index());
throw new IOException(
"failed to write state for [" + indexMetaData.index() + "]", lastFailure);
}
// delete the old files
if (previousIndexMetaData != null
&& previousIndexMetaData.version() != indexMetaData.version()) {
for (File indexLocation : nodeEnv.indexLocations(new Index(indexMetaData.index()))) {
File[] files = new File(indexLocation, "_state").listFiles();
if (files == null) {
continue;
}
for (File file : files) {
if (!file.getName().startsWith("state-")) {
continue;
}
if (file.getName().equals(stateFileName)) {
continue;
}
file.delete();
}
}
}
}
public Builder updateSettings(Settings settings, String... indices) {
if (indices == null || indices.length == 0) {
indices = this.indices.keys().toArray(String.class);
}
for (String index : indices) {
IndexMetaData indexMetaData = this.indices.get(index);
if (indexMetaData == null) {
throw new IndexNotFoundException(index);
}
put(
IndexMetaData.builder(indexMetaData)
.settings(settingsBuilder().put(indexMetaData.getSettings()).put(settings)));
}
return this;
}
@Test
public void testRoutingTableSerialization() throws Exception {
MetaData metaData =
MetaData.builder()
.put(IndexMetaData.builder("test").numberOfShards(10).numberOfReplicas(1))
.build();
RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
DiscoveryNodes nodes =
DiscoveryNodes.builder()
.put(newNode("node1"))
.put(newNode("node2"))
.put(newNode("node3"))
.build();
ClusterState clusterState =
ClusterState.builder().nodes(nodes).metaData(metaData).routingTable(routingTable).build();
AllocationService strategy = createAllocationService();
RoutingTable source = strategy.reroute(clusterState).routingTable();
BytesStreamOutput outStream = new BytesStreamOutput();
RoutingTable.Builder.writeTo(source, outStream);
BytesStreamInput inStream = new BytesStreamInput(outStream.bytes().toBytes(), false);
RoutingTable target = RoutingTable.Builder.readFrom(inStream);
assertThat(target.prettyPrint(), equalTo(source.prettyPrint()));
}
@Override
public void onRecoveryDone(RecoveryState state) {
shardStateAction.shardStarted(
shardRouting,
indexMetaData.getIndexUUID(),
"after recovery (replica) from node [" + state.getSourceNode() + "]");
}
@Test
public void testClusterStateSerialization() throws Exception {
MetaData metaData =
MetaData.builder()
.put(IndexMetaData.builder("test").numberOfShards(10).numberOfReplicas(1))
.build();
RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
DiscoveryNodes nodes =
DiscoveryNodes.builder()
.put(newNode("node1"))
.put(newNode("node2"))
.put(newNode("node3"))
.localNodeId("node1")
.masterNodeId("node2")
.build();
ClusterState clusterState =
ClusterState.builder().nodes(nodes).metaData(metaData).routingTable(routingTable).build();
AllocationService strategy = createAllocationService();
clusterState =
ClusterState.builder(clusterState)
.routingTable(strategy.reroute(clusterState).routingTable())
.build();
ClusterState serializedClusterState =
ClusterState.Builder.fromBytes(
ClusterState.Builder.toBytes(clusterState), newNode("node1"));
assertThat(
serializedClusterState.routingTable().prettyPrint(),
equalTo(clusterState.routingTable().prettyPrint()));
}
@Override
public synchronized void updateMetaData(final IndexMetaData metadata) {
final Translog.Durability oldTranslogDurability = indexSettings.getTranslogDurability();
if (indexSettings.updateIndexMetaData(metadata)) {
for (final IndexShard shard : this.shards.values()) {
try {
shard.onSettingsChanged();
} catch (Exception e) {
logger.warn("[{}] failed to notify shard about setting change", e, shard.shardId().id());
}
}
if (refreshTask.getInterval().equals(indexSettings.getRefreshInterval()) == false) {
rescheduleRefreshTasks();
}
final Translog.Durability durability = indexSettings.getTranslogDurability();
if (durability != oldTranslogDurability) {
rescheduleFsyncTask(durability);
}
}
// update primary terms
for (final IndexShard shard : this.shards.values()) {
shard.updatePrimaryTerm(metadata.primaryTerm(shard.shardId().id()));
}
}