public Builder put(IndexMetaData.Builder indexMetaDataBuilder) { // we know its a new one, increment the version and store indexMetaDataBuilder.version(indexMetaDataBuilder.version() + 1); IndexMetaData indexMetaData = indexMetaDataBuilder.build(); indices.put(indexMetaData.getIndex().getName(), indexMetaData); return this; }
// TODO: This can be moved to IndexNameExpressionResolver too, but this means that we will support // wildcards and other expressions // in the index,bulk,update and delete apis. public String resolveIndexRouting( @Nullable String parent, @Nullable String routing, String aliasOrIndex) { if (aliasOrIndex == null) { if (routing == null) { return parent; } return routing; } AliasOrIndex result = getAliasAndIndexLookup().get(aliasOrIndex); if (result == null || result.isAlias() == false) { if (routing == null) { return parent; } return routing; } AliasOrIndex.Alias alias = (AliasOrIndex.Alias) result; if (result.getIndices().size() > 1) { String[] indexNames = new String[result.getIndices().size()]; int i = 0; for (IndexMetaData indexMetaData : result.getIndices()) { indexNames[i++] = indexMetaData.getIndex().getName(); } throw new IllegalArgumentException( "Alias [" + aliasOrIndex + "] has more than one index associated with it [" + Arrays.toString(indexNames) + "], can't execute a single index op"); } AliasMetaData aliasMd = alias.getFirstAliasMetaData(); if (aliasMd.indexRouting() != null) { if (aliasMd.indexRouting().indexOf(',') != -1) { throw new IllegalArgumentException( "index/alias [" + aliasOrIndex + "] provided with routing value [" + aliasMd.getIndexRouting() + "] that resolved to several routing values, rejecting operation"); } if (routing != null) { if (!routing.equals(aliasMd.indexRouting())) { throw new IllegalArgumentException( "Alias [" + aliasOrIndex + "] has index routing associated with it [" + aliasMd.indexRouting() + "], and was provided with routing value [" + routing + "], rejecting operation"); } } // Alias routing overrides the parent routing (if any). return aliasMd.indexRouting(); } if (routing == null) { return parent; } return routing; }
@Override protected Tuple<IndexResponse, IndexRequest> shardOperationOnPrimary( ClusterState clusterState, PrimaryOperationRequest shardRequest) throws Throwable { final IndexRequest request = shardRequest.request; // validate, if routing is required, that we got routing IndexMetaData indexMetaData = clusterState.metaData().index(shardRequest.shardId.getIndex()); MappingMetaData mappingMd = indexMetaData.mappingOrDefault(request.type()); if (mappingMd != null && mappingMd.routing().required()) { if (request.routing() == null) { throw new RoutingMissingException( shardRequest.shardId.getIndex(), request.type(), request.id()); } } IndexService indexService = indicesService.indexServiceSafe(shardRequest.shardId.getIndex()); IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id()); final WriteResult<IndexResponse> result = executeIndexRequestOnPrimary(null, request, indexShard); final IndexResponse response = result.response; final Translog.Location location = result.location; processAfter(request.refresh(), indexShard, location); return new Tuple<>(response, shardRequest.request); }
/** Checks the mappings for compatibility with the current version */ private void checkMappingsCompatibility(IndexMetaData indexMetaData) { Index index = new Index(indexMetaData.getIndex()); Settings settings = indexMetaData.getSettings(); try { SimilarityService similarityService = new SimilarityService(index, settings); // We cannot instantiate real analysis server at this point because the node might not have // been started yet. However, we don't really need real analyzers at this stage - so we can // fake it try (AnalysisService analysisService = new FakeAnalysisService(index, settings)) { try (MapperService mapperService = new MapperService(index, settings, analysisService, similarityService, scriptService)) { for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) { MappingMetaData mappingMetaData = cursor.value; mapperService.merge(mappingMetaData.type(), mappingMetaData.source(), false, false); } } } } catch (Exception ex) { // Wrap the inner exception so we have the index name in the exception message throw new IllegalStateException( "unable to upgrade the mappings for the index [" + indexMetaData.getIndex() + "], reason: [" + ex.getMessage() + "]", ex); } }
/* * Finds all mappings for types and concrete indices. Types are expanded to * include all types that match the glob patterns in the types array. Empty * types array, null or {"_all"} will be expanded to all types available for * the given indices. */ public ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> findMappings( String[] concreteIndices, final String[] types) { assert types != null; assert concreteIndices != null; if (concreteIndices.length == 0) { return ImmutableOpenMap.of(); } ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder = ImmutableOpenMap.builder(); Iterable<String> intersection = HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys()); for (String index : intersection) { IndexMetaData indexMetaData = indices.get(index); ImmutableOpenMap.Builder<String, MappingMetaData> filteredMappings; if (isAllTypes(types)) { indexMapBuilder.put( index, indexMetaData.getMappings()); // No types specified means get it all } else { filteredMappings = ImmutableOpenMap.builder(); for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.getMappings()) { if (Regex.simpleMatch(types, cursor.key)) { filteredMappings.put(cursor.key, cursor.value); } } if (!filteredMappings.isEmpty()) { indexMapBuilder.put(index, filteredMappings.build()); } } } return indexMapBuilder.build(); }
private void applyCleanedIndices(final ClusterChangedEvent event) { // handle closed indices, since they are not allocated on a node once they are closed // so applyDeletedIndices might not take them into account for (IndexService indexService : indicesService) { String index = indexService.index().getName(); IndexMetaData indexMetaData = event.state().metaData().index(index); if (indexMetaData != null && indexMetaData.state() == IndexMetaData.State.CLOSE) { for (Integer shardId : indexService.shardIds()) { logger.debug("[{}][{}] removing shard (index is closed)", index, shardId); try { indexService.removeShard(shardId, "removing shard (index is closed)"); } catch (Throwable e) { logger.warn("[{}] failed to remove shard (index is closed)", e, index); } } } } for (IndexService indexService : indicesService) { String index = indexService.index().getName(); if (indexService.shardIds().isEmpty()) { if (logger.isDebugEnabled()) { logger.debug("[{}] cleaning index (no shards allocated)", index); } // clean the index removeIndex(index, "removing index (no shards allocated)"); } } }
@Test @SuppressWarnings("squid:S2925") public void testThatMappingFromTemplateIsApplied() throws Exception { registry.counter(name("test", "cache-evictions")).inc(); reportAndRefresh(); // somehow the cluster state is not immediately updated... need to check Thread.sleep(200); ClusterStateResponse clusterStateResponse = client() .admin() .cluster() .prepareState() .setRoutingTable(false) .setLocal(false) .setNodes(true) .setIndices(indexWithDate) .execute() .actionGet(); org.assertj.core.api.Assertions.assertThat( clusterStateResponse.getState().getMetaData().getIndices().containsKey(indexWithDate)) .isTrue(); IndexMetaData indexMetaData = clusterStateResponse.getState().getMetaData().getIndices().get(indexWithDate); org.assertj.core.api.Assertions.assertThat(indexMetaData.getMappings().containsKey("counter")) .isTrue(); Map<String, Object> properties = getAsMap(indexMetaData.mapping("counter").sourceAsMap(), "properties"); Map<String, Object> mapping = getAsMap(properties, "name"); org.assertj.core.api.Assertions.assertThat(mapping).containsKey("index"); org.assertj.core.api.Assertions.assertThat(mapping.get("index").toString()) .isEqualTo("not_analyzed"); }
/** Initializes an index, to be restored from snapshot */ private Builder initializeAsRestore( IndexMetaData indexMetaData, RestoreSource restoreSource, boolean asNew) { if (!shards.isEmpty()) { throw new ElasticSearchIllegalStateException( "trying to initialize an index with fresh shards, but already has shards created"); } for (int shardId = 0; shardId < indexMetaData.numberOfShards(); shardId++) { IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder( new ShardId(indexMetaData.index(), shardId), asNew ? false : true); for (int i = 0; i <= indexMetaData.numberOfReplicas(); i++) { indexShardRoutingBuilder.addShard( new ImmutableShardRouting( index, shardId, null, null, i == 0 ? restoreSource : null, i == 0, ShardRoutingState.UNASSIGNED, 0)); } shards.put(shardId, indexShardRoutingBuilder.build()); } return this; }
@Override public ClusterState execute(ClusterState currentState) throws Exception { if (cancellableThreads.isCancelled() == false) { // no need to run this if recovery is canceled IndexMetaData indexMetaData = clusterService.state().metaData().getIndices().get(indexService.index().getName()); ImmutableOpenMap<String, MappingMetaData> metaDataMappings = null; if (indexMetaData != null) { metaDataMappings = indexMetaData.getMappings(); } // default mapping should not be sent back, it can only be updated by put mapping API, and // its // a full in place replace, we don't want to override a potential update coming into it for (DocumentMapper documentMapper : indexService.mapperService().docMappers(false)) { MappingMetaData mappingMetaData = metaDataMappings == null ? null : metaDataMappings.get(documentMapper.type()); if (mappingMetaData == null || !documentMapper.refreshSource().equals(mappingMetaData.source())) { // not on master yet in the right form documentMappersToUpdate.add(documentMapper); } } } return currentState; }
/** Marks index as upgraded so we don't have to test it again */ private IndexMetaData markAsUpgraded(IndexMetaData indexMetaData) { Settings settings = Settings.builder() .put(indexMetaData.getSettings()) .put(IndexMetaData.SETTING_VERSION_UPGRADED, Version.CURRENT) .build(); return IndexMetaData.builder(indexMetaData).settings(settings).build(); }
public Builder updateBlocks(IndexMetaData indexMetaData) { removeIndexBlock( indexMetaData.getIndex().getName(), MetaDataIndexStateService.INDEX_CLOSED_BLOCK); removeIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_READ_ONLY_BLOCK); removeIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_READ_BLOCK); removeIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_WRITE_BLOCK); removeIndexBlock(indexMetaData.getIndex().getName(), IndexMetaData.INDEX_METADATA_BLOCK); return addBlocks(indexMetaData); }
private void removeIndex( ClusterBlocks.Builder blocks, MetaData.Builder metaData, RoutingTable.Builder routingTable, IndexMetaData index) { metaData.remove(index.getIndex()); routingTable.remove(index.getIndex()); blocks.removeIndexBlocks(index.getIndex()); }
public Builder(IndexMetaData indexMetaData) { this(indexMetaData.index()); settings(indexMetaData.settings()); mappings.putAll(indexMetaData.mappings); aliases.putAll(indexMetaData.aliases); customs.putAll(indexMetaData.customs); this.state = indexMetaData.state; this.version = indexMetaData.version; }
@Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); fromNode.writeTo(out); out.writeVInt(indices.length); for (IndexMetaData indexMetaData : indices) { indexMetaData.writeTo(out); } }
/** * @param concreteIndex The concrete index to check if routing is required * @param type The type to check if routing is required * @return Whether routing is required according to the mapping for the specified index and type */ public boolean routingRequired(String concreteIndex, String type) { IndexMetaData indexMetaData = indices.get(concreteIndex); if (indexMetaData != null) { MappingMetaData mappingMetaData = indexMetaData.getMappings().get(type); if (mappingMetaData != null) { return mappingMetaData.routing().required(); } } return false; }
public Builder(IndexMetaData indexMetaData) { this.index = indexMetaData.getIndex().getName(); this.state = indexMetaData.state; this.version = indexMetaData.version; this.settings = indexMetaData.getSettings(); this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings); this.aliases = ImmutableOpenMap.builder(indexMetaData.aliases); this.customs = ImmutableOpenMap.builder(indexMetaData.customs); this.activeAllocationIds = ImmutableOpenIntMap.builder(indexMetaData.activeAllocationIds); }
private Decision shouldIndexFilter( IndexMetaData indexMd, RoutingNode node, RoutingAllocation allocation) { if (indexMd.requireFilters() != null) { if (!indexMd.requireFilters().match(node.node())) { return allocation.decision( Decision.NO, NAME, "node does not match index required filters [%s]", indexMd.requireFilters()); } } if (indexMd.includeFilters() != null) { if (!indexMd.includeFilters().match(node.node())) { return allocation.decision( Decision.NO, NAME, "node does not match index include filters [%s]", indexMd.includeFilters()); } } if (indexMd.excludeFilters() != null) { if (indexMd.excludeFilters().match(node.node())) { return allocation.decision( Decision.NO, NAME, "node matches index exclude filters [%s]", indexMd.excludeFilters()); } } return null; }
private Decision shouldIndexFilter( IndexMetaData indexMd, RoutingNode node, RoutingAllocation allocation) { if (indexMd.requireFilters() != null) { if (!indexMd.requireFilters().match(node.node())) { return allocation.decision( Decision.NO, NAME, "node does not match index setting [%s] filters [%s]", IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_PREFIX, indexMd.requireFilters()); } } if (indexMd.includeFilters() != null) { if (!indexMd.includeFilters().match(node.node())) { return allocation.decision( Decision.NO, NAME, "node does not match index setting [%s] filters [%s]", IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_PREFIX, indexMd.includeFilters()); } } if (indexMd.excludeFilters() != null) { if (indexMd.excludeFilters().match(node.node())) { return allocation.decision( Decision.NO, NAME, "node matches index setting [%s] filters [%s]", IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING.getKey(), indexMd.excludeFilters()); } } return null; }
public void testCloseIndexAcknowledgement() { createIndex("test"); ensureGreen(); assertAcked(client().admin().indices().prepareClose("test")); for (Client client : clients()) { IndexMetaData indexMetaData = getLocalClusterState(client).metaData().indices().get("test"); assertThat(indexMetaData.getState(), equalTo(State.CLOSE)); } }
/** * Elasticsearch 3.0 no longer supports indices with pre Lucene v5.0 (Elasticsearch v2.0.0.beta1) * segments. All indices that were created before Elasticsearch v2.0.0.beta1 should be upgraded * using upgrade API before they can be open by this version of elasticsearch. */ private void checkSupportedVersion(IndexMetaData indexMetaData) { if (indexMetaData.getState() == IndexMetaData.State.OPEN && isSupportedVersion(indexMetaData) == false) { throw new IllegalStateException( "The index [" + indexMetaData.getIndex() + "] was created before v2.0.0.beta1 and wasn't upgraded." + " This index should be open using a version before " + Version.CURRENT.minimumCompatibilityVersion() + " and upgraded using the upgrade API."); } }
public Builder put(IndexMetaData indexMetaData, boolean incrementVersion) { if (indices.get(indexMetaData.getIndex().getName()) == indexMetaData) { return this; } // if we put a new index metadata, increment its version if (incrementVersion) { indexMetaData = IndexMetaData.builder(indexMetaData).version(indexMetaData.getVersion() + 1).build(); } indices.put(indexMetaData.getIndex().getName(), indexMetaData); return this; }
private void updateMappingOnMaster(final String index, final String type) { try { MapperService mapperService = indicesService.indexServiceSafe(index).mapperService(); final DocumentMapper documentMapper = mapperService.documentMapper(type); if (documentMapper == null) { // should not happen return; } IndexMetaData metaData = clusterService.state().metaData().index(index); if (metaData == null) { return; } long orderId = mappingUpdatedAction.generateNextMappingUpdateOrder(); documentMapper.refreshSource(); DiscoveryNode node = clusterService.localNode(); final MappingUpdatedAction.MappingUpdatedRequest request = new MappingUpdatedAction.MappingUpdatedRequest( index, metaData.uuid(), type, documentMapper.mappingSource(), orderId, node != null ? node.id() : null); mappingUpdatedAction.execute( request, new ActionListener<MappingUpdatedAction.MappingUpdatedResponse>() { @Override public void onResponse( MappingUpdatedAction.MappingUpdatedResponse mappingUpdatedResponse) { // all is well } @Override public void onFailure(Throwable e) { try { logger.warn( "failed to update master on updated mapping for index [{}], type [{}] and source [{}]", e, index, type, documentMapper.mappingSource().string()); } catch (IOException e1) { // ignore } } }); } catch (Exception e) { logger.warn( "failed to update master on updated mapping for index [{}], type [{}]", e, index, type); } }
public boolean equalsAliases(MetaData other) { for (ObjectCursor<IndexMetaData> cursor : other.indices().values()) { IndexMetaData otherIndex = cursor.value; IndexMetaData thisIndex = index(otherIndex.getIndex()); if (thisIndex == null) { return false; } if (otherIndex.getAliases().equals(thisIndex.getAliases()) == false) { return false; } } return true; }
/* * Returns true if this index can be supported by the current version of elasticsearch */ private static boolean isSupportedVersion(IndexMetaData indexMetaData) { if (indexMetaData.getCreationVersion().onOrAfter(Version.V_2_0_0_beta1)) { // The index was created with elasticsearch that was using Lucene 5.2.1 return true; } if (indexMetaData.getMinimumCompatibleVersion() != null && indexMetaData .getMinimumCompatibleVersion() .onOrAfter(org.apache.lucene.util.Version.LUCENE_5_0_0)) { // The index was upgraded we can work with it return true; } return false; }
private void writeIndex( String reason, IndexMetaData indexMetaData, @Nullable IndexMetaData previousIndexMetaData) throws Exception { logger.trace("[{}] writing state, reason [{}]", indexMetaData.index(), reason); XContentBuilder builder = XContentFactory.contentBuilder(format, new BytesStreamOutput()); builder.startObject(); IndexMetaData.Builder.toXContent(indexMetaData, builder, formatParams); builder.endObject(); builder.flush(); String stateFileName = "state-" + indexMetaData.version(); Exception lastFailure = null; boolean wroteAtLeastOnce = false; for (File indexLocation : nodeEnv.indexLocations(new Index(indexMetaData.index()))) { File stateLocation = new File(indexLocation, "_state"); FileSystemUtils.mkdirs(stateLocation); File stateFile = new File(stateLocation, stateFileName); FileOutputStream fos = null; try { fos = new FileOutputStream(stateFile); BytesReference bytes = builder.bytes(); fos.write(bytes.array(), bytes.arrayOffset(), bytes.length()); fos.getChannel().force(true); fos.close(); wroteAtLeastOnce = true; } catch (Exception e) { lastFailure = e; } finally { IOUtils.closeWhileHandlingException(fos); } } if (!wroteAtLeastOnce) { logger.warn("[{}]: failed to state", lastFailure, indexMetaData.index()); throw new IOException( "failed to write state for [" + indexMetaData.index() + "]", lastFailure); } // delete the old files if (previousIndexMetaData != null && previousIndexMetaData.version() != indexMetaData.version()) { for (File indexLocation : nodeEnv.indexLocations(new Index(indexMetaData.index()))) { File[] files = new File(indexLocation, "_state").listFiles(); if (files == null) { continue; } for (File file : files) { if (!file.getName().startsWith("state-")) { continue; } if (file.getName().equals(stateFileName)) { continue; } file.delete(); } } } }
public Builder updateSettings(Settings settings, String... indices) { if (indices == null || indices.length == 0) { indices = this.indices.keys().toArray(String.class); } for (String index : indices) { IndexMetaData indexMetaData = this.indices.get(index); if (indexMetaData == null) { throw new IndexNotFoundException(index); } put( IndexMetaData.builder(indexMetaData) .settings(settingsBuilder().put(indexMetaData.getSettings()).put(settings))); } return this; }
@Test public void testRoutingTableSerialization() throws Exception { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").numberOfShards(10).numberOfReplicas(1)) .build(); RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build(); DiscoveryNodes nodes = DiscoveryNodes.builder() .put(newNode("node1")) .put(newNode("node2")) .put(newNode("node3")) .build(); ClusterState clusterState = ClusterState.builder().nodes(nodes).metaData(metaData).routingTable(routingTable).build(); AllocationService strategy = createAllocationService(); RoutingTable source = strategy.reroute(clusterState).routingTable(); BytesStreamOutput outStream = new BytesStreamOutput(); RoutingTable.Builder.writeTo(source, outStream); BytesStreamInput inStream = new BytesStreamInput(outStream.bytes().toBytes(), false); RoutingTable target = RoutingTable.Builder.readFrom(inStream); assertThat(target.prettyPrint(), equalTo(source.prettyPrint())); }
@Override public void onRecoveryDone(RecoveryState state) { shardStateAction.shardStarted( shardRouting, indexMetaData.getIndexUUID(), "after recovery (replica) from node [" + state.getSourceNode() + "]"); }
@Test public void testClusterStateSerialization() throws Exception { MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").numberOfShards(10).numberOfReplicas(1)) .build(); RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build(); DiscoveryNodes nodes = DiscoveryNodes.builder() .put(newNode("node1")) .put(newNode("node2")) .put(newNode("node3")) .localNodeId("node1") .masterNodeId("node2") .build(); ClusterState clusterState = ClusterState.builder().nodes(nodes).metaData(metaData).routingTable(routingTable).build(); AllocationService strategy = createAllocationService(); clusterState = ClusterState.builder(clusterState) .routingTable(strategy.reroute(clusterState).routingTable()) .build(); ClusterState serializedClusterState = ClusterState.Builder.fromBytes( ClusterState.Builder.toBytes(clusterState), newNode("node1")); assertThat( serializedClusterState.routingTable().prettyPrint(), equalTo(clusterState.routingTable().prettyPrint())); }
@Override public synchronized void updateMetaData(final IndexMetaData metadata) { final Translog.Durability oldTranslogDurability = indexSettings.getTranslogDurability(); if (indexSettings.updateIndexMetaData(metadata)) { for (final IndexShard shard : this.shards.values()) { try { shard.onSettingsChanged(); } catch (Exception e) { logger.warn("[{}] failed to notify shard about setting change", e, shard.shardId().id()); } } if (refreshTask.getInterval().equals(indexSettings.getRefreshInterval()) == false) { rescheduleRefreshTasks(); } final Translog.Durability durability = indexSettings.getTranslogDurability(); if (durability != oldTranslogDurability) { rescheduleFsyncTask(durability); } } // update primary terms for (final IndexShard shard : this.shards.values()) { shard.updatePrimaryTerm(metadata.primaryTerm(shard.shardId().id())); } }