/**
* A base class for read operations that needs to be performed on the master node. Can also be
* executed on the local node if needed.
*/
public abstract class TransportMasterNodeReadAction<
Request extends MasterNodeReadRequest<Request>, Response extends ActionResponse>
extends TransportMasterNodeAction<Request, Response> {
public static final Setting<Boolean> FORCE_LOCAL_SETTING =
Setting.boolSetting("action.master.force_local", false, Property.NodeScope);
private final boolean forceLocal;
protected TransportMasterNodeReadAction(
Settings settings,
String actionName,
TransportService transportService,
ClusterService clusterService,
ThreadPool threadPool,
ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver,
Supplier<Request> request) {
super(
settings,
actionName,
transportService,
clusterService,
threadPool,
actionFilters,
indexNameExpressionResolver,
request);
this.forceLocal = FORCE_LOCAL_SETTING.get(settings);
}
@Override
protected final boolean localExecute(Request request) {
return forceLocal || request.local();
}
}
public static final class TcpSettings {
public static final Setting<Boolean> TCP_NO_DELAY =
Setting.boolSetting("network.tcp.no_delay", true, false, Setting.Scope.CLUSTER);
public static final Setting<Boolean> TCP_KEEP_ALIVE =
Setting.boolSetting("network.tcp.keep_alive", true, false, Setting.Scope.CLUSTER);
public static final Setting<Boolean> TCP_REUSE_ADDRESS =
Setting.boolSetting(
"network.tcp.reuse_address",
NetworkUtils.defaultReuseAddress(),
false,
Setting.Scope.CLUSTER);
public static final Setting<ByteSizeValue> TCP_SEND_BUFFER_SIZE =
Setting.byteSizeSetting(
"network.tcp.send_buffer_size", new ByteSizeValue(-1), false, Setting.Scope.CLUSTER);
public static final Setting<ByteSizeValue> TCP_RECEIVE_BUFFER_SIZE =
Setting.byteSizeSetting(
"network.tcp.receive_buffer_size", new ByteSizeValue(-1), false, Setting.Scope.CLUSTER);
public static final Setting<Boolean> TCP_BLOCKING =
Setting.boolSetting("network.tcp.blocking", false, false, Setting.Scope.CLUSTER);
public static final Setting<Boolean> TCP_BLOCKING_SERVER =
Setting.boolSetting(
"network.tcp.blocking_server", TCP_BLOCKING, false, Setting.Scope.CLUSTER);
public static final Setting<Boolean> TCP_BLOCKING_CLIENT =
Setting.boolSetting(
"network.tcp.blocking_client", TCP_BLOCKING, false, Setting.Scope.CLUSTER);
public static final Setting<TimeValue> TCP_CONNECT_TIMEOUT =
Setting.timeSetting(
"network.tcp.connect_timeout",
new TimeValue(30, TimeUnit.SECONDS),
false,
Setting.Scope.CLUSTER);
}
public class PercolatorFieldMapper extends FieldMapper {
public static final XContentType QUERY_BUILDER_CONTENT_TYPE = XContentType.SMILE;
public static final Setting<Boolean> INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING =
Setting.boolSetting(
"index.percolator.map_unmapped_fields_as_string", false, Setting.Property.IndexScope);
public static final String CONTENT_TYPE = "percolator";
private static final FieldType FIELD_TYPE = new FieldType();
static final byte FIELD_VALUE_SEPARATOR = 0; // nul code point
static final String EXTRACTION_COMPLETE = "complete";
static final String EXTRACTION_PARTIAL = "partial";
static final String EXTRACTION_FAILED = "failed";
public static final String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms";
public static final String EXTRACTION_RESULT_FIELD_NAME = "extraction_result";
public static final String QUERY_BUILDER_FIELD_NAME = "query_builder_field";
public static class Builder extends FieldMapper.Builder<Builder, PercolatorFieldMapper> {
private final QueryShardContext queryShardContext;
public Builder(String fieldName, QueryShardContext queryShardContext) {
super(fieldName, FIELD_TYPE, FIELD_TYPE);
this.queryShardContext = queryShardContext;
}
@Override
public PercolatorFieldMapper build(BuilderContext context) {
context.path().add(name());
FieldType fieldType = (FieldType) this.fieldType;
KeywordFieldMapper extractedTermsField =
createExtractQueryFieldBuilder(EXTRACTED_TERMS_FIELD_NAME, context);
fieldType.queryTermsField = extractedTermsField.fieldType();
KeywordFieldMapper extractionResultField =
createExtractQueryFieldBuilder(EXTRACTION_RESULT_FIELD_NAME, context);
fieldType.extractionResultField = extractionResultField.fieldType();
BinaryFieldMapper queryBuilderField = createQueryBuilderFieldBuilder(context);
fieldType.queryBuilderField = queryBuilderField.fieldType();
context.path().remove();
setupFieldType(context);
return new PercolatorFieldMapper(
name(),
fieldType,
defaultFieldType,
context.indexSettings(),
multiFieldsBuilder.build(this, context),
copyTo,
queryShardContext,
extractedTermsField,
extractionResultField,
queryBuilderField);
}
static KeywordFieldMapper createExtractQueryFieldBuilder(String name, BuilderContext context) {
KeywordFieldMapper.Builder queryMetaDataFieldBuilder = new KeywordFieldMapper.Builder(name);
queryMetaDataFieldBuilder.docValues(false);
queryMetaDataFieldBuilder.store(false);
queryMetaDataFieldBuilder.indexOptions(IndexOptions.DOCS);
return queryMetaDataFieldBuilder.build(context);
}
static BinaryFieldMapper createQueryBuilderFieldBuilder(BuilderContext context) {
BinaryFieldMapper.Builder builder = new BinaryFieldMapper.Builder(QUERY_BUILDER_FIELD_NAME);
builder.docValues(true);
builder.indexOptions(IndexOptions.NONE);
builder.store(false);
builder.fieldType().setDocValuesType(DocValuesType.BINARY);
return builder.build(context);
}
}
public static class TypeParser implements FieldMapper.TypeParser {
@Override
public Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
return new Builder(name, parserContext.queryShardContext());
}
}
public static class FieldType extends MappedFieldType {
MappedFieldType queryTermsField;
MappedFieldType extractionResultField;
MappedFieldType queryBuilderField;
public FieldType() {
setIndexOptions(IndexOptions.NONE);
setDocValuesType(DocValuesType.NONE);
setStored(false);
}
public FieldType(FieldType ref) {
super(ref);
queryTermsField = ref.queryTermsField;
extractionResultField = ref.extractionResultField;
queryBuilderField = ref.queryBuilderField;
}
@Override
public MappedFieldType clone() {
return new FieldType(this);
}
@Override
public String typeName() {
return CONTENT_TYPE;
}
@Override
public Query termQuery(Object value, QueryShardContext context) {
throw new QueryShardException(
context, "Percolator fields are not searchable directly, use a percolate query instead");
}
public Query percolateQuery(
String documentType,
PercolateQuery.QueryStore queryStore,
BytesReference documentSource,
IndexSearcher searcher)
throws IOException {
IndexReader indexReader = searcher.getIndexReader();
Query candidateMatchesQuery = createCandidateQuery(indexReader);
Query verifiedMatchesQuery;
// We can only skip the MemoryIndex verification when percolating a single document.
// When the document being percolated contains a nested object field then the MemoryIndex
// contains multiple
// documents. In this case the term query that indicates whether memory index verification can
// be skipped
// can incorrectly indicate that non nested queries would match, while their nested variants
// would not.
if (indexReader.maxDoc() == 1) {
verifiedMatchesQuery =
new TermQuery(new Term(extractionResultField.name(), EXTRACTION_COMPLETE));
} else {
verifiedMatchesQuery = new MatchNoDocsQuery("nested docs, so no verified matches");
}
return new PercolateQuery(
documentType,
queryStore,
documentSource,
candidateMatchesQuery,
searcher,
verifiedMatchesQuery);
}
Query createCandidateQuery(IndexReader indexReader) throws IOException {
List<Term> extractedTerms = new ArrayList<>();
// include extractionResultField:failed, because docs with this term have no
// extractedTermsField
// and otherwise we would fail to return these docs. Docs that failed query term extraction
// always need to be verified by MemoryIndex:
extractedTerms.add(new Term(extractionResultField.name(), EXTRACTION_FAILED));
LeafReader reader = indexReader.leaves().get(0).reader();
Fields fields = reader.fields();
for (String field : fields) {
Terms terms = fields.terms(field);
if (terms == null) {
continue;
}
BytesRef fieldBr = new BytesRef(field);
TermsEnum tenum = terms.iterator();
for (BytesRef term = tenum.next(); term != null; term = tenum.next()) {
BytesRefBuilder builder = new BytesRefBuilder();
builder.append(fieldBr);
builder.append(FIELD_VALUE_SEPARATOR);
builder.append(term);
extractedTerms.add(new Term(queryTermsField.name(), builder.toBytesRef()));
}
}
return new TermsQuery(extractedTerms);
}
}
private final boolean mapUnmappedFieldAsString;
private final QueryShardContext queryShardContext;
private KeywordFieldMapper queryTermsField;
private KeywordFieldMapper extractionResultField;
private BinaryFieldMapper queryBuilderField;
public PercolatorFieldMapper(
String simpleName,
MappedFieldType fieldType,
MappedFieldType defaultFieldType,
Settings indexSettings,
MultiFields multiFields,
CopyTo copyTo,
QueryShardContext queryShardContext,
KeywordFieldMapper queryTermsField,
KeywordFieldMapper extractionResultField,
BinaryFieldMapper queryBuilderField) {
super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
this.queryShardContext = queryShardContext;
this.queryTermsField = queryTermsField;
this.extractionResultField = extractionResultField;
this.queryBuilderField = queryBuilderField;
this.mapUnmappedFieldAsString = INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING.get(indexSettings);
}
@Override
public FieldMapper updateFieldType(Map<String, MappedFieldType> fullNameToFieldType) {
PercolatorFieldMapper updated =
(PercolatorFieldMapper) super.updateFieldType(fullNameToFieldType);
KeywordFieldMapper queryTermsUpdated =
(KeywordFieldMapper) queryTermsField.updateFieldType(fullNameToFieldType);
KeywordFieldMapper extractionResultUpdated =
(KeywordFieldMapper) extractionResultField.updateFieldType(fullNameToFieldType);
BinaryFieldMapper queryBuilderUpdated =
(BinaryFieldMapper) queryBuilderField.updateFieldType(fullNameToFieldType);
if (updated == this
&& queryTermsUpdated == queryTermsField
&& extractionResultUpdated == extractionResultField
&& queryBuilderUpdated == queryBuilderField) {
return this;
}
if (updated == this) {
updated = (PercolatorFieldMapper) updated.clone();
}
updated.queryTermsField = queryTermsUpdated;
updated.extractionResultField = extractionResultUpdated;
updated.queryBuilderField = queryBuilderUpdated;
return updated;
}
@Override
public Mapper parse(ParseContext context) throws IOException {
QueryShardContext queryShardContext = new QueryShardContext(this.queryShardContext);
if (context.doc().getField(queryBuilderField.name()) != null) {
// If a percolator query has been defined in an array object then multiple percolator queries
// could be provided. In order to prevent this we fail if we try to parse more than one query
// for the current document.
throw new IllegalArgumentException("a document can only contain one percolator query");
}
XContentParser parser = context.parser();
QueryBuilder queryBuilder =
parseQueryBuilder(queryShardContext.newParseContext(parser), parser.getTokenLocation());
verifyQuery(queryBuilder);
// Fetching of terms, shapes and indexed scripts happen during this rewrite:
queryBuilder = queryBuilder.rewrite(queryShardContext);
try (XContentBuilder builder = XContentFactory.contentBuilder(QUERY_BUILDER_CONTENT_TYPE)) {
queryBuilder.toXContent(builder, new MapParams(Collections.emptyMap()));
builder.flush();
byte[] queryBuilderAsBytes = BytesReference.toBytes(builder.bytes());
context
.doc()
.add(
new Field(
queryBuilderField.name(), queryBuilderAsBytes, queryBuilderField.fieldType()));
}
Query query = toQuery(queryShardContext, mapUnmappedFieldAsString, queryBuilder);
processQuery(query, context);
return null;
}
void processQuery(Query query, ParseContext context) {
ParseContext.Document doc = context.doc();
FieldType pft = (FieldType) this.fieldType();
QueryAnalyzer.Result result;
try {
result = QueryAnalyzer.analyze(query);
} catch (QueryAnalyzer.UnsupportedQueryException e) {
doc.add(
new Field(
pft.extractionResultField.name(),
EXTRACTION_FAILED,
extractionResultField.fieldType()));
return;
}
for (Term term : result.terms) {
BytesRefBuilder builder = new BytesRefBuilder();
builder.append(new BytesRef(term.field()));
builder.append(FIELD_VALUE_SEPARATOR);
builder.append(term.bytes());
doc.add(new Field(queryTermsField.name(), builder.toBytesRef(), queryTermsField.fieldType()));
}
if (result.verified) {
doc.add(
new Field(
extractionResultField.name(),
EXTRACTION_COMPLETE,
extractionResultField.fieldType()));
} else {
doc.add(
new Field(
extractionResultField.name(), EXTRACTION_PARTIAL, extractionResultField.fieldType()));
}
}
public static Query parseQuery(
QueryShardContext context, boolean mapUnmappedFieldsAsString, XContentParser parser)
throws IOException {
return parseQuery(context, mapUnmappedFieldsAsString, context.newParseContext(parser), parser);
}
public static Query parseQuery(
QueryShardContext context,
boolean mapUnmappedFieldsAsString,
QueryParseContext queryParseContext,
XContentParser parser)
throws IOException {
return toQuery(
context,
mapUnmappedFieldsAsString,
parseQueryBuilder(queryParseContext, parser.getTokenLocation()));
}
static Query toQuery(
QueryShardContext context, boolean mapUnmappedFieldsAsString, QueryBuilder queryBuilder)
throws IOException {
// This means that fields in the query need to exist in the mapping prior to registering this
// query
// The reason that this is required, is that if a field doesn't exist then the query assumes
// defaults, which may be undesired.
//
// Even worse when fields mentioned in percolator queries do go added to map after the queries
// have been registered
// then the percolator queries don't work as expected any more.
//
// Query parsing can't introduce new fields in mappings (which happens when registering a
// percolator query),
// because field type can't be inferred from queries (like document do) so the best option here
// is to disallow
// the usage of unmapped fields in percolator queries to avoid unexpected behaviour
//
// if index.percolator.map_unmapped_fields_as_string is set to true, query can contain unmapped
// fields which will be mapped
// as an analyzed string.
context.setAllowUnmappedFields(false);
context.setMapUnmappedFieldAsString(mapUnmappedFieldsAsString);
return queryBuilder.toQuery(context);
}
private static QueryBuilder parseQueryBuilder(
QueryParseContext context, XContentLocation location) {
try {
return context
.parseInnerQueryBuilder()
.orElseThrow(
() -> new ParsingException(location, "Failed to parse inner query, was empty"));
} catch (IOException e) {
throw new ParsingException(location, "Failed to parse", e);
}
}
@Override
public Iterator<Mapper> iterator() {
return Arrays.<Mapper>asList(queryTermsField, extractionResultField, queryBuilderField)
.iterator();
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
throw new UnsupportedOperationException("should not be invoked");
}
@Override
protected String contentType() {
return CONTENT_TYPE;
}
/**
* Fails if a percolator contains an unsupported query. The following queries are not supported:
* 1) a range query with a date range based on current time 2) a has_child query 3) a has_parent
* query
*/
static void verifyQuery(QueryBuilder queryBuilder) {
if (queryBuilder instanceof RangeQueryBuilder) {
RangeQueryBuilder rangeQueryBuilder = (RangeQueryBuilder) queryBuilder;
if (rangeQueryBuilder.from() instanceof String) {
String from = (String) rangeQueryBuilder.from();
String to = (String) rangeQueryBuilder.to();
if (from.contains("now") || to.contains("now")) {
throw new IllegalArgumentException(
"percolator queries containing time range queries based on the "
+ "current time is unsupported");
}
}
} else if (queryBuilder instanceof HasChildQueryBuilder) {
throw new IllegalArgumentException(
"the [has_child] query is unsupported inside a percolator query");
} else if (queryBuilder instanceof HasParentQueryBuilder) {
throw new IllegalArgumentException(
"the [has_parent] query is unsupported inside a percolator query");
} else if (queryBuilder instanceof BoolQueryBuilder) {
BoolQueryBuilder boolQueryBuilder = (BoolQueryBuilder) queryBuilder;
List<QueryBuilder> clauses = new ArrayList<>();
clauses.addAll(boolQueryBuilder.filter());
clauses.addAll(boolQueryBuilder.must());
clauses.addAll(boolQueryBuilder.mustNot());
clauses.addAll(boolQueryBuilder.should());
for (QueryBuilder clause : clauses) {
verifyQuery(clause);
}
} else if (queryBuilder instanceof ConstantScoreQueryBuilder) {
verifyQuery(((ConstantScoreQueryBuilder) queryBuilder).innerQuery());
} else if (queryBuilder instanceof FunctionScoreQueryBuilder) {
verifyQuery(((FunctionScoreQueryBuilder) queryBuilder).query());
} else if (queryBuilder instanceof BoostingQueryBuilder) {
verifyQuery(((BoostingQueryBuilder) queryBuilder).negativeQuery());
verifyQuery(((BoostingQueryBuilder) queryBuilder).positiveQuery());
}
}
}
/**
* The merge scheduler (<code>ConcurrentMergeScheduler</code>) controls the execution of merge
* operations once they are needed (according to the merge policy). Merges run in separate threads,
* and when the maximum number of threads is reached, further merges will wait until a merge thread
* becomes available.
*
* <p>The merge scheduler supports the following <b>dynamic</b> settings:
*
* <ul>
* <li><code>index.merge.scheduler.max_thread_count</code>:
* <p>The maximum number of threads that may be merging at once. Defaults to <code>
* Math.max(1, Math.min(4, Runtime.getRuntime().availableProcessors() / 2))</code> which works
* well for a good solid-state-disk (SSD). If your index is on spinning platter drives
* instead, decrease this to 1.
* <li><code>index.merge.scheduler.auto_throttle</code>:
* <p>If this is true (the default), then the merge scheduler will rate-limit IO (writes) for
* merges to an adaptive value depending on how many merges are requested over time. An
* application with a low indexing rate that unluckily suddenly requires a large merge will
* see that merge aggressively throttled, while an application doing heavy indexing will see
* the throttle move higher to allow merges to keep up with ongoing indexing.
* </ul>
*/
public final class MergeSchedulerConfig {
public static final Setting<Integer> MAX_THREAD_COUNT_SETTING =
new Setting<>(
"index.merge.scheduler.max_thread_count",
(s) ->
Integer.toString(
Math.max(1, Math.min(4, EsExecutors.boundedNumberOfProcessors(s) / 2))),
(s) -> Setting.parseInt(s, 1, "index.merge.scheduler.max_thread_count"),
Property.Dynamic,
Property.IndexScope);
public static final Setting<Integer> MAX_MERGE_COUNT_SETTING =
new Setting<>(
"index.merge.scheduler.max_merge_count",
(s) -> Integer.toString(MAX_THREAD_COUNT_SETTING.get(s) + 5),
(s) -> Setting.parseInt(s, 1, "index.merge.scheduler.max_merge_count"),
Property.Dynamic,
Property.IndexScope);
public static final Setting<Boolean> AUTO_THROTTLE_SETTING =
Setting.boolSetting(
"index.merge.scheduler.auto_throttle", true, Property.Dynamic, Property.IndexScope);
private volatile boolean autoThrottle;
private volatile int maxThreadCount;
private volatile int maxMergeCount;
MergeSchedulerConfig(IndexSettings indexSettings) {
maxThreadCount = indexSettings.getValue(MAX_THREAD_COUNT_SETTING);
maxMergeCount = indexSettings.getValue(MAX_MERGE_COUNT_SETTING);
this.autoThrottle = indexSettings.getValue(AUTO_THROTTLE_SETTING);
}
/**
* Returns <code>true</code> iff auto throttle is enabled.
*
* @see ConcurrentMergeScheduler#enableAutoIOThrottle()
*/
public boolean isAutoThrottle() {
return autoThrottle;
}
/** Enables / disables auto throttling on the {@link ConcurrentMergeScheduler} */
void setAutoThrottle(boolean autoThrottle) {
this.autoThrottle = autoThrottle;
}
/** Returns {@code maxThreadCount}. */
public int getMaxThreadCount() {
return maxThreadCount;
}
/** Expert: directly set the maximum number of merge threads and simultaneous merges allowed. */
void setMaxThreadCount(int maxThreadCount) {
this.maxThreadCount = maxThreadCount;
}
/** Returns {@code maxMergeCount}. */
public int getMaxMergeCount() {
return maxMergeCount;
}
/** Expert: set the maximum number of simultaneous merges allowed. */
void setMaxMergeCount(int maxMergeCount) {
this.maxMergeCount = maxMergeCount;
}
}
public class MapperService extends AbstractIndexComponent {
/** The reason why a mapping is being merged. */
public enum MergeReason {
/** Create or update a mapping. */
MAPPING_UPDATE,
/**
* Recovery of an existing mapping, for instance because of a restart, if a shard was moved to a
* different node or for administrative purposes.
*/
MAPPING_RECOVERY;
}
public static final String DEFAULT_MAPPING = "_default_";
public static final Setting<Long> INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING =
Setting.longSetting(
"index.mapping.nested_fields.limit", 50L, 0, Property.Dynamic, Property.IndexScope);
public static final Setting<Long> INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING =
Setting.longSetting(
"index.mapping.total_fields.limit", 1000L, 0, Property.Dynamic, Property.IndexScope);
public static final Setting<Long> INDEX_MAPPING_DEPTH_LIMIT_SETTING =
Setting.longSetting(
"index.mapping.depth.limit", 20L, 1, Property.Dynamic, Property.IndexScope);
public static final boolean INDEX_MAPPER_DYNAMIC_DEFAULT = true;
public static final Setting<Boolean> INDEX_MAPPER_DYNAMIC_SETTING =
Setting.boolSetting(
"index.mapper.dynamic", INDEX_MAPPER_DYNAMIC_DEFAULT, Property.IndexScope);
private static ObjectHashSet<String> META_FIELDS =
ObjectHashSet.from(
"_uid",
"_id",
"_type",
"_all",
"_parent",
"_routing",
"_index",
"_size",
"_timestamp",
"_ttl");
private final AnalysisService analysisService;
/** Will create types automatically if they do not exists in the mapping definition yet */
private final boolean dynamic;
private volatile String defaultMappingSource;
private volatile Map<String, DocumentMapper> mappers = emptyMap();
private volatile FieldTypeLookup fieldTypes;
private volatile Map<String, ObjectMapper> fullPathObjectMappers = new HashMap<>();
private boolean hasNested = false; // updated dynamically to true when a nested object is added
private final DocumentMapperParser documentParser;
private final MapperAnalyzerWrapper indexAnalyzer;
private final MapperAnalyzerWrapper searchAnalyzer;
private final MapperAnalyzerWrapper searchQuoteAnalyzer;
private volatile Map<String, MappedFieldType> unmappedFieldTypes = emptyMap();
private volatile Set<String> parentTypes = emptySet();
final MapperRegistry mapperRegistry;
public MapperService(
IndexSettings indexSettings,
AnalysisService analysisService,
SimilarityService similarityService,
MapperRegistry mapperRegistry,
Supplier<QueryShardContext> queryShardContextSupplier) {
super(indexSettings);
this.analysisService = analysisService;
this.fieldTypes = new FieldTypeLookup();
this.documentParser =
new DocumentMapperParser(
indexSettings,
this,
analysisService,
similarityService,
mapperRegistry,
queryShardContextSupplier);
this.indexAnalyzer =
new MapperAnalyzerWrapper(analysisService.defaultIndexAnalyzer(), p -> p.indexAnalyzer());
this.searchAnalyzer =
new MapperAnalyzerWrapper(analysisService.defaultSearchAnalyzer(), p -> p.searchAnalyzer());
this.searchQuoteAnalyzer =
new MapperAnalyzerWrapper(
analysisService.defaultSearchQuoteAnalyzer(), p -> p.searchQuoteAnalyzer());
this.mapperRegistry = mapperRegistry;
this.dynamic = this.indexSettings.getValue(INDEX_MAPPER_DYNAMIC_SETTING);
if (index().getName().equals(ScriptService.SCRIPT_INDEX)) {
defaultMappingSource =
"{"
+ "\"_default_\": {"
+ "\"properties\": {"
+ "\"script\": { \"enabled\": false },"
+ "\"template\": { \"enabled\": false }"
+ "}"
+ "}"
+ "}";
} else {
defaultMappingSource = "{\"_default_\":{}}";
}
if (logger.isTraceEnabled()) {
logger.trace("using dynamic[{}], default mapping source[{}]", dynamic, defaultMappingSource);
} else if (logger.isDebugEnabled()) {
logger.debug("using dynamic[{}]", dynamic);
}
}
public boolean hasNested() {
return this.hasNested;
}
/**
* returns an immutable iterator over current document mappers.
*
* @param includingDefaultMapping indicates whether the iterator should contain the {@link
* #DEFAULT_MAPPING} document mapper. As is this not really an active type, you would
* typically set this to false
*/
public Iterable<DocumentMapper> docMappers(final boolean includingDefaultMapping) {
return () -> {
final Collection<DocumentMapper> documentMappers;
if (includingDefaultMapping) {
documentMappers = mappers.values();
} else {
documentMappers =
mappers
.values()
.stream()
.filter(mapper -> !DEFAULT_MAPPING.equals(mapper.type()))
.collect(Collectors.toList());
}
return Collections.unmodifiableCollection(documentMappers).iterator();
};
}
public AnalysisService analysisService() {
return this.analysisService;
}
public DocumentMapperParser documentMapperParser() {
return this.documentParser;
}
public DocumentMapper merge(
String type, CompressedXContent mappingSource, MergeReason reason, boolean updateAllTypes) {
if (DEFAULT_MAPPING.equals(type)) {
// verify we can parse it
// NOTE: never apply the default here
DocumentMapper mapper = documentParser.parse(type, mappingSource);
// still add it as a document mapper so we have it registered and, for example, persisted back
// into
// the cluster meta data if needed, or checked for existence
synchronized (this) {
mappers = newMapBuilder(mappers).put(type, mapper).map();
}
try {
defaultMappingSource = mappingSource.string();
} catch (IOException e) {
throw new ElasticsearchGenerationException("failed to un-compress", e);
}
return mapper;
} else {
synchronized (this) {
final boolean applyDefault =
// the default was already applied if we are recovering
reason != MergeReason.MAPPING_RECOVERY
// only apply the default mapping if we don't have the type yet
&& mappers.containsKey(type) == false;
DocumentMapper mergeWith = parse(type, mappingSource, applyDefault);
return merge(mergeWith, reason, updateAllTypes);
}
}
}
private synchronized DocumentMapper merge(
DocumentMapper mapper, MergeReason reason, boolean updateAllTypes) {
if (mapper.type().length() == 0) {
throw new InvalidTypeNameException("mapping type name is empty");
}
if (mapper.type().length() > 255) {
throw new InvalidTypeNameException(
"mapping type name ["
+ mapper.type()
+ "] is too long; limit is length 255 but was ["
+ mapper.type().length()
+ "]");
}
if (mapper.type().charAt(0) == '_') {
throw new InvalidTypeNameException(
"mapping type name [" + mapper.type() + "] can't start with '_'");
}
if (mapper.type().contains("#")) {
throw new InvalidTypeNameException(
"mapping type name [" + mapper.type() + "] should not include '#' in it");
}
if (mapper.type().contains(",")) {
throw new InvalidTypeNameException(
"mapping type name [" + mapper.type() + "] should not include ',' in it");
}
if (mapper.type().equals(mapper.parentFieldMapper().type())) {
throw new IllegalArgumentException("The [_parent.type] option can't point to the same type");
}
if (typeNameStartsWithIllegalDot(mapper)) {
throw new IllegalArgumentException(
"mapping type name [" + mapper.type() + "] must not start with a '.'");
}
// 1. compute the merged DocumentMapper
DocumentMapper oldMapper = mappers.get(mapper.type());
DocumentMapper newMapper;
if (oldMapper != null) {
newMapper = oldMapper.merge(mapper.mapping(), updateAllTypes);
} else {
newMapper = mapper;
}
// 2. check basic sanity of the new mapping
List<ObjectMapper> objectMappers = new ArrayList<>();
List<FieldMapper> fieldMappers = new ArrayList<>();
Collections.addAll(fieldMappers, newMapper.mapping().metadataMappers);
MapperUtils.collect(newMapper.mapping().root(), objectMappers, fieldMappers);
checkFieldUniqueness(newMapper.type(), objectMappers, fieldMappers);
checkObjectsCompatibility(newMapper.type(), objectMappers, fieldMappers, updateAllTypes);
// 3. update lookup data-structures
// this will in particular make sure that the merged fields are compatible with other types
FieldTypeLookup fieldTypes =
this.fieldTypes.copyAndAddAll(newMapper.type(), fieldMappers, updateAllTypes);
boolean hasNested = this.hasNested;
Map<String, ObjectMapper> fullPathObjectMappers = new HashMap<>(this.fullPathObjectMappers);
for (ObjectMapper objectMapper : objectMappers) {
fullPathObjectMappers.put(objectMapper.fullPath(), objectMapper);
if (objectMapper.nested().isNested()) {
hasNested = true;
}
}
fullPathObjectMappers = Collections.unmodifiableMap(fullPathObjectMappers);
if (reason == MergeReason.MAPPING_UPDATE) {
// this check will only be performed on the master node when there is
// a call to the update mapping API. For all other cases like
// the master node restoring mappings from disk or data nodes
// deserializing cluster state that was sent by the master node,
// this check will be skipped.
checkNestedFieldsLimit(fullPathObjectMappers);
checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size());
checkDepthLimit(fullPathObjectMappers.keySet());
checkPercolatorFieldLimit(fieldTypes);
}
Set<String> parentTypes = this.parentTypes;
if (oldMapper == null && newMapper.parentFieldMapper().active()) {
parentTypes = new HashSet<>(parentTypes.size() + 1);
parentTypes.addAll(this.parentTypes);
parentTypes.add(mapper.parentFieldMapper().type());
parentTypes = Collections.unmodifiableSet(parentTypes);
}
Map<String, DocumentMapper> mappers = new HashMap<>(this.mappers);
mappers.put(newMapper.type(), newMapper);
for (Map.Entry<String, DocumentMapper> entry : mappers.entrySet()) {
if (entry.getKey().equals(DEFAULT_MAPPING)) {
continue;
}
DocumentMapper m = entry.getValue();
// apply changes to the field types back
m = m.updateFieldType(fieldTypes.fullNameToFieldType);
entry.setValue(m);
}
mappers = Collections.unmodifiableMap(mappers);
// 4. commit the change
this.mappers = mappers;
this.fieldTypes = fieldTypes;
this.hasNested = hasNested;
this.fullPathObjectMappers = fullPathObjectMappers;
this.parentTypes = parentTypes;
assert assertSerialization(newMapper);
assert assertMappersShareSameFieldType();
return newMapper;
}
private boolean assertMappersShareSameFieldType() {
for (DocumentMapper mapper : docMappers(false)) {
List<FieldMapper> fieldMappers = new ArrayList<>();
Collections.addAll(fieldMappers, mapper.mapping().metadataMappers);
MapperUtils.collect(mapper.root(), new ArrayList<ObjectMapper>(), fieldMappers);
for (FieldMapper fieldMapper : fieldMappers) {
assert fieldMapper.fieldType() == fieldTypes.get(fieldMapper.name()) : fieldMapper.name();
}
}
return true;
}
private boolean typeNameStartsWithIllegalDot(DocumentMapper mapper) {
boolean legacyIndex =
getIndexSettings().getIndexVersionCreated().before(Version.V_5_0_0_alpha1);
if (legacyIndex) {
return mapper.type().startsWith(".")
&& !PercolatorFieldMapper.LEGACY_TYPE_NAME.equals(mapper.type());
} else {
return mapper.type().startsWith(".");
}
}
private boolean assertSerialization(DocumentMapper mapper) {
// capture the source now, it may change due to concurrent parsing
final CompressedXContent mappingSource = mapper.mappingSource();
DocumentMapper newMapper = parse(mapper.type(), mappingSource, false);
if (newMapper.mappingSource().equals(mappingSource) == false) {
throw new IllegalStateException(
"DocumentMapper serialization result is different from source. \n--> Source ["
+ mappingSource
+ "]\n--> Result ["
+ newMapper.mappingSource()
+ "]");
}
return true;
}
private void checkFieldUniqueness(
String type, Collection<ObjectMapper> objectMappers, Collection<FieldMapper> fieldMappers) {
assert Thread.holdsLock(this);
// first check within mapping
final Set<String> objectFullNames = new HashSet<>();
for (ObjectMapper objectMapper : objectMappers) {
final String fullPath = objectMapper.fullPath();
if (objectFullNames.add(fullPath) == false) {
throw new IllegalArgumentException(
"Object mapper [" + fullPath + "] is defined twice in mapping for type [" + type + "]");
}
}
final Set<String> fieldNames = new HashSet<>();
for (FieldMapper fieldMapper : fieldMappers) {
final String name = fieldMapper.name();
if (objectFullNames.contains(name)) {
throw new IllegalArgumentException(
"Field [" + name + "] is defined both as an object and a field in [" + type + "]");
} else if (fieldNames.add(name) == false) {
throw new IllegalArgumentException(
"Field [" + name + "] is defined twice in [" + type + "]");
}
}
// then check other types
for (String fieldName : fieldNames) {
if (fullPathObjectMappers.containsKey(fieldName)) {
throw new IllegalArgumentException(
"["
+ fieldName
+ "] is defined as a field in mapping ["
+ type
+ "] but this name is already used for an object in other types");
}
}
for (String objectPath : objectFullNames) {
if (fieldTypes.get(objectPath) != null) {
throw new IllegalArgumentException(
"["
+ objectPath
+ "] is defined as an object in mapping ["
+ type
+ "] but this name is already used for a field in other types");
}
}
}
private void checkObjectsCompatibility(
String type,
Collection<ObjectMapper> objectMappers,
Collection<FieldMapper> fieldMappers,
boolean updateAllTypes) {
assert Thread.holdsLock(this);
for (ObjectMapper newObjectMapper : objectMappers) {
ObjectMapper existingObjectMapper = fullPathObjectMappers.get(newObjectMapper.fullPath());
if (existingObjectMapper != null) {
// simulate a merge and ignore the result, we are just interested
// in exceptions here
existingObjectMapper.merge(newObjectMapper, updateAllTypes);
}
}
}
private void checkNestedFieldsLimit(Map<String, ObjectMapper> fullPathObjectMappers) {
long allowedNestedFields = indexSettings.getValue(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING);
long actualNestedFields = 0;
for (ObjectMapper objectMapper : fullPathObjectMappers.values()) {
if (objectMapper.nested().isNested()) {
actualNestedFields++;
}
}
if (actualNestedFields > allowedNestedFields) {
throw new IllegalArgumentException(
"Limit of nested fields ["
+ allowedNestedFields
+ "] in index ["
+ index().getName()
+ "] has been exceeded");
}
}
private void checkTotalFieldsLimit(long totalMappers) {
long allowedTotalFields = indexSettings.getValue(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
if (allowedTotalFields < totalMappers) {
throw new IllegalArgumentException(
"Limit of total fields ["
+ allowedTotalFields
+ "] in index ["
+ index().getName()
+ "] has been exceeded");
}
}
private void checkDepthLimit(Collection<String> objectPaths) {
final long maxDepth = indexSettings.getValue(INDEX_MAPPING_DEPTH_LIMIT_SETTING);
for (String objectPath : objectPaths) {
checkDepthLimit(objectPath, maxDepth);
}
}
private void checkDepthLimit(String objectPath, long maxDepth) {
int numDots = 0;
for (int i = 0; i < objectPath.length(); ++i) {
if (objectPath.charAt(i) == '.') {
numDots += 1;
}
}
final int depth = numDots + 2;
if (depth > maxDepth) {
throw new IllegalArgumentException(
"Limit of mapping depth ["
+ maxDepth
+ "] in index ["
+ index().getName()
+ "] has been exceeded due to object field ["
+ objectPath
+ "]");
}
}
/**
* We only allow upto 1 percolator field per index.
*
* <p>Reasoning here is that the PercolatorQueryCache only supports a single document having a
* percolator query. Also specifying multiple queries per document feels like an anti pattern
*/
private void checkPercolatorFieldLimit(Iterable<MappedFieldType> fieldTypes) {
List<String> percolatorFieldTypes = new ArrayList<>();
for (MappedFieldType fieldType : fieldTypes) {
if (fieldType instanceof PercolatorFieldMapper.PercolatorFieldType) {
percolatorFieldTypes.add(fieldType.name());
}
}
if (percolatorFieldTypes.size() > 1) {
throw new IllegalArgumentException(
"Up to one percolator field type is allowed per index, "
+ "found the following percolator fields ["
+ percolatorFieldTypes
+ "]");
}
}
public DocumentMapper parse(
String mappingType, CompressedXContent mappingSource, boolean applyDefault)
throws MapperParsingException {
return documentParser.parse(
mappingType, mappingSource, applyDefault ? defaultMappingSource : null);
}
public boolean hasMapping(String mappingType) {
return mappers.containsKey(mappingType);
}
/**
* Return the set of concrete types that have a mapping. NOTE: this does not return the default
* mapping.
*/
public Collection<String> types() {
final Set<String> types = new HashSet<>(mappers.keySet());
types.remove(DEFAULT_MAPPING);
return Collections.unmodifiableSet(types);
}
/**
* Return the {@link DocumentMapper} for the given type. By using the special {@value
* #DEFAULT_MAPPING} type, you can get a {@link DocumentMapper} for the default mapping.
*/
public DocumentMapper documentMapper(String type) {
return mappers.get(type);
}
/**
* Returns the document mapper created, including a mapping update if the type has been
* dynamically created.
*/
public DocumentMapperForType documentMapperWithAutoCreate(String type) {
DocumentMapper mapper = mappers.get(type);
if (mapper != null) {
return new DocumentMapperForType(mapper, null);
}
if (!dynamic) {
throw new TypeMissingException(
index(), type, "trying to auto create mapping, but dynamic mapping is disabled");
}
mapper = parse(type, null, true);
return new DocumentMapperForType(mapper, mapper.mapping());
}
/**
* Returns the {@link MappedFieldType} for the give fullName.
*
* <p>If multiple types have fields with the same full name, the first is returned.
*/
public MappedFieldType fullName(String fullName) {
return fieldTypes.get(fullName);
}
/**
* Returns all the fields that match the given pattern. If the pattern is prefixed with a type
* then the fields will be returned with a type prefix.
*/
public Collection<String> simpleMatchToIndexNames(String pattern) {
if (Regex.isSimpleMatchPattern(pattern) == false) {
// no wildcards
return Collections.singletonList(pattern);
}
return fieldTypes.simpleMatchToFullName(pattern);
}
public ObjectMapper getObjectMapper(String name) {
return fullPathObjectMappers.get(name);
}
/**
* Given a type (eg. long, string, ...), return an anonymous field mapper that can be used for
* search operations.
*/
public MappedFieldType unmappedFieldType(String type) {
if (type.equals("string")) {
deprecationLogger.deprecated(
"[unmapped_type:string] should be replaced with [unmapped_type:keyword]");
type = "keyword";
}
MappedFieldType fieldType = unmappedFieldTypes.get(type);
if (fieldType == null) {
final Mapper.TypeParser.ParserContext parserContext =
documentMapperParser().parserContext(type);
Mapper.TypeParser typeParser = parserContext.typeParser(type);
if (typeParser == null) {
throw new IllegalArgumentException("No mapper found for type [" + type + "]");
}
final Mapper.Builder<?, ?> builder =
typeParser.parse("__anonymous_" + type, emptyMap(), parserContext);
final BuilderContext builderContext =
new BuilderContext(indexSettings.getSettings(), new ContentPath(1));
fieldType = ((FieldMapper) builder.build(builderContext)).fieldType();
// There is no need to synchronize writes here. In the case of concurrent access, we could
// just
// compute some mappers several times, which is not a big deal
Map<String, MappedFieldType> newUnmappedFieldTypes = new HashMap<>();
newUnmappedFieldTypes.putAll(unmappedFieldTypes);
newUnmappedFieldTypes.put(type, fieldType);
unmappedFieldTypes = unmodifiableMap(newUnmappedFieldTypes);
}
return fieldType;
}
public Analyzer indexAnalyzer() {
return this.indexAnalyzer;
}
public Analyzer searchAnalyzer() {
return this.searchAnalyzer;
}
public Analyzer searchQuoteAnalyzer() {
return this.searchQuoteAnalyzer;
}
public Set<String> getParentTypes() {
return parentTypes;
}
/** @return Whether a field is a metadata field. */
public static boolean isMetadataField(String fieldName) {
return META_FIELDS.contains(fieldName);
}
public static String[] getAllMetaFields() {
return META_FIELDS.toArray(String.class);
}
/** An analyzer wrapper that can lookup fields within the index mappings */
final class MapperAnalyzerWrapper extends DelegatingAnalyzerWrapper {
private final Analyzer defaultAnalyzer;
private final Function<MappedFieldType, Analyzer> extractAnalyzer;
MapperAnalyzerWrapper(
Analyzer defaultAnalyzer, Function<MappedFieldType, Analyzer> extractAnalyzer) {
super(Analyzer.PER_FIELD_REUSE_STRATEGY);
this.defaultAnalyzer = defaultAnalyzer;
this.extractAnalyzer = extractAnalyzer;
}
@Override
protected Analyzer getWrappedAnalyzer(String fieldName) {
MappedFieldType fieldType = fullName(fieldName);
if (fieldType != null) {
Analyzer analyzer = extractAnalyzer.apply(fieldType);
if (analyzer != null) {
return analyzer;
}
}
return defaultAnalyzer;
}
}
}
public final class HttpTransportSettings {
public static final Setting<Boolean> SETTING_CORS_ENABLED =
Setting.boolSetting("http.cors.enabled", false, false, Scope.CLUSTER);
public static final Setting<String> SETTING_CORS_ALLOW_ORIGIN =
new Setting<String>("http.cors.allow-origin", "", (value) -> value, false, Scope.CLUSTER);
public static final Setting<Integer> SETTING_CORS_MAX_AGE =
Setting.intSetting("http.cors.max-age", 1728000, false, Scope.CLUSTER);
public static final Setting<String> SETTING_CORS_ALLOW_METHODS =
new Setting<String>(
"http.cors.allow-methods",
"OPTIONS, HEAD, GET, POST, PUT, DELETE",
(value) -> value,
false,
Scope.CLUSTER);
public static final Setting<String> SETTING_CORS_ALLOW_HEADERS =
new Setting<String>(
"http.cors.allow-headers",
"X-Requested-With, Content-Type, Content-Length",
(value) -> value,
false,
Scope.CLUSTER);
public static final Setting<Boolean> SETTING_CORS_ALLOW_CREDENTIALS =
Setting.boolSetting("http.cors.allow-credentials", false, false, Scope.CLUSTER);
public static final Setting<Boolean> SETTING_PIPELINING =
Setting.boolSetting("http.pipelining", true, false, Scope.CLUSTER);
public static final Setting<Integer> SETTING_PIPELINING_MAX_EVENTS =
Setting.intSetting("http.pipelining.max_events", 10000, false, Scope.CLUSTER);
public static final Setting<Boolean> SETTING_HTTP_COMPRESSION =
Setting.boolSetting("http.compression", false, false, Scope.CLUSTER);
public static final Setting<Integer> SETTING_HTTP_COMPRESSION_LEVEL =
Setting.intSetting("http.compression_level", 6, false, Scope.CLUSTER);
public static final Setting<List<String>> SETTING_HTTP_HOST =
listSetting("http.host", emptyList(), s -> s, false, Scope.CLUSTER);
public static final Setting<List<String>> SETTING_HTTP_PUBLISH_HOST =
listSetting("http.publish_host", SETTING_HTTP_HOST, s -> s, false, Scope.CLUSTER);
public static final Setting<List<String>> SETTING_HTTP_BIND_HOST =
listSetting("http.bind_host", SETTING_HTTP_HOST, s -> s, false, Scope.CLUSTER);
public static final Setting<PortsRange> SETTING_HTTP_PORT =
new Setting<PortsRange>("http.port", "9200-9300", PortsRange::new, false, Scope.CLUSTER);
public static final Setting<Integer> SETTING_HTTP_PUBLISH_PORT =
Setting.intSetting("http.publish_port", 0, 0, false, Scope.CLUSTER);
public static final Setting<Boolean> SETTING_HTTP_DETAILED_ERRORS_ENABLED =
Setting.boolSetting("http.detailed_errors.enabled", true, false, Scope.CLUSTER);
public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_CONTENT_LENGTH =
Setting.byteSizeSetting(
"http.max_content_length", new ByteSizeValue(100, ByteSizeUnit.MB), false, Scope.CLUSTER);
public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_CHUNK_SIZE =
Setting.byteSizeSetting(
"http.max_chunk_size", new ByteSizeValue(8, ByteSizeUnit.KB), false, Scope.CLUSTER);
public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_HEADER_SIZE =
Setting.byteSizeSetting(
"http.max_header_size", new ByteSizeValue(8, ByteSizeUnit.KB), false, Scope.CLUSTER);
public static final Setting<ByteSizeValue> SETTING_HTTP_MAX_INITIAL_LINE_LENGTH =
Setting.byteSizeSetting(
"http.max_initial_line_length",
new ByteSizeValue(4, ByteSizeUnit.KB),
false,
Scope.CLUSTER);
// don't reset cookies by default, since I don't think we really need to
// note, parsing cookies was fixed in netty 3.5.1 regarding stack allocation, but still,
// currently, we don't need cookies
public static final Setting<Boolean> SETTING_HTTP_RESET_COOKIES =
Setting.boolSetting("http.reset_cookies", false, false, Scope.CLUSTER);
private HttpTransportSettings() {}
}
/**
* A base class for {@link org.elasticsearch.discovery.zen.fd.MasterFaultDetection} & {@link
* org.elasticsearch.discovery.zen.fd.NodesFaultDetection}, making sure both use the same setting.
*/
public abstract class FaultDetection extends AbstractComponent {
public static final Setting<Boolean> CONNECT_ON_NETWORK_DISCONNECT_SETTING =
Setting.boolSetting(
"discovery.zen.fd.connect_on_network_disconnect", false, Property.NodeScope);
public static final Setting<TimeValue> PING_INTERVAL_SETTING =
Setting.positiveTimeSetting(
"discovery.zen.fd.ping_interval", timeValueSeconds(1), Property.NodeScope);
public static final Setting<TimeValue> PING_TIMEOUT_SETTING =
Setting.timeSetting(
"discovery.zen.fd.ping_timeout", timeValueSeconds(30), Property.NodeScope);
public static final Setting<Integer> PING_RETRIES_SETTING =
Setting.intSetting("discovery.zen.fd.ping_retries", 3, Property.NodeScope);
public static final Setting<Boolean> REGISTER_CONNECTION_LISTENER_SETTING =
Setting.boolSetting(
"discovery.zen.fd.register_connection_listener", true, Property.NodeScope);
protected final ThreadPool threadPool;
protected final ClusterName clusterName;
protected final TransportService transportService;
// used mainly for testing, should always be true
protected final boolean registerConnectionListener;
protected final FDConnectionListener connectionListener;
protected final boolean connectOnNetworkDisconnect;
protected final TimeValue pingInterval;
protected final TimeValue pingRetryTimeout;
protected final int pingRetryCount;
public FaultDetection(
Settings settings,
ThreadPool threadPool,
TransportService transportService,
ClusterName clusterName) {
super(settings);
this.threadPool = threadPool;
this.transportService = transportService;
this.clusterName = clusterName;
this.connectOnNetworkDisconnect = CONNECT_ON_NETWORK_DISCONNECT_SETTING.get(settings);
this.pingInterval = PING_INTERVAL_SETTING.get(settings);
this.pingRetryTimeout = PING_TIMEOUT_SETTING.get(settings);
this.pingRetryCount = PING_RETRIES_SETTING.get(settings);
this.registerConnectionListener = REGISTER_CONNECTION_LISTENER_SETTING.get(settings);
this.connectionListener = new FDConnectionListener();
if (registerConnectionListener) {
transportService.addConnectionListener(connectionListener);
}
}
public void close() {
transportService.removeConnectionListener(connectionListener);
}
/**
* This method will be called when the {@link org.elasticsearch.transport.TransportService} raised
* a node disconnected event
*/
abstract void handleTransportDisconnect(DiscoveryNode node);
private class FDConnectionListener implements TransportConnectionListener {
@Override
public void onNodeConnected(DiscoveryNode node) {}
@Override
public void onNodeDisconnected(DiscoveryNode node) {
handleTransportDisconnect(node);
}
}
}
public class MetaData
implements Iterable<IndexMetaData>,
Diffable<MetaData>,
FromXContentBuilder<MetaData>,
ToXContent {
public static final MetaData PROTO = builder().build();
public static final String ALL = "_all";
public enum XContentContext {
/* Custom metadata should be returns as part of API call */
API,
/* Custom metadata should be stored as part of the persistent cluster state */
GATEWAY,
/* Custom metadata should be stored as part of a snapshot */
SNAPSHOT
}
public static EnumSet<XContentContext> API_ONLY = EnumSet.of(XContentContext.API);
public static EnumSet<XContentContext> API_AND_GATEWAY =
EnumSet.of(XContentContext.API, XContentContext.GATEWAY);
public static EnumSet<XContentContext> API_AND_SNAPSHOT =
EnumSet.of(XContentContext.API, XContentContext.SNAPSHOT);
public interface Custom extends Diffable<Custom>, ToXContent {
String type();
Custom fromXContent(XContentParser parser) throws IOException;
EnumSet<XContentContext> context();
}
public static Map<String, Custom> customPrototypes = new HashMap<>();
static {
// register non plugin custom metadata
registerPrototype(RepositoriesMetaData.TYPE, RepositoriesMetaData.PROTO);
registerPrototype(IngestMetadata.TYPE, IngestMetadata.PROTO);
}
/** Register a custom index meta data factory. Make sure to call it from a static block. */
public static void registerPrototype(String type, Custom proto) {
customPrototypes.put(type, proto);
}
@Nullable
public static <T extends Custom> T lookupPrototype(String type) {
//noinspection unchecked
return (T) customPrototypes.get(type);
}
public static <T extends Custom> T lookupPrototypeSafe(String type) {
//noinspection unchecked
T proto = (T) customPrototypes.get(type);
if (proto == null) {
throw new IllegalArgumentException(
"No custom metadata prototype registered for type ["
+ type
+ "], node likely missing plugins");
}
return proto;
}
public static final Setting<Boolean> SETTING_READ_ONLY_SETTING =
Setting.boolSetting("cluster.blocks.read_only", false, true, Setting.Scope.CLUSTER);
public static final ClusterBlock CLUSTER_READ_ONLY_BLOCK =
new ClusterBlock(
6,
"cluster read-only (api)",
false,
false,
RestStatus.FORBIDDEN,
EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA_WRITE));
public static final MetaData EMPTY_META_DATA = builder().build();
public static final String CONTEXT_MODE_PARAM = "context_mode";
public static final String CONTEXT_MODE_SNAPSHOT = XContentContext.SNAPSHOT.toString();
public static final String CONTEXT_MODE_GATEWAY = XContentContext.GATEWAY.toString();
private final String clusterUUID;
private final long version;
private final Settings transientSettings;
private final Settings persistentSettings;
private final Settings settings;
private final ImmutableOpenMap<String, IndexMetaData> indices;
private final ImmutableOpenMap<String, IndexTemplateMetaData> templates;
private final ImmutableOpenMap<String, Custom> customs;
private final transient int totalNumberOfShards; // Transient ? not serializable anyway?
private final int numberOfShards;
private final String[] allIndices;
private final String[] allOpenIndices;
private final String[] allClosedIndices;
private final SortedMap<String, AliasOrIndex> aliasAndIndexLookup;
@SuppressWarnings("unchecked")
MetaData(
String clusterUUID,
long version,
Settings transientSettings,
Settings persistentSettings,
ImmutableOpenMap<String, IndexMetaData> indices,
ImmutableOpenMap<String, IndexTemplateMetaData> templates,
ImmutableOpenMap<String, Custom> customs,
String[] allIndices,
String[] allOpenIndices,
String[] allClosedIndices,
SortedMap<String, AliasOrIndex> aliasAndIndexLookup) {
this.clusterUUID = clusterUUID;
this.version = version;
this.transientSettings = transientSettings;
this.persistentSettings = persistentSettings;
this.settings =
Settings.settingsBuilder().put(persistentSettings).put(transientSettings).build();
this.indices = indices;
this.customs = customs;
this.templates = templates;
int totalNumberOfShards = 0;
int numberOfShards = 0;
for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
totalNumberOfShards += cursor.value.getTotalNumberOfShards();
numberOfShards += cursor.value.getNumberOfShards();
}
this.totalNumberOfShards = totalNumberOfShards;
this.numberOfShards = numberOfShards;
this.allIndices = allIndices;
this.allOpenIndices = allOpenIndices;
this.allClosedIndices = allClosedIndices;
this.aliasAndIndexLookup = aliasAndIndexLookup;
}
public long version() {
return this.version;
}
public String clusterUUID() {
return this.clusterUUID;
}
/** Returns the merged transient and persistent settings. */
public Settings settings() {
return this.settings;
}
public Settings transientSettings() {
return this.transientSettings;
}
public Settings persistentSettings() {
return this.persistentSettings;
}
public boolean hasAlias(String alias) {
AliasOrIndex aliasOrIndex = getAliasAndIndexLookup().get(alias);
if (aliasOrIndex != null) {
return aliasOrIndex.isAlias();
} else {
return false;
}
}
public boolean equalsAliases(MetaData other) {
for (ObjectCursor<IndexMetaData> cursor : other.indices().values()) {
IndexMetaData otherIndex = cursor.value;
IndexMetaData thisIndex = index(otherIndex.getIndex());
if (thisIndex == null) {
return false;
}
if (otherIndex.getAliases().equals(thisIndex.getAliases()) == false) {
return false;
}
}
return true;
}
public SortedMap<String, AliasOrIndex> getAliasAndIndexLookup() {
return aliasAndIndexLookup;
}
/**
* Finds the specific index aliases that match with the specified aliases directly or partially
* via wildcards and that point to the specified concrete indices or match partially with the
* indices via wildcards.
*
* @param aliases The names of the index aliases to find
* @param concreteIndices The concrete indexes the index aliases must point to order to be
* returned.
* @return the found index aliases grouped by index
*/
public ImmutableOpenMap<String, List<AliasMetaData>> findAliases(
final String[] aliases, String[] concreteIndices) {
assert aliases != null;
assert concreteIndices != null;
if (concreteIndices.length == 0) {
return ImmutableOpenMap.of();
}
boolean matchAllAliases = matchAllAliases(aliases);
ImmutableOpenMap.Builder<String, List<AliasMetaData>> mapBuilder = ImmutableOpenMap.builder();
Iterable<String> intersection =
HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys());
for (String index : intersection) {
IndexMetaData indexMetaData = indices.get(index);
List<AliasMetaData> filteredValues = new ArrayList<>();
for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
AliasMetaData value = cursor.value;
if (matchAllAliases || Regex.simpleMatch(aliases, value.alias())) {
filteredValues.add(value);
}
}
if (!filteredValues.isEmpty()) {
// Make the list order deterministic
CollectionUtil.timSort(
filteredValues,
new Comparator<AliasMetaData>() {
@Override
public int compare(AliasMetaData o1, AliasMetaData o2) {
return o1.alias().compareTo(o2.alias());
}
});
mapBuilder.put(index, Collections.unmodifiableList(filteredValues));
}
}
return mapBuilder.build();
}
private static boolean matchAllAliases(final String[] aliases) {
for (String alias : aliases) {
if (alias.equals(ALL)) {
return true;
}
}
return aliases.length == 0;
}
/**
* Checks if at least one of the specified aliases exists in the specified concrete indices.
* Wildcards are supported in the alias names for partial matches.
*
* @param aliases The names of the index aliases to find
* @param concreteIndices The concrete indexes the index aliases must point to order to be
* returned.
* @return whether at least one of the specified aliases exists in one of the specified concrete
* indices.
*/
public boolean hasAliases(final String[] aliases, String[] concreteIndices) {
assert aliases != null;
assert concreteIndices != null;
if (concreteIndices.length == 0) {
return false;
}
Iterable<String> intersection =
HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys());
for (String index : intersection) {
IndexMetaData indexMetaData = indices.get(index);
List<AliasMetaData> filteredValues = new ArrayList<>();
for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
AliasMetaData value = cursor.value;
if (Regex.simpleMatch(aliases, value.alias())) {
filteredValues.add(value);
}
}
if (!filteredValues.isEmpty()) {
return true;
}
}
return false;
}
/*
* Finds all mappings for types and concrete indices. Types are expanded to
* include all types that match the glob patterns in the types array. Empty
* types array, null or {"_all"} will be expanded to all types available for
* the given indices.
*/
public ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> findMappings(
String[] concreteIndices, final String[] types) {
assert types != null;
assert concreteIndices != null;
if (concreteIndices.length == 0) {
return ImmutableOpenMap.of();
}
ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder =
ImmutableOpenMap.builder();
Iterable<String> intersection =
HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys());
for (String index : intersection) {
IndexMetaData indexMetaData = indices.get(index);
ImmutableOpenMap.Builder<String, MappingMetaData> filteredMappings;
if (isAllTypes(types)) {
indexMapBuilder.put(
index, indexMetaData.getMappings()); // No types specified means get it all
} else {
filteredMappings = ImmutableOpenMap.builder();
for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.getMappings()) {
if (Regex.simpleMatch(types, cursor.key)) {
filteredMappings.put(cursor.key, cursor.value);
}
}
if (!filteredMappings.isEmpty()) {
indexMapBuilder.put(index, filteredMappings.build());
}
}
}
return indexMapBuilder.build();
}
/** Returns all the concrete indices. */
public String[] concreteAllIndices() {
return allIndices;
}
public String[] getConcreteAllIndices() {
return concreteAllIndices();
}
public String[] concreteAllOpenIndices() {
return allOpenIndices;
}
public String[] getConcreteAllOpenIndices() {
return allOpenIndices;
}
public String[] concreteAllClosedIndices() {
return allClosedIndices;
}
public String[] getConcreteAllClosedIndices() {
return allClosedIndices;
}
/** Returns indexing routing for the given index. */
// TODO: This can be moved to IndexNameExpressionResolver too, but this means that we will support
// wildcards and other expressions
// in the index,bulk,update and delete apis.
public String resolveIndexRouting(
@Nullable String parent, @Nullable String routing, String aliasOrIndex) {
if (aliasOrIndex == null) {
if (routing == null) {
return parent;
}
return routing;
}
AliasOrIndex result = getAliasAndIndexLookup().get(aliasOrIndex);
if (result == null || result.isAlias() == false) {
if (routing == null) {
return parent;
}
return routing;
}
AliasOrIndex.Alias alias = (AliasOrIndex.Alias) result;
if (result.getIndices().size() > 1) {
String[] indexNames = new String[result.getIndices().size()];
int i = 0;
for (IndexMetaData indexMetaData : result.getIndices()) {
indexNames[i++] = indexMetaData.getIndex().getName();
}
throw new IllegalArgumentException(
"Alias ["
+ aliasOrIndex
+ "] has more than one index associated with it ["
+ Arrays.toString(indexNames)
+ "], can't execute a single index op");
}
AliasMetaData aliasMd = alias.getFirstAliasMetaData();
if (aliasMd.indexRouting() != null) {
if (aliasMd.indexRouting().indexOf(',') != -1) {
throw new IllegalArgumentException(
"index/alias ["
+ aliasOrIndex
+ "] provided with routing value ["
+ aliasMd.getIndexRouting()
+ "] that resolved to several routing values, rejecting operation");
}
if (routing != null) {
if (!routing.equals(aliasMd.indexRouting())) {
throw new IllegalArgumentException(
"Alias ["
+ aliasOrIndex
+ "] has index routing associated with it ["
+ aliasMd.indexRouting()
+ "], and was provided with routing value ["
+ routing
+ "], rejecting operation");
}
}
// Alias routing overrides the parent routing (if any).
return aliasMd.indexRouting();
}
if (routing == null) {
return parent;
}
return routing;
}
public boolean hasIndex(String index) {
return indices.containsKey(index);
}
public boolean hasConcreteIndex(String index) {
return getAliasAndIndexLookup().containsKey(index);
}
public IndexMetaData index(String index) {
return indices.get(index);
}
public IndexMetaData index(Index index) {
return index(index.getName());
}
public ImmutableOpenMap<String, IndexMetaData> indices() {
return this.indices;
}
public ImmutableOpenMap<String, IndexMetaData> getIndices() {
return indices();
}
public ImmutableOpenMap<String, IndexTemplateMetaData> templates() {
return this.templates;
}
public ImmutableOpenMap<String, IndexTemplateMetaData> getTemplates() {
return this.templates;
}
public ImmutableOpenMap<String, Custom> customs() {
return this.customs;
}
public ImmutableOpenMap<String, Custom> getCustoms() {
return this.customs;
}
public <T extends Custom> T custom(String type) {
return (T) customs.get(type);
}
public int totalNumberOfShards() {
return this.totalNumberOfShards;
}
public int getTotalNumberOfShards() {
return totalNumberOfShards();
}
public int numberOfShards() {
return this.numberOfShards;
}
public int getNumberOfShards() {
return numberOfShards();
}
/**
* Identifies whether the array containing type names given as argument refers to all types The
* empty or null array identifies all types
*
* @param types the array containing types
* @return true if the provided array maps to all types, false otherwise
*/
public static boolean isAllTypes(String[] types) {
return types == null || types.length == 0 || isExplicitAllType(types);
}
/**
* Identifies whether the array containing type names given as argument explicitly refers to all
* types The empty or null array doesn't explicitly map to all types
*
* @param types the array containing index names
* @return true if the provided array explicitly maps to all types, false otherwise
*/
public static boolean isExplicitAllType(String[] types) {
return types != null && types.length == 1 && ALL.equals(types[0]);
}
/**
* @param concreteIndex The concrete index to check if routing is required
* @param type The type to check if routing is required
* @return Whether routing is required according to the mapping for the specified index and type
*/
public boolean routingRequired(String concreteIndex, String type) {
IndexMetaData indexMetaData = indices.get(concreteIndex);
if (indexMetaData != null) {
MappingMetaData mappingMetaData = indexMetaData.getMappings().get(type);
if (mappingMetaData != null) {
return mappingMetaData.routing().required();
}
}
return false;
}
@Override
public Iterator<IndexMetaData> iterator() {
return indices.valuesIt();
}
public static boolean isGlobalStateEquals(MetaData metaData1, MetaData metaData2) {
if (!metaData1.persistentSettings.equals(metaData2.persistentSettings)) {
return false;
}
if (!metaData1.templates.equals(metaData2.templates())) {
return false;
}
// Check if any persistent metadata needs to be saved
int customCount1 = 0;
for (ObjectObjectCursor<String, Custom> cursor : metaData1.customs) {
if (customPrototypes.get(cursor.key).context().contains(XContentContext.GATEWAY)) {
if (!cursor.value.equals(metaData2.custom(cursor.key))) return false;
customCount1++;
}
}
int customCount2 = 0;
for (ObjectObjectCursor<String, Custom> cursor : metaData2.customs) {
if (customPrototypes.get(cursor.key).context().contains(XContentContext.GATEWAY)) {
customCount2++;
}
}
if (customCount1 != customCount2) return false;
return true;
}
@Override
public Diff<MetaData> diff(MetaData previousState) {
return new MetaDataDiff(previousState, this);
}
@Override
public Diff<MetaData> readDiffFrom(StreamInput in) throws IOException {
return new MetaDataDiff(in);
}
@Override
public MetaData fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher)
throws IOException {
return Builder.fromXContent(parser);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
Builder.toXContent(this, builder, params);
return builder;
}
private static class MetaDataDiff implements Diff<MetaData> {
private long version;
private String clusterUUID;
private Settings transientSettings;
private Settings persistentSettings;
private Diff<ImmutableOpenMap<String, IndexMetaData>> indices;
private Diff<ImmutableOpenMap<String, IndexTemplateMetaData>> templates;
private Diff<ImmutableOpenMap<String, Custom>> customs;
public MetaDataDiff(MetaData before, MetaData after) {
clusterUUID = after.clusterUUID;
version = after.version;
transientSettings = after.transientSettings;
persistentSettings = after.persistentSettings;
indices =
DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer());
templates =
DiffableUtils.diff(
before.templates, after.templates, DiffableUtils.getStringKeySerializer());
customs =
DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer());
}
public MetaDataDiff(StreamInput in) throws IOException {
clusterUUID = in.readString();
version = in.readLong();
transientSettings = Settings.readSettingsFromStream(in);
persistentSettings = Settings.readSettingsFromStream(in);
indices =
DiffableUtils.readImmutableOpenMapDiff(
in, DiffableUtils.getStringKeySerializer(), IndexMetaData.PROTO);
templates =
DiffableUtils.readImmutableOpenMapDiff(
in, DiffableUtils.getStringKeySerializer(), IndexTemplateMetaData.PROTO);
customs =
DiffableUtils.readImmutableOpenMapDiff(
in,
DiffableUtils.getStringKeySerializer(),
new DiffableUtils.DiffableValueSerializer<String, Custom>() {
@Override
public Custom read(StreamInput in, String key) throws IOException {
return lookupPrototypeSafe(key).readFrom(in);
}
@Override
public Diff<Custom> readDiff(StreamInput in, String key) throws IOException {
return lookupPrototypeSafe(key).readDiffFrom(in);
}
});
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(clusterUUID);
out.writeLong(version);
Settings.writeSettingsToStream(transientSettings, out);
Settings.writeSettingsToStream(persistentSettings, out);
indices.writeTo(out);
templates.writeTo(out);
customs.writeTo(out);
}
@Override
public MetaData apply(MetaData part) {
Builder builder = builder();
builder.clusterUUID(clusterUUID);
builder.version(version);
builder.transientSettings(transientSettings);
builder.persistentSettings(persistentSettings);
builder.indices(indices.apply(part.indices));
builder.templates(templates.apply(part.templates));
builder.customs(customs.apply(part.customs));
return builder.build();
}
}
@Override
public MetaData readFrom(StreamInput in) throws IOException {
Builder builder = new Builder();
builder.version = in.readLong();
builder.clusterUUID = in.readString();
builder.transientSettings(readSettingsFromStream(in));
builder.persistentSettings(readSettingsFromStream(in));
int size = in.readVInt();
for (int i = 0; i < size; i++) {
builder.put(IndexMetaData.Builder.readFrom(in), false);
}
size = in.readVInt();
for (int i = 0; i < size; i++) {
builder.put(IndexTemplateMetaData.Builder.readFrom(in));
}
int customSize = in.readVInt();
for (int i = 0; i < customSize; i++) {
String type = in.readString();
Custom customIndexMetaData = lookupPrototypeSafe(type).readFrom(in);
builder.putCustom(type, customIndexMetaData);
}
return builder.build();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeLong(version);
out.writeString(clusterUUID);
writeSettingsToStream(transientSettings, out);
writeSettingsToStream(persistentSettings, out);
out.writeVInt(indices.size());
for (IndexMetaData indexMetaData : this) {
indexMetaData.writeTo(out);
}
out.writeVInt(templates.size());
for (ObjectCursor<IndexTemplateMetaData> cursor : templates.values()) {
cursor.value.writeTo(out);
}
out.writeVInt(customs.size());
for (ObjectObjectCursor<String, Custom> cursor : customs) {
out.writeString(cursor.key);
cursor.value.writeTo(out);
}
}
public static Builder builder() {
return new Builder();
}
public static Builder builder(MetaData metaData) {
return new Builder(metaData);
}
/** All known byte-sized cluster settings. */
public static final Set<String> CLUSTER_BYTES_SIZE_SETTINGS =
unmodifiableSet(
newHashSet(
IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING.getKey(),
RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()));
/** All known time cluster settings. */
public static final Set<String> CLUSTER_TIME_SETTINGS =
unmodifiableSet(
newHashSet(
IndicesTTLService.INDICES_TTL_INTERVAL_SETTING.getKey(),
RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING.getKey(),
RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING.getKey(),
RecoverySettings.INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING.getKey(),
RecoverySettings.INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING.getKey(),
RecoverySettings.INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING.getKey(),
DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(),
InternalClusterInfoService.INTERNAL_CLUSTER_INFO_UPDATE_INTERVAL_SETTING.getKey(),
InternalClusterInfoService.INTERNAL_CLUSTER_INFO_TIMEOUT_SETTING.getKey(),
DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(),
InternalClusterService.CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.getKey()));
/**
* As of 2.0 we require units for time and byte-sized settings. This methods adds default units to
* any cluster settings that don't specify a unit.
*/
public static MetaData addDefaultUnitsIfNeeded(ESLogger logger, MetaData metaData) {
Settings.Builder newPersistentSettings = null;
for (Map.Entry<String, String> ent : metaData.persistentSettings().getAsMap().entrySet()) {
String settingName = ent.getKey();
String settingValue = ent.getValue();
if (CLUSTER_BYTES_SIZE_SETTINGS.contains(settingName)) {
try {
Long.parseLong(settingValue);
} catch (NumberFormatException nfe) {
continue;
}
// It's a naked number that previously would be interpreted as default unit (bytes); now we
// add it:
logger.warn(
"byte-sized cluster setting [{}] with value [{}] is missing units; assuming default units (b) but in future versions this will be a hard error",
settingName,
settingValue);
if (newPersistentSettings == null) {
newPersistentSettings = Settings.builder();
newPersistentSettings.put(metaData.persistentSettings());
}
newPersistentSettings.put(settingName, settingValue + "b");
}
if (CLUSTER_TIME_SETTINGS.contains(settingName)) {
try {
Long.parseLong(settingValue);
} catch (NumberFormatException nfe) {
continue;
}
// It's a naked number that previously would be interpreted as default unit (ms); now we add
// it:
logger.warn(
"time cluster setting [{}] with value [{}] is missing units; assuming default units (ms) but in future versions this will be a hard error",
settingName,
settingValue);
if (newPersistentSettings == null) {
newPersistentSettings = Settings.builder();
newPersistentSettings.put(metaData.persistentSettings());
}
newPersistentSettings.put(settingName, settingValue + "ms");
}
}
if (newPersistentSettings != null) {
return new MetaData(
metaData.clusterUUID(),
metaData.version(),
metaData.transientSettings(),
newPersistentSettings.build(),
metaData.getIndices(),
metaData.getTemplates(),
metaData.getCustoms(),
metaData.concreteAllIndices(),
metaData.concreteAllOpenIndices(),
metaData.concreteAllClosedIndices(),
metaData.getAliasAndIndexLookup());
} else {
// No changes:
return metaData;
}
}
public static class Builder {
private String clusterUUID;
private long version;
private Settings transientSettings = Settings.Builder.EMPTY_SETTINGS;
private Settings persistentSettings = Settings.Builder.EMPTY_SETTINGS;
private final ImmutableOpenMap.Builder<String, IndexMetaData> indices;
private final ImmutableOpenMap.Builder<String, IndexTemplateMetaData> templates;
private final ImmutableOpenMap.Builder<String, Custom> customs;
public Builder() {
clusterUUID = "_na_";
indices = ImmutableOpenMap.builder();
templates = ImmutableOpenMap.builder();
customs = ImmutableOpenMap.builder();
}
public Builder(MetaData metaData) {
this.clusterUUID = metaData.clusterUUID;
this.transientSettings = metaData.transientSettings;
this.persistentSettings = metaData.persistentSettings;
this.version = metaData.version;
this.indices = ImmutableOpenMap.builder(metaData.indices);
this.templates = ImmutableOpenMap.builder(metaData.templates);
this.customs = ImmutableOpenMap.builder(metaData.customs);
}
public Builder put(IndexMetaData.Builder indexMetaDataBuilder) {
// we know its a new one, increment the version and store
indexMetaDataBuilder.version(indexMetaDataBuilder.version() + 1);
IndexMetaData indexMetaData = indexMetaDataBuilder.build();
indices.put(indexMetaData.getIndex().getName(), indexMetaData);
return this;
}
public Builder put(IndexMetaData indexMetaData, boolean incrementVersion) {
if (indices.get(indexMetaData.getIndex().getName()) == indexMetaData) {
return this;
}
// if we put a new index metadata, increment its version
if (incrementVersion) {
indexMetaData =
IndexMetaData.builder(indexMetaData).version(indexMetaData.getVersion() + 1).build();
}
indices.put(indexMetaData.getIndex().getName(), indexMetaData);
return this;
}
public IndexMetaData get(String index) {
return indices.get(index);
}
public Builder remove(String index) {
indices.remove(index);
return this;
}
public Builder removeAllIndices() {
indices.clear();
return this;
}
public Builder indices(ImmutableOpenMap<String, IndexMetaData> indices) {
this.indices.putAll(indices);
return this;
}
public Builder put(IndexTemplateMetaData.Builder template) {
return put(template.build());
}
public Builder put(IndexTemplateMetaData template) {
templates.put(template.name(), template);
return this;
}
public Builder removeTemplate(String templateName) {
templates.remove(templateName);
return this;
}
public Builder templates(ImmutableOpenMap<String, IndexTemplateMetaData> templates) {
this.templates.putAll(templates);
return this;
}
public Custom getCustom(String type) {
return customs.get(type);
}
public Builder putCustom(String type, Custom custom) {
customs.put(type, custom);
return this;
}
public Builder removeCustom(String type) {
customs.remove(type);
return this;
}
public Builder customs(ImmutableOpenMap<String, Custom> customs) {
this.customs.putAll(customs);
return this;
}
public Builder updateSettings(Settings settings, String... indices) {
if (indices == null || indices.length == 0) {
indices = this.indices.keys().toArray(String.class);
}
for (String index : indices) {
IndexMetaData indexMetaData = this.indices.get(index);
if (indexMetaData == null) {
throw new IndexNotFoundException(index);
}
put(
IndexMetaData.builder(indexMetaData)
.settings(settingsBuilder().put(indexMetaData.getSettings()).put(settings)));
}
return this;
}
public Builder updateNumberOfReplicas(int numberOfReplicas, String... indices) {
if (indices == null || indices.length == 0) {
indices = this.indices.keys().toArray(String.class);
}
for (String index : indices) {
IndexMetaData indexMetaData = this.indices.get(index);
if (indexMetaData == null) {
throw new IndexNotFoundException(index);
}
put(IndexMetaData.builder(indexMetaData).numberOfReplicas(numberOfReplicas));
}
return this;
}
public Settings transientSettings() {
return this.transientSettings;
}
public Builder transientSettings(Settings settings) {
this.transientSettings = settings;
return this;
}
public Settings persistentSettings() {
return this.persistentSettings;
}
public Builder persistentSettings(Settings settings) {
this.persistentSettings = settings;
return this;
}
public Builder version(long version) {
this.version = version;
return this;
}
public Builder clusterUUID(String clusterUUID) {
this.clusterUUID = clusterUUID;
return this;
}
public Builder generateClusterUuidIfNeeded() {
if (clusterUUID.equals("_na_")) {
clusterUUID = Strings.randomBase64UUID();
}
return this;
}
public MetaData build() {
// TODO: We should move these datastructures to IndexNameExpressionResolver, this will give
// the following benefits:
// 1) The datastructures will only be rebuilded when needed. Now during serializing we rebuild
// these datastructures
// while these datastructures aren't even used.
// 2) The aliasAndIndexLookup can be updated instead of rebuilding it all the time.
// build all concrete indices arrays:
// TODO: I think we can remove these arrays. it isn't worth the effort, for operations on all
// indices.
// When doing an operation across all indices, most of the time is spent on actually going to
// all shards and
// do the required operations, the bottleneck isn't resolving expressions into concrete
// indices.
List<String> allIndicesLst = new ArrayList<>();
for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
allIndicesLst.add(cursor.value.getIndex().getName());
}
String[] allIndices = allIndicesLst.toArray(new String[allIndicesLst.size()]);
List<String> allOpenIndicesLst = new ArrayList<>();
List<String> allClosedIndicesLst = new ArrayList<>();
for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
IndexMetaData indexMetaData = cursor.value;
if (indexMetaData.getState() == IndexMetaData.State.OPEN) {
allOpenIndicesLst.add(indexMetaData.getIndex().getName());
} else if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
allClosedIndicesLst.add(indexMetaData.getIndex().getName());
}
}
String[] allOpenIndices = allOpenIndicesLst.toArray(new String[allOpenIndicesLst.size()]);
String[] allClosedIndices =
allClosedIndicesLst.toArray(new String[allClosedIndicesLst.size()]);
// build all indices map
SortedMap<String, AliasOrIndex> aliasAndIndexLookup = new TreeMap<>();
for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
IndexMetaData indexMetaData = cursor.value;
aliasAndIndexLookup.put(
indexMetaData.getIndex().getName(), new AliasOrIndex.Index(indexMetaData));
for (ObjectObjectCursor<String, AliasMetaData> aliasCursor : indexMetaData.getAliases()) {
AliasMetaData aliasMetaData = aliasCursor.value;
AliasOrIndex aliasOrIndex = aliasAndIndexLookup.get(aliasMetaData.getAlias());
if (aliasOrIndex == null) {
aliasOrIndex = new AliasOrIndex.Alias(aliasMetaData, indexMetaData);
aliasAndIndexLookup.put(aliasMetaData.getAlias(), aliasOrIndex);
} else if (aliasOrIndex instanceof AliasOrIndex.Alias) {
AliasOrIndex.Alias alias = (AliasOrIndex.Alias) aliasOrIndex;
alias.addIndex(indexMetaData);
} else if (aliasOrIndex instanceof AliasOrIndex.Index) {
AliasOrIndex.Index index = (AliasOrIndex.Index) aliasOrIndex;
throw new IllegalStateException(
"index and alias names need to be unique, but alias ["
+ aliasMetaData.getAlias()
+ "] and index "
+ index.getIndex().getIndex()
+ " have the same name");
} else {
throw new IllegalStateException(
"unexpected alias [" + aliasMetaData.getAlias() + "][" + aliasOrIndex + "]");
}
}
}
aliasAndIndexLookup = Collections.unmodifiableSortedMap(aliasAndIndexLookup);
return new MetaData(
clusterUUID,
version,
transientSettings,
persistentSettings,
indices.build(),
templates.build(),
customs.build(),
allIndices,
allOpenIndices,
allClosedIndices,
aliasAndIndexLookup);
}
public static String toXContent(MetaData metaData) throws IOException {
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
builder.startObject();
toXContent(metaData, builder, ToXContent.EMPTY_PARAMS);
builder.endObject();
return builder.string();
}
public static void toXContent(
MetaData metaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
XContentContext context = XContentContext.valueOf(params.param(CONTEXT_MODE_PARAM, "API"));
builder.startObject("meta-data");
builder.field("version", metaData.version());
builder.field("cluster_uuid", metaData.clusterUUID);
if (!metaData.persistentSettings().getAsMap().isEmpty()) {
builder.startObject("settings");
for (Map.Entry<String, String> entry :
metaData.persistentSettings().getAsMap().entrySet()) {
builder.field(entry.getKey(), entry.getValue());
}
builder.endObject();
}
if (context == XContentContext.API && !metaData.transientSettings().getAsMap().isEmpty()) {
builder.startObject("transient_settings");
for (Map.Entry<String, String> entry : metaData.transientSettings().getAsMap().entrySet()) {
builder.field(entry.getKey(), entry.getValue());
}
builder.endObject();
}
builder.startObject("templates");
for (ObjectCursor<IndexTemplateMetaData> cursor : metaData.templates().values()) {
IndexTemplateMetaData.Builder.toXContent(cursor.value, builder, params);
}
builder.endObject();
if (context == XContentContext.API && !metaData.indices().isEmpty()) {
builder.startObject("indices");
for (IndexMetaData indexMetaData : metaData) {
IndexMetaData.Builder.toXContent(indexMetaData, builder, params);
}
builder.endObject();
}
for (ObjectObjectCursor<String, Custom> cursor : metaData.customs()) {
Custom proto = lookupPrototypeSafe(cursor.key);
if (proto.context().contains(context)) {
builder.startObject(cursor.key);
cursor.value.toXContent(builder, params);
builder.endObject();
}
}
builder.endObject();
}
public static MetaData fromXContent(XContentParser parser) throws IOException {
Builder builder = new Builder();
// we might get here after the meta-data element, or on a fresh parser
XContentParser.Token token = parser.currentToken();
String currentFieldName = parser.currentName();
if (!"meta-data".equals(currentFieldName)) {
token = parser.nextToken();
if (token == XContentParser.Token.START_OBJECT) {
// move to the field name (meta-data)
token = parser.nextToken();
if (token != XContentParser.Token.FIELD_NAME) {
throw new IllegalArgumentException("Expected a field name but got " + token);
}
// move to the next object
token = parser.nextToken();
}
currentFieldName = parser.currentName();
}
if (!"meta-data".equals(parser.currentName())) {
throw new IllegalArgumentException(
"Expected [meta-data] as a field name but got " + currentFieldName);
}
if (token != XContentParser.Token.START_OBJECT) {
throw new IllegalArgumentException("Expected a START_OBJECT but got " + token);
}
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if ("settings".equals(currentFieldName)) {
builder.persistentSettings(
Settings.settingsBuilder()
.put(SettingsLoader.Helper.loadNestedFromMap(parser.mapOrdered()))
.build());
} else if ("indices".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
builder.put(IndexMetaData.Builder.fromXContent(parser), false);
}
} else if ("templates".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
builder.put(IndexTemplateMetaData.Builder.fromXContent(parser, parser.currentName()));
}
} else {
// check if its a custom index metadata
Custom proto = lookupPrototype(currentFieldName);
if (proto == null) {
// TODO warn
parser.skipChildren();
} else {
Custom custom = proto.fromXContent(parser);
builder.putCustom(custom.type(), custom);
}
}
} else if (token.isValue()) {
if ("version".equals(currentFieldName)) {
builder.version = parser.longValue();
} else if ("cluster_uuid".equals(currentFieldName) || "uuid".equals(currentFieldName)) {
builder.clusterUUID = parser.text();
} else {
throw new IllegalArgumentException("Unexpected field [" + currentFieldName + "]");
}
} else {
throw new IllegalArgumentException("Unexpected token " + token);
}
}
return builder.build();
}
public static MetaData readFrom(StreamInput in) throws IOException {
return PROTO.readFrom(in);
}
}
}
/**
* Serves as a node level registry for hunspell dictionaries. This services expects all dictionaries
* to be located under the {@code <path.conf>/hunspell} directory, where each locale has its
* dedicated sub-directory which holds the dictionary files. For example, the dictionary files for
* {@code en_US} locale must be placed under {@code <path.conf>/hunspell/en_US} directory.
*
* <p>The following settings can be set for each dictionary:
*
* <ul>
* <li>{@code ignore_case} - If true, dictionary matching will be case insensitive (defaults to
* {@code false})
* <li>{@code strict_affix_parsing} - Determines whether errors while reading a affix rules file
* will cause exception or simple be ignored (defaults to {@code true})
* </ul>
*
* <p>These settings can either be configured as node level configuration, such as: <br>
* <br>
*
* <pre><code>
* indices.analysis.hunspell.dictionary.en_US.ignore_case: true
* indices.analysis.hunspell.dictionary.en_US.strict_affix_parsing: false
* </code></pre>
*
* <p>or, as dedicated configuration per dictionary, placed in a {@code settings.yml} file under the
* dictionary directory. For example, the following can be the content of the {@code
* <path.config>/hunspell/en_US/settings.yml} file: <br>
* <br>
*
* <pre><code>
* ignore_case: true
* strict_affix_parsing: false
* </code></pre>
*
* @see org.elasticsearch.index.analysis.HunspellTokenFilterFactory
*/
public class HunspellService extends AbstractComponent {
public static final Setting<Boolean> HUNSPELL_LAZY_LOAD =
Setting.boolSetting(
"indices.analysis.hunspell.dictionary.lazy", Boolean.FALSE, Property.NodeScope);
public static final Setting<Boolean> HUNSPELL_IGNORE_CASE =
Setting.boolSetting(
"indices.analysis.hunspell.dictionary.ignore_case", Boolean.FALSE, Property.NodeScope);
public static final Setting<Settings> HUNSPELL_DICTIONARY_OPTIONS =
Setting.groupSetting("indices.analysis.hunspell.dictionary.", Property.NodeScope);
private final ConcurrentHashMap<String, Dictionary> dictionaries = new ConcurrentHashMap<>();
private final Map<String, Dictionary> knownDictionaries;
private final boolean defaultIgnoreCase;
private final Path hunspellDir;
private final Function<String, Dictionary> loadingFunction;
public HunspellService(
final Settings settings,
final Environment env,
final Map<String, Dictionary> knownDictionaries)
throws IOException {
super(settings);
this.knownDictionaries = Collections.unmodifiableMap(knownDictionaries);
this.hunspellDir = resolveHunspellDirectory(env);
this.defaultIgnoreCase = HUNSPELL_IGNORE_CASE.get(settings);
this.loadingFunction =
(locale) -> {
try {
return loadDictionary(locale, settings, env);
} catch (Throwable e) {
throw new IllegalStateException(
"failed to load hunspell dictionary for locale: " + locale, e);
}
};
if (!HUNSPELL_LAZY_LOAD.get(settings)) {
scanAndLoadDictionaries();
}
}
/**
* Returns the hunspell dictionary for the given locale.
*
* @param locale The name of the locale
*/
public Dictionary getDictionary(String locale) {
Dictionary dictionary = knownDictionaries.get(locale);
if (dictionary == null) {
dictionary = dictionaries.computeIfAbsent(locale, loadingFunction);
}
return dictionary;
}
private Path resolveHunspellDirectory(Environment env) {
return env.configFile().resolve("hunspell");
}
/** Scans the hunspell directory and loads all found dictionaries */
private void scanAndLoadDictionaries() throws IOException {
if (Files.isDirectory(hunspellDir)) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(hunspellDir)) {
for (Path file : stream) {
if (Files.isDirectory(file)) {
try (DirectoryStream<Path> inner =
Files.newDirectoryStream(hunspellDir.resolve(file), "*.dic")) {
if (inner.iterator().hasNext()) { // just making sure it's indeed a dictionary dir
try {
getDictionary(file.getFileName().toString());
} catch (Throwable e) {
// The cache loader throws unchecked exception (see #loadDictionary()),
// here we simply report the exception and continue loading the dictionaries
logger.error("exception while loading dictionary {}", e, file.getFileName());
}
}
}
}
}
}
}
}
/**
* Loads the hunspell dictionary for the given local.
*
* @param locale The locale of the hunspell dictionary to be loaded.
* @param nodeSettings The node level settings
* @param env The node environment (from which the conf path will be resolved)
* @return The loaded Hunspell dictionary
* @throws Exception when loading fails (due to IO errors or malformed dictionary files)
*/
private Dictionary loadDictionary(String locale, Settings nodeSettings, Environment env)
throws Exception {
if (logger.isDebugEnabled()) {
logger.debug("Loading hunspell dictionary [{}]...", locale);
}
Path dicDir = hunspellDir.resolve(locale);
if (FileSystemUtils.isAccessibleDirectory(dicDir, logger) == false) {
throw new ElasticsearchException(
String.format(Locale.ROOT, "Could not find hunspell dictionary [%s]", locale));
}
// merging node settings with hunspell dictionary specific settings
Settings dictSettings = HUNSPELL_DICTIONARY_OPTIONS.get(nodeSettings);
nodeSettings = loadDictionarySettings(dicDir, dictSettings.getByPrefix(locale));
boolean ignoreCase = nodeSettings.getAsBoolean("ignore_case", defaultIgnoreCase);
Path[] affixFiles = FileSystemUtils.files(dicDir, "*.aff");
if (affixFiles.length == 0) {
throw new ElasticsearchException(
String.format(Locale.ROOT, "Missing affix file for hunspell dictionary [%s]", locale));
}
if (affixFiles.length != 1) {
throw new ElasticsearchException(
String.format(
Locale.ROOT, "Too many affix files exist for hunspell dictionary [%s]", locale));
}
InputStream affixStream = null;
Path[] dicFiles = FileSystemUtils.files(dicDir, "*.dic");
List<InputStream> dicStreams = new ArrayList<>(dicFiles.length);
try {
for (int i = 0; i < dicFiles.length; i++) {
dicStreams.add(Files.newInputStream(dicFiles[i]));
}
affixStream = Files.newInputStream(affixFiles[0]);
try (Directory tmp = new SimpleFSDirectory(env.tmpFile())) {
return new Dictionary(tmp, "hunspell", affixStream, dicStreams, ignoreCase);
}
} catch (Exception e) {
logger.error("Could not load hunspell dictionary [{}]", e, locale);
throw e;
} finally {
IOUtils.close(affixStream);
IOUtils.close(dicStreams);
}
}
/**
* Each hunspell dictionary directory may contain a {@code settings.yml} which holds dictionary
* specific settings. Default values for these settings are defined in the given default settings.
*
* @param dir The directory of the dictionary
* @param defaults The default settings for this dictionary
* @return The resolved settings.
*/
private static Settings loadDictionarySettings(Path dir, Settings defaults) {
Path file = dir.resolve("settings.yml");
if (Files.exists(file)) {
return Settings.builder().loadFromPath(file).put(defaults).build();
}
file = dir.resolve("settings.json");
if (Files.exists(file)) {
return Settings.builder().loadFromPath(file).put(defaults).build();
}
return defaults;
}
}
public class IndexMetaData
implements Diffable<IndexMetaData>, FromXContentBuilder<IndexMetaData>, ToXContent {
public interface Custom extends Diffable<Custom>, ToXContent {
String type();
Custom fromMap(Map<String, Object> map) throws IOException;
Custom fromXContent(XContentParser parser) throws IOException;
/**
* Merges from this to another, with this being more important, i.e., if something exists in
* this and another, this will prevail.
*/
Custom mergeWith(Custom another);
}
public static Map<String, Custom> customPrototypes = new HashMap<>();
/** Register a custom index meta data factory. Make sure to call it from a static block. */
public static void registerPrototype(String type, Custom proto) {
customPrototypes.put(type, proto);
}
@Nullable
public static <T extends Custom> T lookupPrototype(String type) {
//noinspection unchecked
return (T) customPrototypes.get(type);
}
public static <T extends Custom> T lookupPrototypeSafe(String type) {
//noinspection unchecked
T proto = (T) customPrototypes.get(type);
if (proto == null) {
throw new IllegalArgumentException(
"No custom metadata prototype registered for type [" + type + "]");
}
return proto;
}
public static final ClusterBlock INDEX_READ_ONLY_BLOCK =
new ClusterBlock(
5,
"index read-only (api)",
false,
false,
RestStatus.FORBIDDEN,
EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA_WRITE));
public static final ClusterBlock INDEX_READ_BLOCK =
new ClusterBlock(
7,
"index read (api)",
false,
false,
RestStatus.FORBIDDEN,
EnumSet.of(ClusterBlockLevel.READ));
public static final ClusterBlock INDEX_WRITE_BLOCK =
new ClusterBlock(
8,
"index write (api)",
false,
false,
RestStatus.FORBIDDEN,
EnumSet.of(ClusterBlockLevel.WRITE));
public static final ClusterBlock INDEX_METADATA_BLOCK =
new ClusterBlock(
9,
"index metadata (api)",
false,
false,
RestStatus.FORBIDDEN,
EnumSet.of(ClusterBlockLevel.METADATA_WRITE, ClusterBlockLevel.METADATA_READ));
public static enum State {
OPEN((byte) 0),
CLOSE((byte) 1);
private final byte id;
State(byte id) {
this.id = id;
}
public byte id() {
return this.id;
}
public static State fromId(byte id) {
if (id == 0) {
return OPEN;
} else if (id == 1) {
return CLOSE;
}
throw new IllegalStateException("No state match for id [" + id + "]");
}
public static State fromString(String state) {
if ("open".equals(state)) {
return OPEN;
} else if ("close".equals(state)) {
return CLOSE;
}
throw new IllegalStateException("No state match for [" + state + "]");
}
}
public static final String INDEX_SETTING_PREFIX = "index.";
public static final String SETTING_NUMBER_OF_SHARDS = "index.number_of_shards";
public static final Setting<Integer> INDEX_NUMBER_OF_SHARDS_SETTING =
Setting.intSetting(SETTING_NUMBER_OF_SHARDS, 5, 1, false, Setting.Scope.INDEX);
public static final String SETTING_NUMBER_OF_REPLICAS = "index.number_of_replicas";
public static final Setting<Integer> INDEX_NUMBER_OF_REPLICAS_SETTING =
Setting.intSetting(SETTING_NUMBER_OF_REPLICAS, 1, 0, true, Setting.Scope.INDEX);
public static final String SETTING_SHADOW_REPLICAS = "index.shadow_replicas";
public static final Setting<Boolean> INDEX_SHADOW_REPLICAS_SETTING =
Setting.boolSetting(SETTING_SHADOW_REPLICAS, false, false, Setting.Scope.INDEX);
public static final String SETTING_SHARED_FILESYSTEM = "index.shared_filesystem";
public static final Setting<Boolean> INDEX_SHARED_FILESYSTEM_SETTING =
Setting.boolSetting(SETTING_SHARED_FILESYSTEM, false, false, Setting.Scope.INDEX);
public static final String SETTING_AUTO_EXPAND_REPLICAS = "index.auto_expand_replicas";
public static final Setting<AutoExpandReplicas> INDEX_AUTO_EXPAND_REPLICAS_SETTING =
AutoExpandReplicas.SETTING;
public static final String SETTING_READ_ONLY = "index.blocks.read_only";
public static final Setting<Boolean> INDEX_READ_ONLY_SETTING =
Setting.boolSetting(SETTING_READ_ONLY, false, true, Setting.Scope.INDEX);
public static final String SETTING_BLOCKS_READ = "index.blocks.read";
public static final Setting<Boolean> INDEX_BLOCKS_READ_SETTING =
Setting.boolSetting(SETTING_BLOCKS_READ, false, true, Setting.Scope.INDEX);
public static final String SETTING_BLOCKS_WRITE = "index.blocks.write";
public static final Setting<Boolean> INDEX_BLOCKS_WRITE_SETTING =
Setting.boolSetting(SETTING_BLOCKS_WRITE, false, true, Setting.Scope.INDEX);
public static final String SETTING_BLOCKS_METADATA = "index.blocks.metadata";
public static final Setting<Boolean> INDEX_BLOCKS_METADATA_SETTING =
Setting.boolSetting(SETTING_BLOCKS_METADATA, false, true, Setting.Scope.INDEX);
public static final String SETTING_VERSION_CREATED = "index.version.created";
public static final String SETTING_VERSION_CREATED_STRING = "index.version.created_string";
public static final String SETTING_VERSION_UPGRADED = "index.version.upgraded";
public static final String SETTING_VERSION_UPGRADED_STRING = "index.version.upgraded_string";
public static final String SETTING_VERSION_MINIMUM_COMPATIBLE =
"index.version.minimum_compatible";
public static final String SETTING_CREATION_DATE = "index.creation_date";
public static final String SETTING_PRIORITY = "index.priority";
public static final Setting<Integer> INDEX_PRIORITY_SETTING =
Setting.intSetting("index.priority", 1, 0, true, Setting.Scope.INDEX);
public static final String SETTING_CREATION_DATE_STRING = "index.creation_date_string";
public static final String SETTING_INDEX_UUID = "index.uuid";
public static final String SETTING_DATA_PATH = "index.data_path";
public static final Setting<String> INDEX_DATA_PATH_SETTING =
new Setting<>(SETTING_DATA_PATH, "", Function.identity(), false, Setting.Scope.INDEX);
public static final String SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE =
"index.shared_filesystem.recover_on_any_node";
public static final Setting<Boolean> INDEX_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE_SETTING =
Setting.boolSetting(
SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE, false, true, Setting.Scope.INDEX);
public static final String INDEX_UUID_NA_VALUE = "_na_";
public static final Setting<Settings> INDEX_ROUTING_REQUIRE_GROUP_SETTING =
Setting.groupSetting("index.routing.allocation.require.", true, Setting.Scope.INDEX);
public static final Setting<Settings> INDEX_ROUTING_INCLUDE_GROUP_SETTING =
Setting.groupSetting("index.routing.allocation.include.", true, Setting.Scope.INDEX);
public static final Setting<Settings> INDEX_ROUTING_EXCLUDE_GROUP_SETTING =
Setting.groupSetting("index.routing.allocation.exclude.", true, Setting.Scope.INDEX);
public static final IndexMetaData PROTO =
IndexMetaData.builder("")
.settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(0)
.build();
public static final String KEY_ACTIVE_ALLOCATIONS = "active_allocations";
private final int numberOfShards;
private final int numberOfReplicas;
private final Index index;
private final long version;
private final State state;
private final ImmutableOpenMap<String, AliasMetaData> aliases;
private final Settings settings;
private final ImmutableOpenMap<String, MappingMetaData> mappings;
private final ImmutableOpenMap<String, Custom> customs;
private final ImmutableOpenIntMap<Set<String>> activeAllocationIds;
private final transient int totalNumberOfShards;
private final DiscoveryNodeFilters requireFilters;
private final DiscoveryNodeFilters includeFilters;
private final DiscoveryNodeFilters excludeFilters;
private final Version indexCreatedVersion;
private final Version indexUpgradedVersion;
private final org.apache.lucene.util.Version minimumCompatibleLuceneVersion;
private IndexMetaData(
Index index,
long version,
State state,
int numberOfShards,
int numberOfReplicas,
Settings settings,
ImmutableOpenMap<String, MappingMetaData> mappings,
ImmutableOpenMap<String, AliasMetaData> aliases,
ImmutableOpenMap<String, Custom> customs,
ImmutableOpenIntMap<Set<String>> activeAllocationIds,
DiscoveryNodeFilters requireFilters,
DiscoveryNodeFilters includeFilters,
DiscoveryNodeFilters excludeFilters,
Version indexCreatedVersion,
Version indexUpgradedVersion,
org.apache.lucene.util.Version minimumCompatibleLuceneVersion) {
this.index = index;
this.version = version;
this.state = state;
this.numberOfShards = numberOfShards;
this.numberOfReplicas = numberOfReplicas;
this.totalNumberOfShards = numberOfShards * (numberOfReplicas + 1);
this.settings = settings;
this.mappings = mappings;
this.customs = customs;
this.aliases = aliases;
this.activeAllocationIds = activeAllocationIds;
this.requireFilters = requireFilters;
this.includeFilters = includeFilters;
this.excludeFilters = excludeFilters;
this.indexCreatedVersion = indexCreatedVersion;
this.indexUpgradedVersion = indexUpgradedVersion;
this.minimumCompatibleLuceneVersion = minimumCompatibleLuceneVersion;
}
public Index getIndex() {
return index;
}
public String getIndexUUID() {
return index.getUUID();
}
/**
* Test whether the current index UUID is the same as the given one. Returns true if either are
* _na_
*/
public boolean isSameUUID(String otherUUID) {
assert otherUUID != null;
assert getIndexUUID() != null;
if (INDEX_UUID_NA_VALUE.equals(otherUUID) || INDEX_UUID_NA_VALUE.equals(getIndexUUID())) {
return true;
}
return otherUUID.equals(getIndexUUID());
}
public long getVersion() {
return this.version;
}
/**
* Return the {@link Version} on which this index has been created. This information is typically
* useful for backward compatibility.
*/
public Version getCreationVersion() {
return indexCreatedVersion;
}
/**
* Return the {@link Version} on which this index has been upgraded. This information is typically
* useful for backward compatibility.
*/
public Version getUpgradedVersion() {
return indexUpgradedVersion;
}
/** Return the {@link org.apache.lucene.util.Version} of the oldest lucene segment in the index */
public org.apache.lucene.util.Version getMinimumCompatibleVersion() {
return minimumCompatibleLuceneVersion;
}
public long getCreationDate() {
return settings.getAsLong(SETTING_CREATION_DATE, -1l);
}
public State getState() {
return this.state;
}
public int getNumberOfShards() {
return numberOfShards;
}
public int getNumberOfReplicas() {
return numberOfReplicas;
}
public int getTotalNumberOfShards() {
return totalNumberOfShards;
}
public Settings getSettings() {
return settings;
}
public ImmutableOpenMap<String, AliasMetaData> getAliases() {
return this.aliases;
}
public ImmutableOpenMap<String, MappingMetaData> getMappings() {
return mappings;
}
@Nullable
public MappingMetaData mapping(String mappingType) {
return mappings.get(mappingType);
}
/**
* Sometimes, the default mapping exists and an actual mapping is not created yet (introduced), in
* this case, we want to return the default mapping in case it has some default mapping
* definitions.
*
* <p>Note, once the mapping type is introduced, the default mapping is applied on the actual
* typed MappingMetaData, setting its routing, timestamp, and so on if needed.
*/
@Nullable
public MappingMetaData mappingOrDefault(String mappingType) {
MappingMetaData mapping = mappings.get(mappingType);
if (mapping != null) {
return mapping;
}
return mappings.get(MapperService.DEFAULT_MAPPING);
}
public ImmutableOpenMap<String, Custom> getCustoms() {
return this.customs;
}
@SuppressWarnings("unchecked")
public <T extends Custom> T custom(String type) {
return (T) customs.get(type);
}
public ImmutableOpenIntMap<Set<String>> getActiveAllocationIds() {
return activeAllocationIds;
}
public Set<String> activeAllocationIds(int shardId) {
assert shardId >= 0 && shardId < numberOfShards;
return activeAllocationIds.get(shardId);
}
@Nullable
public DiscoveryNodeFilters requireFilters() {
return requireFilters;
}
@Nullable
public DiscoveryNodeFilters includeFilters() {
return includeFilters;
}
@Nullable
public DiscoveryNodeFilters excludeFilters() {
return excludeFilters;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
IndexMetaData that = (IndexMetaData) o;
if (!aliases.equals(that.aliases)) {
return false;
}
if (!index.equals(that.index)) {
return false;
}
if (!mappings.equals(that.mappings)) {
return false;
}
if (!settings.equals(that.settings)) {
return false;
}
if (state != that.state) {
return false;
}
if (!customs.equals(that.customs)) {
return false;
}
if (!activeAllocationIds.equals(that.activeAllocationIds)) {
return false;
}
return true;
}
@Override
public int hashCode() {
int result = index.hashCode();
result = 31 * result + state.hashCode();
result = 31 * result + aliases.hashCode();
result = 31 * result + settings.hashCode();
result = 31 * result + mappings.hashCode();
result = 31 * result + activeAllocationIds.hashCode();
return result;
}
@Override
public Diff<IndexMetaData> diff(IndexMetaData previousState) {
return new IndexMetaDataDiff(previousState, this);
}
@Override
public Diff<IndexMetaData> readDiffFrom(StreamInput in) throws IOException {
return new IndexMetaDataDiff(in);
}
@Override
public IndexMetaData fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher)
throws IOException {
return Builder.fromXContent(parser);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
Builder.toXContent(this, builder, params);
return builder;
}
private static class IndexMetaDataDiff implements Diff<IndexMetaData> {
private final String index;
private final long version;
private final State state;
private final Settings settings;
private final Diff<ImmutableOpenMap<String, MappingMetaData>> mappings;
private final Diff<ImmutableOpenMap<String, AliasMetaData>> aliases;
private final Diff<ImmutableOpenMap<String, Custom>> customs;
private final Diff<ImmutableOpenIntMap<Set<String>>> activeAllocationIds;
public IndexMetaDataDiff(IndexMetaData before, IndexMetaData after) {
index = after.index.getName();
version = after.version;
state = after.state;
settings = after.settings;
mappings =
DiffableUtils.diff(
before.mappings, after.mappings, DiffableUtils.getStringKeySerializer());
aliases =
DiffableUtils.diff(before.aliases, after.aliases, DiffableUtils.getStringKeySerializer());
customs =
DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer());
activeAllocationIds =
DiffableUtils.diff(
before.activeAllocationIds,
after.activeAllocationIds,
DiffableUtils.getVIntKeySerializer(),
DiffableUtils.StringSetValueSerializer.getInstance());
}
public IndexMetaDataDiff(StreamInput in) throws IOException {
index = in.readString();
version = in.readLong();
state = State.fromId(in.readByte());
settings = Settings.readSettingsFromStream(in);
mappings =
DiffableUtils.readImmutableOpenMapDiff(
in, DiffableUtils.getStringKeySerializer(), MappingMetaData.PROTO);
aliases =
DiffableUtils.readImmutableOpenMapDiff(
in, DiffableUtils.getStringKeySerializer(), AliasMetaData.PROTO);
customs =
DiffableUtils.readImmutableOpenMapDiff(
in,
DiffableUtils.getStringKeySerializer(),
new DiffableUtils.DiffableValueSerializer<String, Custom>() {
@Override
public Custom read(StreamInput in, String key) throws IOException {
return lookupPrototypeSafe(key).readFrom(in);
}
@Override
public Diff<Custom> readDiff(StreamInput in, String key) throws IOException {
return lookupPrototypeSafe(key).readDiffFrom(in);
}
});
activeAllocationIds =
DiffableUtils.readImmutableOpenIntMapDiff(
in,
DiffableUtils.getVIntKeySerializer(),
DiffableUtils.StringSetValueSerializer.getInstance());
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(index);
out.writeLong(version);
out.writeByte(state.id);
Settings.writeSettingsToStream(settings, out);
mappings.writeTo(out);
aliases.writeTo(out);
customs.writeTo(out);
activeAllocationIds.writeTo(out);
}
@Override
public IndexMetaData apply(IndexMetaData part) {
Builder builder = builder(index);
builder.version(version);
builder.state(state);
builder.settings(settings);
builder.mappings.putAll(mappings.apply(part.mappings));
builder.aliases.putAll(aliases.apply(part.aliases));
builder.customs.putAll(customs.apply(part.customs));
builder.activeAllocationIds.putAll(activeAllocationIds.apply(part.activeAllocationIds));
return builder.build();
}
}
@Override
public IndexMetaData readFrom(StreamInput in) throws IOException {
Builder builder = new Builder(in.readString());
builder.version(in.readLong());
builder.state(State.fromId(in.readByte()));
builder.settings(readSettingsFromStream(in));
int mappingsSize = in.readVInt();
for (int i = 0; i < mappingsSize; i++) {
MappingMetaData mappingMd = MappingMetaData.PROTO.readFrom(in);
builder.putMapping(mappingMd);
}
int aliasesSize = in.readVInt();
for (int i = 0; i < aliasesSize; i++) {
AliasMetaData aliasMd = AliasMetaData.Builder.readFrom(in);
builder.putAlias(aliasMd);
}
int customSize = in.readVInt();
for (int i = 0; i < customSize; i++) {
String type = in.readString();
Custom customIndexMetaData = lookupPrototypeSafe(type).readFrom(in);
builder.putCustom(type, customIndexMetaData);
}
int activeAllocationIdsSize = in.readVInt();
for (int i = 0; i < activeAllocationIdsSize; i++) {
int key = in.readVInt();
Set<String> allocationIds =
DiffableUtils.StringSetValueSerializer.getInstance().read(in, key);
builder.putActiveAllocationIds(key, allocationIds);
}
return builder.build();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(index.getName()); // uuid will come as part of settings
out.writeLong(version);
out.writeByte(state.id());
writeSettingsToStream(settings, out);
out.writeVInt(mappings.size());
for (ObjectCursor<MappingMetaData> cursor : mappings.values()) {
cursor.value.writeTo(out);
}
out.writeVInt(aliases.size());
for (ObjectCursor<AliasMetaData> cursor : aliases.values()) {
cursor.value.writeTo(out);
}
out.writeVInt(customs.size());
for (ObjectObjectCursor<String, Custom> cursor : customs) {
out.writeString(cursor.key);
cursor.value.writeTo(out);
}
out.writeVInt(activeAllocationIds.size());
for (IntObjectCursor<Set<String>> cursor : activeAllocationIds) {
out.writeVInt(cursor.key);
DiffableUtils.StringSetValueSerializer.getInstance().write(cursor.value, out);
}
}
public static Builder builder(String index) {
return new Builder(index);
}
public static Builder builder(IndexMetaData indexMetaData) {
return new Builder(indexMetaData);
}
public static class Builder {
private String index;
private State state = State.OPEN;
private long version = 1;
private Settings settings = Settings.Builder.EMPTY_SETTINGS;
private final ImmutableOpenMap.Builder<String, MappingMetaData> mappings;
private final ImmutableOpenMap.Builder<String, AliasMetaData> aliases;
private final ImmutableOpenMap.Builder<String, Custom> customs;
private final ImmutableOpenIntMap.Builder<Set<String>> activeAllocationIds;
public Builder(String index) {
this.index = index;
this.mappings = ImmutableOpenMap.builder();
this.aliases = ImmutableOpenMap.builder();
this.customs = ImmutableOpenMap.builder();
this.activeAllocationIds = ImmutableOpenIntMap.builder();
}
public Builder(IndexMetaData indexMetaData) {
this.index = indexMetaData.getIndex().getName();
this.state = indexMetaData.state;
this.version = indexMetaData.version;
this.settings = indexMetaData.getSettings();
this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings);
this.aliases = ImmutableOpenMap.builder(indexMetaData.aliases);
this.customs = ImmutableOpenMap.builder(indexMetaData.customs);
this.activeAllocationIds = ImmutableOpenIntMap.builder(indexMetaData.activeAllocationIds);
}
public String index() {
return index;
}
public Builder index(String index) {
this.index = index;
return this;
}
public Builder numberOfShards(int numberOfShards) {
settings =
settingsBuilder().put(settings).put(SETTING_NUMBER_OF_SHARDS, numberOfShards).build();
return this;
}
public int numberOfShards() {
return settings.getAsInt(SETTING_NUMBER_OF_SHARDS, -1);
}
public Builder numberOfReplicas(int numberOfReplicas) {
settings =
settingsBuilder().put(settings).put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas).build();
return this;
}
public int numberOfReplicas() {
return settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1);
}
public Builder creationDate(long creationDate) {
settings = settingsBuilder().put(settings).put(SETTING_CREATION_DATE, creationDate).build();
return this;
}
public Builder settings(Settings.Builder settings) {
this.settings = settings.build();
return this;
}
public Builder settings(Settings settings) {
this.settings = settings;
return this;
}
public MappingMetaData mapping(String type) {
return mappings.get(type);
}
public Builder putMapping(String type, String source) throws IOException {
try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
putMapping(new MappingMetaData(type, parser.mapOrdered()));
}
return this;
}
public Builder putMapping(MappingMetaData mappingMd) {
mappings.put(mappingMd.type(), mappingMd);
return this;
}
public Builder state(State state) {
this.state = state;
return this;
}
public Builder putAlias(AliasMetaData aliasMetaData) {
aliases.put(aliasMetaData.alias(), aliasMetaData);
return this;
}
public Builder putAlias(AliasMetaData.Builder aliasMetaData) {
aliases.put(aliasMetaData.alias(), aliasMetaData.build());
return this;
}
public Builder removeAlias(String alias) {
aliases.remove(alias);
return this;
}
public Builder removeAllAliases() {
aliases.clear();
return this;
}
public Builder putCustom(String type, Custom customIndexMetaData) {
this.customs.put(type, customIndexMetaData);
return this;
}
public Builder putActiveAllocationIds(int shardId, Set<String> allocationIds) {
activeAllocationIds.put(shardId, new HashSet(allocationIds));
return this;
}
public long version() {
return this.version;
}
public Builder version(long version) {
this.version = version;
return this;
}
public IndexMetaData build() {
ImmutableOpenMap.Builder<String, AliasMetaData> tmpAliases = aliases;
Settings tmpSettings = settings;
// update default mapping on the MappingMetaData
if (mappings.containsKey(MapperService.DEFAULT_MAPPING)) {
MappingMetaData defaultMapping = mappings.get(MapperService.DEFAULT_MAPPING);
for (ObjectCursor<MappingMetaData> cursor : mappings.values()) {
cursor.value.updateDefaultMapping(defaultMapping);
}
}
Integer maybeNumberOfShards = settings.getAsInt(SETTING_NUMBER_OF_SHARDS, null);
if (maybeNumberOfShards == null) {
throw new IllegalArgumentException("must specify numberOfShards for index [" + index + "]");
}
int numberOfShards = maybeNumberOfShards;
if (numberOfShards <= 0) {
throw new IllegalArgumentException(
"must specify positive number of shards for index [" + index + "]");
}
Integer maybeNumberOfReplicas = settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, null);
if (maybeNumberOfReplicas == null) {
throw new IllegalArgumentException(
"must specify numberOfReplicas for index [" + index + "]");
}
int numberOfReplicas = maybeNumberOfReplicas;
if (numberOfReplicas < 0) {
throw new IllegalArgumentException(
"must specify non-negative number of shards for index [" + index + "]");
}
// fill missing slots in activeAllocationIds with empty set if needed and make all entries
// immutable
ImmutableOpenIntMap.Builder<Set<String>> filledActiveAllocationIds =
ImmutableOpenIntMap.builder();
for (int i = 0; i < numberOfShards; i++) {
if (activeAllocationIds.containsKey(i)) {
filledActiveAllocationIds.put(
i, Collections.unmodifiableSet(new HashSet<>(activeAllocationIds.get(i))));
} else {
filledActiveAllocationIds.put(i, Collections.emptySet());
}
}
final Map<String, String> requireMap =
INDEX_ROUTING_REQUIRE_GROUP_SETTING.get(settings).getAsMap();
final DiscoveryNodeFilters requireFilters;
if (requireMap.isEmpty()) {
requireFilters = null;
} else {
requireFilters = DiscoveryNodeFilters.buildFromKeyValue(AND, requireMap);
}
Map<String, String> includeMap = INDEX_ROUTING_INCLUDE_GROUP_SETTING.get(settings).getAsMap();
final DiscoveryNodeFilters includeFilters;
if (includeMap.isEmpty()) {
includeFilters = null;
} else {
includeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, includeMap);
}
Map<String, String> excludeMap = INDEX_ROUTING_EXCLUDE_GROUP_SETTING.get(settings).getAsMap();
final DiscoveryNodeFilters excludeFilters;
if (excludeMap.isEmpty()) {
excludeFilters = null;
} else {
excludeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, excludeMap);
}
Version indexCreatedVersion = Version.indexCreated(settings);
Version indexUpgradedVersion =
settings.getAsVersion(IndexMetaData.SETTING_VERSION_UPGRADED, indexCreatedVersion);
String stringLuceneVersion = settings.get(SETTING_VERSION_MINIMUM_COMPATIBLE);
final org.apache.lucene.util.Version minimumCompatibleLuceneVersion;
if (stringLuceneVersion != null) {
try {
minimumCompatibleLuceneVersion =
org.apache.lucene.util.Version.parse(stringLuceneVersion);
} catch (ParseException ex) {
throw new IllegalStateException(
"Cannot parse lucene version ["
+ stringLuceneVersion
+ "] in the ["
+ SETTING_VERSION_MINIMUM_COMPATIBLE
+ "] setting",
ex);
}
} else {
minimumCompatibleLuceneVersion = null;
}
final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE);
return new IndexMetaData(
new Index(index, uuid),
version,
state,
numberOfShards,
numberOfReplicas,
tmpSettings,
mappings.build(),
tmpAliases.build(),
customs.build(),
filledActiveAllocationIds.build(),
requireFilters,
includeFilters,
excludeFilters,
indexCreatedVersion,
indexUpgradedVersion,
minimumCompatibleLuceneVersion);
}
public static void toXContent(
IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params)
throws IOException {
builder.startObject(
indexMetaData.getIndex().getName(), XContentBuilder.FieldCaseConversion.NONE);
builder.field("version", indexMetaData.getVersion());
builder.field("state", indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH));
boolean binary = params.paramAsBoolean("binary", false);
builder.startObject("settings");
for (Map.Entry<String, String> entry : indexMetaData.getSettings().getAsMap().entrySet()) {
builder.field(entry.getKey(), entry.getValue());
}
builder.endObject();
builder.startArray("mappings");
for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.getMappings()) {
if (binary) {
builder.value(cursor.value.source().compressed());
} else {
byte[] data = cursor.value.source().uncompressed();
XContentParser parser = XContentFactory.xContent(data).createParser(data);
Map<String, Object> mapping = parser.mapOrdered();
parser.close();
builder.map(mapping);
}
}
builder.endArray();
for (ObjectObjectCursor<String, Custom> cursor : indexMetaData.getCustoms()) {
builder.startObject(cursor.key, XContentBuilder.FieldCaseConversion.NONE);
cursor.value.toXContent(builder, params);
builder.endObject();
}
builder.startObject("aliases");
for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
AliasMetaData.Builder.toXContent(cursor.value, builder, params);
}
builder.endObject();
builder.startObject(KEY_ACTIVE_ALLOCATIONS);
for (IntObjectCursor<Set<String>> cursor : indexMetaData.activeAllocationIds) {
builder.startArray(String.valueOf(cursor.key));
for (String allocationId : cursor.value) {
builder.value(allocationId);
}
builder.endArray();
}
builder.endObject();
builder.endObject();
}
public static IndexMetaData fromXContent(XContentParser parser) throws IOException {
if (parser.currentToken() == null) { // fresh parser? move to the first token
parser.nextToken();
}
if (parser.currentToken()
== XContentParser.Token.START_OBJECT) { // on a start object move to next token
parser.nextToken();
}
if (parser.currentToken() != XContentParser.Token.FIELD_NAME) {
throw new IllegalArgumentException(
"expected field name but got a " + parser.currentToken());
}
Builder builder = new Builder(parser.currentName());
String currentFieldName = null;
XContentParser.Token token = parser.nextToken();
if (token != XContentParser.Token.START_OBJECT) {
throw new IllegalArgumentException("expected object but got a " + token);
}
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if ("settings".equals(currentFieldName)) {
builder.settings(
Settings.settingsBuilder()
.put(SettingsLoader.Helper.loadNestedFromMap(parser.mapOrdered())));
} else if ("mappings".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
String mappingType = currentFieldName;
Map<String, Object> mappingSource =
MapBuilder.<String, Object>newMapBuilder()
.put(mappingType, parser.mapOrdered())
.map();
builder.putMapping(new MappingMetaData(mappingType, mappingSource));
} else {
throw new IllegalArgumentException("Unexpected token: " + token);
}
}
} else if ("aliases".equals(currentFieldName)) {
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
builder.putAlias(AliasMetaData.Builder.fromXContent(parser));
}
} else if (KEY_ACTIVE_ALLOCATIONS.equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_ARRAY) {
String shardId = currentFieldName;
Set<String> allocationIds = new HashSet<>();
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
if (token == XContentParser.Token.VALUE_STRING) {
allocationIds.add(parser.text());
}
}
builder.putActiveAllocationIds(Integer.valueOf(shardId), allocationIds);
} else {
throw new IllegalArgumentException("Unexpected token: " + token);
}
}
} else if ("warmers".equals(currentFieldName)) {
// TODO: do this in 4.0:
// throw new IllegalArgumentException("Warmers are not supported anymore - are you
// upgrading from 1.x?");
// ignore: warmers have been removed in 3.0 and are
// simply ignored when upgrading from 2.x
assert Version.CURRENT.major <= 3;
parser.skipChildren();
} else {
// check if its a custom index metadata
Custom proto = lookupPrototype(currentFieldName);
if (proto == null) {
// TODO warn
parser.skipChildren();
} else {
Custom custom = proto.fromXContent(parser);
builder.putCustom(custom.type(), custom);
}
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("mappings".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
builder.putMapping(
new MappingMetaData(new CompressedXContent(parser.binaryValue())));
} else {
Map<String, Object> mapping = parser.mapOrdered();
if (mapping.size() == 1) {
String mappingType = mapping.keySet().iterator().next();
builder.putMapping(new MappingMetaData(mappingType, mapping));
}
}
}
} else {
throw new IllegalArgumentException("Unexpected field for an array " + currentFieldName);
}
} else if (token.isValue()) {
if ("state".equals(currentFieldName)) {
builder.state(State.fromString(parser.text()));
} else if ("version".equals(currentFieldName)) {
builder.version(parser.longValue());
} else {
throw new IllegalArgumentException("Unexpected field [" + currentFieldName + "]");
}
} else {
throw new IllegalArgumentException("Unexpected token " + token);
}
}
return builder.build();
}
public static IndexMetaData readFrom(StreamInput in) throws IOException {
return PROTO.readFrom(in);
}
}
/**
* Returns <code>true</code> iff the given settings indicate that the index associated with these
* settings allocates it's shards on a shared filesystem. Otherwise <code>false</code>. The
* default setting for this is the returned value from {@link
* #isIndexUsingShadowReplicas(org.elasticsearch.common.settings.Settings)}.
*/
public static boolean isOnSharedFilesystem(Settings settings) {
return settings.getAsBoolean(SETTING_SHARED_FILESYSTEM, isIndexUsingShadowReplicas(settings));
}
/**
* Returns <code>true</code> iff the given settings indicate that the index associated with these
* settings uses shadow replicas. Otherwise <code>false</code>. The default setting for this is
* <code>false</code>.
*/
public static boolean isIndexUsingShadowReplicas(Settings settings) {
return settings.getAsBoolean(SETTING_SHADOW_REPLICAS, false);
}
/**
* Adds human readable version and creation date settings. This method is used to display the
* settings in a human readable format in REST API
*/
public static Settings addHumanReadableSettings(Settings settings) {
Settings.Builder builder = Settings.builder().put(settings);
Version version = settings.getAsVersion(SETTING_VERSION_CREATED, null);
if (version != null) {
builder.put(SETTING_VERSION_CREATED_STRING, version.toString());
}
Version versionUpgraded = settings.getAsVersion(SETTING_VERSION_UPGRADED, null);
if (versionUpgraded != null) {
builder.put(SETTING_VERSION_UPGRADED_STRING, versionUpgraded.toString());
}
Long creationDate = settings.getAsLong(SETTING_CREATION_DATE, null);
if (creationDate != null) {
DateTime creationDateTime = new DateTime(creationDate, DateTimeZone.UTC);
builder.put(SETTING_CREATION_DATE_STRING, creationDateTime.toString());
}
return builder.build();
}
}
/** A module to handle registering and binding all network related classes. */
public final class NetworkModule {
public static final String TRANSPORT_TYPE_KEY = "transport.type";
public static final String HTTP_TYPE_KEY = "http.type";
public static final String LOCAL_TRANSPORT = "local";
public static final String HTTP_TYPE_DEFAULT_KEY = "http.type.default";
public static final String TRANSPORT_TYPE_DEFAULT_KEY = "transport.type.default";
public static final Setting<String> TRANSPORT_DEFAULT_TYPE_SETTING =
Setting.simpleString(TRANSPORT_TYPE_DEFAULT_KEY, Property.NodeScope);
public static final Setting<String> HTTP_DEFAULT_TYPE_SETTING =
Setting.simpleString(HTTP_TYPE_DEFAULT_KEY, Property.NodeScope);
public static final Setting<String> HTTP_TYPE_SETTING =
Setting.simpleString(HTTP_TYPE_KEY, Property.NodeScope);
public static final Setting<Boolean> HTTP_ENABLED =
Setting.boolSetting("http.enabled", true, Property.NodeScope);
public static final Setting<String> TRANSPORT_TYPE_SETTING =
Setting.simpleString(TRANSPORT_TYPE_KEY, Property.NodeScope);
private final Settings settings;
private final boolean transportClient;
private static final AllocationCommandRegistry allocationCommandRegistry =
new AllocationCommandRegistry();
private static final List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
private final Map<String, Supplier<Transport>> transportFactories = new HashMap<>();
private final Map<String, Supplier<HttpServerTransport>> transportHttpFactories = new HashMap<>();
private final List<TransportInterceptor> transportIntercetors = new ArrayList<>();
static {
registerAllocationCommand(
CancelAllocationCommand::new,
CancelAllocationCommand::fromXContent,
CancelAllocationCommand.COMMAND_NAME_FIELD);
registerAllocationCommand(
MoveAllocationCommand::new,
MoveAllocationCommand::fromXContent,
MoveAllocationCommand.COMMAND_NAME_FIELD);
registerAllocationCommand(
AllocateReplicaAllocationCommand::new,
AllocateReplicaAllocationCommand::fromXContent,
AllocateReplicaAllocationCommand.COMMAND_NAME_FIELD);
registerAllocationCommand(
AllocateEmptyPrimaryAllocationCommand::new,
AllocateEmptyPrimaryAllocationCommand::fromXContent,
AllocateEmptyPrimaryAllocationCommand.COMMAND_NAME_FIELD);
registerAllocationCommand(
AllocateStalePrimaryAllocationCommand::new,
AllocateStalePrimaryAllocationCommand::fromXContent,
AllocateStalePrimaryAllocationCommand.COMMAND_NAME_FIELD);
namedWriteables.add(
new NamedWriteableRegistry.Entry(
Task.Status.class, ReplicationTask.Status.NAME, ReplicationTask.Status::new));
namedWriteables.add(
new NamedWriteableRegistry.Entry(
Task.Status.class, RawTaskStatus.NAME, RawTaskStatus::new));
}
/**
* Creates a network module that custom networking classes can be plugged into.
*
* @param settings The settings for the node
* @param transportClient True if only transport classes should be allowed to be registered, false
* otherwise.
*/
public NetworkModule(
Settings settings,
boolean transportClient,
List<NetworkPlugin> plugins,
ThreadPool threadPool,
BigArrays bigArrays,
CircuitBreakerService circuitBreakerService,
NamedWriteableRegistry namedWriteableRegistry,
NetworkService networkService) {
this.settings = settings;
this.transportClient = transportClient;
for (NetworkPlugin plugin : plugins) {
if (transportClient == false && HTTP_ENABLED.get(settings)) {
Map<String, Supplier<HttpServerTransport>> httpTransportFactory =
plugin.getHttpTransports(
settings,
threadPool,
bigArrays,
circuitBreakerService,
namedWriteableRegistry,
networkService);
for (Map.Entry<String, Supplier<HttpServerTransport>> entry :
httpTransportFactory.entrySet()) {
registerHttpTransport(entry.getKey(), entry.getValue());
}
}
Map<String, Supplier<Transport>> httpTransportFactory =
plugin.getTransports(
settings,
threadPool,
bigArrays,
circuitBreakerService,
namedWriteableRegistry,
networkService);
for (Map.Entry<String, Supplier<Transport>> entry : httpTransportFactory.entrySet()) {
registerTransport(entry.getKey(), entry.getValue());
}
List<TransportInterceptor> transportInterceptors =
plugin.getTransportInterceptors(namedWriteableRegistry);
for (TransportInterceptor interceptor : transportInterceptors) {
registerTransportInterceptor(interceptor);
}
}
}
public boolean isTransportClient() {
return transportClient;
}
/**
* Adds a transport implementation that can be selected by setting {@link #TRANSPORT_TYPE_KEY}.
*/
private void registerTransport(String key, Supplier<Transport> factory) {
if (transportFactories.putIfAbsent(key, factory) != null) {
throw new IllegalArgumentException("transport for name: " + key + " is already registered");
}
}
/**
* Adds an http transport implementation that can be selected by setting {@link #HTTP_TYPE_KEY}.
*/
// TODO: we need another name than "http transport"....so confusing with transportClient...
private void registerHttpTransport(String key, Supplier<HttpServerTransport> factory) {
if (transportClient) {
throw new IllegalArgumentException(
"Cannot register http transport " + key + " for transport client");
}
if (transportHttpFactories.putIfAbsent(key, factory) != null) {
throw new IllegalArgumentException("transport for name: " + key + " is already registered");
}
}
/**
* Register an allocation command.
*
* <p>This lives here instead of the more aptly named ClusterModule because the Transport client
* needs these to be registered.
*
* @param reader the reader to read it from a stream
* @param parser the parser to read it from XContent
* @param commandName the names under which the command should be parsed. The {@link
* ParseField#getPreferredName()} is special because it is the name under which the command's
* reader is registered.
*/
private static <T extends AllocationCommand> void registerAllocationCommand(
Writeable.Reader<T> reader, AllocationCommand.Parser<T> parser, ParseField commandName) {
allocationCommandRegistry.register(parser, commandName);
namedWriteables.add(new Entry(AllocationCommand.class, commandName.getPreferredName(), reader));
}
/** The registry of allocation command parsers. */
public static AllocationCommandRegistry getAllocationCommandRegistry() {
return allocationCommandRegistry;
}
public static List<Entry> getNamedWriteables() {
return Collections.unmodifiableList(namedWriteables);
}
public Supplier<HttpServerTransport> getHttpServerTransportSupplier() {
final String name;
if (HTTP_TYPE_SETTING.exists(settings)) {
name = HTTP_TYPE_SETTING.get(settings);
} else {
name = HTTP_DEFAULT_TYPE_SETTING.get(settings);
}
final Supplier<HttpServerTransport> factory = transportHttpFactories.get(name);
if (factory == null) {
throw new IllegalStateException("Unsupported http.type [" + name + "]");
}
return factory;
}
public boolean isHttpEnabled() {
return transportClient == false && HTTP_ENABLED.get(settings);
}
public Supplier<Transport> getTransportSupplier() {
final String name;
if (TRANSPORT_TYPE_SETTING.exists(settings)) {
name = TRANSPORT_TYPE_SETTING.get(settings);
} else {
name = TRANSPORT_DEFAULT_TYPE_SETTING.get(settings);
}
final Supplier<Transport> factory = transportFactories.get(name);
if (factory == null) {
throw new IllegalStateException("Unsupported transport.type [" + name + "]");
}
return factory;
}
/** Registers a new {@link TransportInterceptor} */
private void registerTransportInterceptor(TransportInterceptor interceptor) {
this.transportIntercetors.add(
Objects.requireNonNull(interceptor, "interceptor must not be null"));
}
/**
* Returns a composite {@link TransportInterceptor} containing all registered interceptors
*
* @see #registerTransportInterceptor(TransportInterceptor)
*/
public TransportInterceptor getTransportInterceptor() {
return new CompositeTransportInterceptor(this.transportIntercetors);
}
static final class CompositeTransportInterceptor implements TransportInterceptor {
final List<TransportInterceptor> transportInterceptors;
private CompositeTransportInterceptor(List<TransportInterceptor> transportInterceptors) {
this.transportInterceptors = new ArrayList<>(transportInterceptors);
}
@Override
public <T extends TransportRequest> TransportRequestHandler<T> interceptHandler(
String action, String executor, TransportRequestHandler<T> actualHandler) {
for (TransportInterceptor interceptor : this.transportInterceptors) {
actualHandler = interceptor.interceptHandler(action, executor, actualHandler);
}
return actualHandler;
}
@Override
public AsyncSender interceptSender(AsyncSender sender) {
for (TransportInterceptor interceptor : this.transportInterceptors) {
sender = interceptor.interceptSender(sender);
}
return sender;
}
}
}
/** A component that holds all data paths for a single node. */
public final class NodeEnvironment implements Closeable {
private final Logger logger;
public static class NodePath {
/* ${data.paths}/nodes/{node.id} */
public final Path path;
/* ${data.paths}/nodes/{node.id}/indices */
public final Path indicesPath;
/** Cached FileStore from path */
public final FileStore fileStore;
/**
* Cached result of Lucene's {@code IOUtils.spins} on path. This is a trilean value: null means
* we could not determine it (we are not running on Linux, or we hit an exception trying), True
* means the device possibly spins and False means it does not.
*/
public final Boolean spins;
public final int majorDeviceNumber;
public final int minorDeviceNumber;
public NodePath(Path path) throws IOException {
this.path = path;
this.indicesPath = path.resolve(INDICES_FOLDER);
this.fileStore = Environment.getFileStore(path);
if (fileStore.supportsFileAttributeView("lucene")) {
this.spins = (Boolean) fileStore.getAttribute("lucene:spins");
this.majorDeviceNumber = (int) fileStore.getAttribute("lucene:major_device_number");
this.minorDeviceNumber = (int) fileStore.getAttribute("lucene:minor_device_number");
} else {
this.spins = null;
this.majorDeviceNumber = -1;
this.minorDeviceNumber = -1;
}
}
/**
* Resolves the given shards directory against this NodePath
* ${data.paths}/nodes/{node.id}/indices/{index.uuid}/{shard.id}
*/
public Path resolve(ShardId shardId) {
return resolve(shardId.getIndex()).resolve(Integer.toString(shardId.id()));
}
/**
* Resolves index directory against this NodePath
* ${data.paths}/nodes/{node.id}/indices/{index.uuid}
*/
public Path resolve(Index index) {
return indicesPath.resolve(index.getUUID());
}
@Override
public String toString() {
return "NodePath{" + "path=" + path + ", spins=" + spins + '}';
}
}
private final NodePath[] nodePaths;
private final Path sharedDataPath;
private final Lock[] locks;
private final int nodeLockId;
private final AtomicBoolean closed = new AtomicBoolean(false);
private final Map<ShardId, InternalShardLock> shardLocks = new HashMap<>();
private final NodeMetaData nodeMetaData;
/** Maximum number of data nodes that should run in an environment. */
public static final Setting<Integer> MAX_LOCAL_STORAGE_NODES_SETTING =
Setting.intSetting("node.max_local_storage_nodes", 1, 1, Property.NodeScope);
/** If true automatically append node lock id to custom data paths. */
public static final Setting<Boolean> ADD_NODE_LOCK_ID_TO_CUSTOM_PATH =
Setting.boolSetting("node.add_lock_id_to_custom_path", true, Property.NodeScope);
/**
* Seed for determining a persisted unique uuid of this node. If the node has already a persisted
* uuid on disk, this seed will be ignored and the uuid from disk will be reused.
*/
public static final Setting<Long> NODE_ID_SEED_SETTING =
Setting.longSetting("node.id.seed", 0L, Long.MIN_VALUE, Property.NodeScope);
/** If true the [verbose] SegmentInfos.infoStream logging is sent to System.out. */
public static final Setting<Boolean> ENABLE_LUCENE_SEGMENT_INFOS_TRACE_SETTING =
Setting.boolSetting("node.enable_lucene_segment_infos_trace", false, Property.NodeScope);
public static final String NODES_FOLDER = "nodes";
public static final String INDICES_FOLDER = "indices";
public static final String NODE_LOCK_FILENAME = "node.lock";
public NodeEnvironment(Settings settings, Environment environment) throws IOException {
if (!DiscoveryNode.nodeRequiresLocalStorage(settings)) {
nodePaths = null;
sharedDataPath = null;
locks = null;
nodeLockId = -1;
nodeMetaData = new NodeMetaData(generateNodeId(settings));
logger =
Loggers.getLogger(
getClass(), Node.addNodeNameIfNeeded(settings, this.nodeMetaData.nodeId()));
return;
}
final NodePath[] nodePaths = new NodePath[environment.dataWithClusterFiles().length];
final Lock[] locks = new Lock[nodePaths.length];
boolean success = false;
// trace logger to debug issues before the default node name is derived from the node id
Logger startupTraceLogger = Loggers.getLogger(getClass(), settings);
try {
sharedDataPath = environment.sharedDataFile();
int nodeLockId = -1;
IOException lastException = null;
int maxLocalStorageNodes = MAX_LOCAL_STORAGE_NODES_SETTING.get(settings);
for (int possibleLockId = 0; possibleLockId < maxLocalStorageNodes; possibleLockId++) {
for (int dirIndex = 0; dirIndex < environment.dataFiles().length; dirIndex++) {
Path dataDirWithClusterName = environment.dataWithClusterFiles()[dirIndex];
Path dataDir = environment.dataFiles()[dirIndex];
Path dir = dataDir.resolve(NODES_FOLDER).resolve(Integer.toString(possibleLockId));
Files.createDirectories(dir);
try (Directory luceneDir = FSDirectory.open(dir, NativeFSLockFactory.INSTANCE)) {
startupTraceLogger.trace("obtaining node lock on {} ...", dir.toAbsolutePath());
try {
locks[dirIndex] = luceneDir.obtainLock(NODE_LOCK_FILENAME);
nodePaths[dirIndex] = new NodePath(dir);
nodeLockId = possibleLockId;
} catch (LockObtainFailedException ex) {
startupTraceLogger.trace("failed to obtain node lock on {}", dir.toAbsolutePath());
// release all the ones that were obtained up until now
releaseAndNullLocks(locks);
break;
}
} catch (IOException e) {
startupTraceLogger.trace(
(Supplier<?>)
() ->
new ParameterizedMessage(
"failed to obtain node lock on {}", dir.toAbsolutePath()),
e);
lastException = new IOException("failed to obtain lock on " + dir.toAbsolutePath(), e);
// release all the ones that were obtained up until now
releaseAndNullLocks(locks);
break;
}
}
if (locks[0] != null) {
// we found a lock, break
break;
}
}
if (locks[0] == null) {
final String message =
String.format(
Locale.ROOT,
"failed to obtain node locks, tried [%s] with lock id%s;"
+ " maybe these locations are not writable or multiple nodes were started without increasing [%s] (was [%d])?",
Arrays.toString(environment.dataWithClusterFiles()),
maxLocalStorageNodes == 1 ? " [0]" : "s [0--" + (maxLocalStorageNodes - 1) + "]",
MAX_LOCAL_STORAGE_NODES_SETTING.getKey(),
maxLocalStorageNodes);
throw new IllegalStateException(message, lastException);
}
this.nodeMetaData = loadOrCreateNodeMetaData(settings, startupTraceLogger, nodePaths);
this.logger =
Loggers.getLogger(
getClass(), Node.addNodeNameIfNeeded(settings, this.nodeMetaData.nodeId()));
this.nodeLockId = nodeLockId;
this.locks = locks;
this.nodePaths = nodePaths;
if (logger.isDebugEnabled()) {
logger.debug("using node location [{}], local_lock_id [{}]", nodePaths, nodeLockId);
}
maybeLogPathDetails();
maybeLogHeapDetails();
applySegmentInfosTrace(settings);
assertCanWrite();
success = true;
} finally {
if (success == false) {
IOUtils.closeWhileHandlingException(locks);
}
}
}
/** Returns true if the directory is empty */
private static boolean dirEmpty(final Path path) throws IOException {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
return stream.iterator().hasNext() == false;
}
}
private static void releaseAndNullLocks(Lock[] locks) {
for (int i = 0; i < locks.length; i++) {
if (locks[i] != null) {
IOUtils.closeWhileHandlingException(locks[i]);
}
locks[i] = null;
}
}
private void maybeLogPathDetails() throws IOException {
// We do some I/O in here, so skip this if DEBUG/INFO are not enabled:
if (logger.isDebugEnabled()) {
// Log one line per path.data:
StringBuilder sb = new StringBuilder();
for (NodePath nodePath : nodePaths) {
sb.append('\n').append(" -> ").append(nodePath.path.toAbsolutePath());
String spinsDesc;
if (nodePath.spins == null) {
spinsDesc = "unknown";
} else if (nodePath.spins) {
spinsDesc = "possibly";
} else {
spinsDesc = "no";
}
FsInfo.Path fsPath = FsProbe.getFSInfo(nodePath);
sb.append(", free_space [")
.append(fsPath.getFree())
.append("], usable_space [")
.append(fsPath.getAvailable())
.append("], total_space [")
.append(fsPath.getTotal())
.append("], spins? [")
.append(spinsDesc)
.append("], mount [")
.append(fsPath.getMount())
.append("], type [")
.append(fsPath.getType())
.append(']');
}
logger.debug("node data locations details:{}", sb);
} else if (logger.isInfoEnabled()) {
FsInfo.Path totFSPath = new FsInfo.Path();
Set<String> allTypes = new HashSet<>();
Set<String> allSpins = new HashSet<>();
Set<String> allMounts = new HashSet<>();
for (NodePath nodePath : nodePaths) {
FsInfo.Path fsPath = FsProbe.getFSInfo(nodePath);
String mount = fsPath.getMount();
if (allMounts.contains(mount) == false) {
allMounts.add(mount);
String type = fsPath.getType();
if (type != null) {
allTypes.add(type);
}
Boolean spins = fsPath.getSpins();
if (spins == null) {
allSpins.add("unknown");
} else if (spins.booleanValue()) {
allSpins.add("possibly");
} else {
allSpins.add("no");
}
totFSPath.add(fsPath);
}
}
// Just log a 1-line summary:
logger.info(
"using [{}] data paths, mounts [{}], net usable_space [{}], net total_space [{}], spins? [{}], types [{}]",
nodePaths.length,
allMounts,
totFSPath.getAvailable(),
totFSPath.getTotal(),
toString(allSpins),
toString(allTypes));
}
}
private void maybeLogHeapDetails() {
JvmInfo jvmInfo = JvmInfo.jvmInfo();
ByteSizeValue maxHeapSize = jvmInfo.getMem().getHeapMax();
String useCompressedOops = jvmInfo.useCompressedOops();
logger.info(
"heap size [{}], compressed ordinary object pointers [{}]", maxHeapSize, useCompressedOops);
}
/**
* scans the node paths and loads existing metaData file. If not found a new meta data will be
* generated and persisted into the nodePaths
*/
private static NodeMetaData loadOrCreateNodeMetaData(
Settings settings, Logger logger, NodePath... nodePaths) throws IOException {
final Path[] paths = Arrays.stream(nodePaths).map(np -> np.path).toArray(Path[]::new);
NodeMetaData metaData = NodeMetaData.FORMAT.loadLatestState(logger, paths);
if (metaData == null) {
metaData = new NodeMetaData(generateNodeId(settings));
}
// we write again to make sure all paths have the latest state file
NodeMetaData.FORMAT.write(metaData, paths);
return metaData;
}
public static String generateNodeId(Settings settings) {
Random random = Randomness.get(settings, NODE_ID_SEED_SETTING);
return UUIDs.randomBase64UUID(random);
}
@SuppressForbidden(reason = "System.out.*")
static void applySegmentInfosTrace(Settings settings) {
if (ENABLE_LUCENE_SEGMENT_INFOS_TRACE_SETTING.get(settings)) {
SegmentInfos.setInfoStream(System.out);
}
}
private static String toString(Collection<String> items) {
StringBuilder b = new StringBuilder();
for (String item : items) {
if (b.length() > 0) {
b.append(", ");
}
b.append(item);
}
return b.toString();
}
/**
* Deletes a shard data directory iff the shards locks were successfully acquired.
*
* @param shardId the id of the shard to delete to delete
* @throws IOException if an IOException occurs
*/
public void deleteShardDirectorySafe(ShardId shardId, IndexSettings indexSettings)
throws IOException, ShardLockObtainFailedException {
final Path[] paths = availableShardPaths(shardId);
logger.trace("deleting shard {} directory, paths: [{}]", shardId, paths);
try (ShardLock lock = shardLock(shardId)) {
deleteShardDirectoryUnderLock(lock, indexSettings);
}
}
/**
* Acquires, then releases, all {@code write.lock} files in the given shard paths. The
* "write.lock" file is assumed to be under the shard path's "index" directory as used by
* Elasticsearch.
*
* @throws LockObtainFailedException if any of the locks could not be acquired
*/
public static void acquireFSLockForPaths(IndexSettings indexSettings, Path... shardPaths)
throws IOException {
Lock[] locks = new Lock[shardPaths.length];
Directory[] dirs = new Directory[shardPaths.length];
try {
for (int i = 0; i < shardPaths.length; i++) {
// resolve the directory the shard actually lives in
Path p = shardPaths[i].resolve("index");
// open a directory (will be immediately closed) on the shard's location
dirs[i] =
new SimpleFSDirectory(
p, indexSettings.getValue(FsDirectoryService.INDEX_LOCK_FACTOR_SETTING));
// create a lock for the "write.lock" file
try {
locks[i] = dirs[i].obtainLock(IndexWriter.WRITE_LOCK_NAME);
} catch (IOException ex) {
throw new LockObtainFailedException(
"unable to acquire " + IndexWriter.WRITE_LOCK_NAME + " for " + p, ex);
}
}
} finally {
IOUtils.closeWhileHandlingException(locks);
IOUtils.closeWhileHandlingException(dirs);
}
}
/**
* Deletes a shard data directory. Note: this method assumes that the shard lock is acquired. This
* method will also attempt to acquire the write locks for the shard's paths before deleting the
* data, but this is best effort, as the lock is released before the deletion happens in order to
* allow the folder to be deleted
*
* @param lock the shards lock
* @throws IOException if an IOException occurs
* @throws ElasticsearchException if the write.lock is not acquirable
*/
public void deleteShardDirectoryUnderLock(ShardLock lock, IndexSettings indexSettings)
throws IOException {
final ShardId shardId = lock.getShardId();
assert isShardLocked(shardId) : "shard " + shardId + " is not locked";
final Path[] paths = availableShardPaths(shardId);
logger.trace("acquiring locks for {}, paths: [{}]", shardId, paths);
acquireFSLockForPaths(indexSettings, paths);
IOUtils.rm(paths);
if (indexSettings.hasCustomDataPath()) {
Path customLocation = resolveCustomLocation(indexSettings, shardId);
logger.trace("acquiring lock for {}, custom path: [{}]", shardId, customLocation);
acquireFSLockForPaths(indexSettings, customLocation);
logger.trace("deleting custom shard {} directory [{}]", shardId, customLocation);
IOUtils.rm(customLocation);
}
logger.trace("deleted shard {} directory, paths: [{}]", shardId, paths);
assert FileSystemUtils.exists(paths) == false;
}
private boolean isShardLocked(ShardId id) {
try {
shardLock(id, 0).close();
return false;
} catch (ShardLockObtainFailedException ex) {
return true;
}
}
/**
* Deletes an indexes data directory recursively iff all of the indexes shards locks were
* successfully acquired. If any of the indexes shard directories can't be locked non of the
* shards will be deleted
*
* @param index the index to delete
* @param lockTimeoutMS how long to wait for acquiring the indices shard locks
* @param indexSettings settings for the index being deleted
* @throws IOException if any of the shards data directories can't be locked or deleted
*/
public void deleteIndexDirectorySafe(Index index, long lockTimeoutMS, IndexSettings indexSettings)
throws IOException, ShardLockObtainFailedException {
final List<ShardLock> locks = lockAllForIndex(index, indexSettings, lockTimeoutMS);
try {
deleteIndexDirectoryUnderLock(index, indexSettings);
} finally {
IOUtils.closeWhileHandlingException(locks);
}
}
/**
* Deletes an indexes data directory recursively. Note: this method assumes that the shard lock is
* acquired
*
* @param index the index to delete
* @param indexSettings settings for the index being deleted
*/
public void deleteIndexDirectoryUnderLock(Index index, IndexSettings indexSettings)
throws IOException {
final Path[] indexPaths = indexPaths(index);
logger.trace(
"deleting index {} directory, paths({}): [{}]", index, indexPaths.length, indexPaths);
IOUtils.rm(indexPaths);
if (indexSettings.hasCustomDataPath()) {
Path customLocation = resolveIndexCustomLocation(indexSettings);
logger.trace("deleting custom index {} directory [{}]", index, customLocation);
IOUtils.rm(customLocation);
}
}
/**
* Tries to lock all local shards for the given index. If any of the shard locks can't be acquired
* a {@link ShardLockObtainFailedException} is thrown and all previously acquired locks are
* released.
*
* @param index the index to lock shards for
* @param lockTimeoutMS how long to wait for acquiring the indices shard locks
* @return the {@link ShardLock} instances for this index.
* @throws IOException if an IOException occurs.
*/
public List<ShardLock> lockAllForIndex(Index index, IndexSettings settings, long lockTimeoutMS)
throws IOException, ShardLockObtainFailedException {
final int numShards = settings.getNumberOfShards();
if (numShards <= 0) {
throw new IllegalArgumentException("settings must contain a non-null > 0 number of shards");
}
logger.trace("locking all shards for index {} - [{}]", index, numShards);
List<ShardLock> allLocks = new ArrayList<>(numShards);
boolean success = false;
long startTimeNS = System.nanoTime();
try {
for (int i = 0; i < numShards; i++) {
long timeoutLeftMS =
Math.max(0, lockTimeoutMS - TimeValue.nsecToMSec((System.nanoTime() - startTimeNS)));
allLocks.add(shardLock(new ShardId(index, i), timeoutLeftMS));
}
success = true;
} finally {
if (success == false) {
logger.trace("unable to lock all shards for index {}", index);
IOUtils.closeWhileHandlingException(allLocks);
}
}
return allLocks;
}
/**
* Tries to lock the given shards ID. A shard lock is required to perform any kind of write
* operation on a shards data directory like deleting files, creating a new index writer or
* recover from a different shard instance into it. If the shard lock can not be acquired a {@link
* ShardLockObtainFailedException} is thrown.
*
* <p>Note: this method will return immediately if the lock can't be acquired.
*
* @param id the shard ID to lock
* @return the shard lock. Call {@link ShardLock#close()} to release the lock
*/
public ShardLock shardLock(ShardId id) throws ShardLockObtainFailedException {
return shardLock(id, 0);
}
/**
* Tries to lock the given shards ID. A shard lock is required to perform any kind of write
* operation on a shards data directory like deleting files, creating a new index writer or
* recover from a different shard instance into it. If the shard lock can not be acquired a {@link
* ShardLockObtainFailedException} is thrown
*
* @param shardId the shard ID to lock
* @param lockTimeoutMS the lock timeout in milliseconds
* @return the shard lock. Call {@link ShardLock#close()} to release the lock
*/
public ShardLock shardLock(final ShardId shardId, long lockTimeoutMS)
throws ShardLockObtainFailedException {
logger.trace("acquiring node shardlock on [{}], timeout [{}]", shardId, lockTimeoutMS);
final InternalShardLock shardLock;
final boolean acquired;
synchronized (shardLocks) {
if (shardLocks.containsKey(shardId)) {
shardLock = shardLocks.get(shardId);
shardLock.incWaitCount();
acquired = false;
} else {
shardLock = new InternalShardLock(shardId);
shardLocks.put(shardId, shardLock);
acquired = true;
}
}
if (acquired == false) {
boolean success = false;
try {
shardLock.acquire(lockTimeoutMS);
success = true;
} finally {
if (success == false) {
shardLock.decWaitCount();
}
}
}
logger.trace("successfully acquired shardlock for [{}]", shardId);
return new ShardLock(shardId) { // new instance prevents double closing
@Override
protected void closeInternal() {
shardLock.release();
logger.trace("released shard lock for [{}]", shardId);
}
};
}
/** A functional interface that people can use to reference {@link #shardLock(ShardId, long)} */
@FunctionalInterface
public interface ShardLocker {
ShardLock lock(ShardId shardId, long lockTimeoutMS) throws ShardLockObtainFailedException;
}
/**
* Returns all currently lock shards.
*
* <p>Note: the shard ids return do not contain a valid Index UUID
*/
public Set<ShardId> lockedShards() {
synchronized (shardLocks) {
return unmodifiableSet(new HashSet<>(shardLocks.keySet()));
}
}
private final class InternalShardLock {
/*
* This class holds a mutex for exclusive access and timeout / wait semantics
* and a reference count to cleanup the shard lock instance form the internal data
* structure if nobody is waiting for it. the wait count is guarded by the same lock
* that is used to mutate the map holding the shard locks to ensure exclusive access
*/
private final Semaphore mutex = new Semaphore(1);
private int waitCount = 1; // guarded by shardLocks
private final ShardId shardId;
InternalShardLock(ShardId shardId) {
this.shardId = shardId;
mutex.acquireUninterruptibly();
}
protected void release() {
mutex.release();
decWaitCount();
}
void incWaitCount() {
synchronized (shardLocks) {
assert waitCount > 0 : "waitCount is " + waitCount + " but should be > 0";
waitCount++;
}
}
private void decWaitCount() {
synchronized (shardLocks) {
assert waitCount > 0 : "waitCount is " + waitCount + " but should be > 0";
--waitCount;
logger.trace("shard lock wait count for {} is now [{}]", shardId, waitCount);
if (waitCount == 0) {
logger.trace("last shard lock wait decremented, removing lock for {}", shardId);
InternalShardLock remove = shardLocks.remove(shardId);
assert remove != null : "Removed lock was null";
}
}
}
void acquire(long timeoutInMillis) throws ShardLockObtainFailedException {
try {
if (mutex.tryAcquire(timeoutInMillis, TimeUnit.MILLISECONDS) == false) {
throw new ShardLockObtainFailedException(
shardId, "obtaining shard lock timed out after " + timeoutInMillis + "ms");
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new ShardLockObtainFailedException(
shardId, "thread interrupted while trying to obtain shard lock", e);
}
}
}
public boolean hasNodeFile() {
return nodePaths != null && locks != null;
}
/**
* Returns an array of all of the nodes data locations.
*
* @throws IllegalStateException if the node is not configured to store local locations
*/
public Path[] nodeDataPaths() {
assertEnvIsLocked();
Path[] paths = new Path[nodePaths.length];
for (int i = 0; i < paths.length; i++) {
paths[i] = nodePaths[i].path;
}
return paths;
}
/**
* returns the unique uuid describing this node. The uuid is persistent in the data folder of this
* node and remains across restarts.
*/
public String nodeId() {
// we currently only return the ID and hide the underlying nodeMetaData implementation in order
// to avoid
// confusion with other "metadata" like node settings found in elasticsearch.yml. In future
// we can encapsulate both (and more) in one NodeMetaData (or NodeSettings) object ala
// IndexSettings
return nodeMetaData.nodeId();
}
/** Returns an array of all of the {@link NodePath}s. */
public NodePath[] nodePaths() {
assertEnvIsLocked();
if (nodePaths == null || locks == null) {
throw new IllegalStateException("node is not configured to store local location");
}
return nodePaths;
}
/** Returns all index paths. */
public Path[] indexPaths(Index index) {
assertEnvIsLocked();
Path[] indexPaths = new Path[nodePaths.length];
for (int i = 0; i < nodePaths.length; i++) {
indexPaths[i] = nodePaths[i].resolve(index);
}
return indexPaths;
}
/**
* Returns all shard paths excluding custom shard path. Note: Shards are only allocated on one of
* the returned paths. The returned array may contain paths to non-existing directories.
*
* @see IndexSettings#hasCustomDataPath()
* @see #resolveCustomLocation(IndexSettings, ShardId)
*/
public Path[] availableShardPaths(ShardId shardId) {
assertEnvIsLocked();
final NodePath[] nodePaths = nodePaths();
final Path[] shardLocations = new Path[nodePaths.length];
for (int i = 0; i < nodePaths.length; i++) {
shardLocations[i] = nodePaths[i].resolve(shardId);
}
return shardLocations;
}
/** Returns all folder names in ${data.paths}/nodes/{node.id}/indices folder */
public Set<String> availableIndexFolders() throws IOException {
if (nodePaths == null || locks == null) {
throw new IllegalStateException("node is not configured to store local location");
}
assertEnvIsLocked();
Set<String> indexFolders = new HashSet<>();
for (NodePath nodePath : nodePaths) {
Path indicesLocation = nodePath.indicesPath;
if (Files.isDirectory(indicesLocation)) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(indicesLocation)) {
for (Path index : stream) {
if (Files.isDirectory(index)) {
indexFolders.add(index.getFileName().toString());
}
}
}
}
}
return indexFolders;
}
/**
* Resolves all existing paths to <code>indexFolderName</code> in
* ${data.paths}/nodes/{node.id}/indices
*/
public Path[] resolveIndexFolder(String indexFolderName) throws IOException {
if (nodePaths == null || locks == null) {
throw new IllegalStateException("node is not configured to store local location");
}
assertEnvIsLocked();
List<Path> paths = new ArrayList<>(nodePaths.length);
for (NodePath nodePath : nodePaths) {
Path indexFolder = nodePath.indicesPath.resolve(indexFolderName);
if (Files.exists(indexFolder)) {
paths.add(indexFolder);
}
}
return paths.toArray(new Path[paths.size()]);
}
/**
* Tries to find all allocated shards for the given index on the current node. NOTE: This methods
* is prone to race-conditions on the filesystem layer since it might not see directories created
* concurrently or while it's traversing.
*
* @param index the index to filter shards
* @return a set of shard IDs
* @throws IOException if an IOException occurs
*/
public Set<ShardId> findAllShardIds(final Index index) throws IOException {
assert index != null;
if (nodePaths == null || locks == null) {
throw new IllegalStateException("node is not configured to store local location");
}
assertEnvIsLocked();
final Set<ShardId> shardIds = new HashSet<>();
final String indexUniquePathId = index.getUUID();
for (final NodePath nodePath : nodePaths) {
Path location = nodePath.indicesPath;
if (Files.isDirectory(location)) {
try (DirectoryStream<Path> indexStream = Files.newDirectoryStream(location)) {
for (Path indexPath : indexStream) {
if (indexUniquePathId.equals(indexPath.getFileName().toString())) {
shardIds.addAll(findAllShardsForIndex(indexPath, index));
}
}
}
}
}
return shardIds;
}
private static Set<ShardId> findAllShardsForIndex(Path indexPath, Index index)
throws IOException {
assert indexPath.getFileName().toString().equals(index.getUUID());
Set<ShardId> shardIds = new HashSet<>();
if (Files.isDirectory(indexPath)) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(indexPath)) {
for (Path shardPath : stream) {
String fileName = shardPath.getFileName().toString();
if (Files.isDirectory(shardPath) && fileName.chars().allMatch(Character::isDigit)) {
int shardId = Integer.parseInt(fileName);
ShardId id = new ShardId(index, shardId);
shardIds.add(id);
}
}
}
}
return shardIds;
}
@Override
public void close() {
if (closed.compareAndSet(false, true) && locks != null) {
for (Lock lock : locks) {
try {
logger.trace("releasing lock [{}]", lock);
lock.close();
} catch (IOException e) {
logger.trace(
(Supplier<?>) () -> new ParameterizedMessage("failed to release lock [{}]", lock), e);
}
}
}
}
private void assertEnvIsLocked() {
if (!closed.get() && locks != null) {
for (Lock lock : locks) {
try {
lock.ensureValid();
} catch (IOException e) {
logger.warn("lock assertion failed", e);
throw new IllegalStateException("environment is not locked", e);
}
}
}
}
/**
* This method tries to write an empty file and moves it using an atomic move operation. This
* method throws an {@link IllegalStateException} if this operation is not supported by the
* filesystem. This test is executed on each of the data directories. This method cleans up all
* files even in the case of an error.
*/
public void ensureAtomicMoveSupported() throws IOException {
final NodePath[] nodePaths = nodePaths();
for (NodePath nodePath : nodePaths) {
assert Files.isDirectory(nodePath.path) : nodePath.path + " is not a directory";
final Path src = nodePath.path.resolve("__es__.tmp");
final Path target = nodePath.path.resolve("__es__.final");
try {
Files.createFile(src);
Files.move(src, target, StandardCopyOption.ATOMIC_MOVE);
} catch (AtomicMoveNotSupportedException ex) {
throw new IllegalStateException(
"atomic_move is not supported by the filesystem on path ["
+ nodePath.path
+ "] atomic_move is required for elasticsearch to work correctly.",
ex);
} finally {
try {
Files.deleteIfExists(src);
} finally {
Files.deleteIfExists(target);
}
}
}
}
/**
* Resolve the custom path for a index's shard. Uses the {@code IndexMetaData.SETTING_DATA_PATH}
* setting to determine the root path for the index.
*
* @param indexSettings settings for the index
*/
public Path resolveBaseCustomLocation(IndexSettings indexSettings) {
String customDataDir = indexSettings.customDataPath();
if (customDataDir != null) {
// This assert is because this should be caught by MetaDataCreateIndexService
assert sharedDataPath != null;
if (ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.get(indexSettings.getNodeSettings())) {
return sharedDataPath.resolve(customDataDir).resolve(Integer.toString(this.nodeLockId));
} else {
return sharedDataPath.resolve(customDataDir);
}
} else {
throw new IllegalArgumentException(
"no custom " + IndexMetaData.SETTING_DATA_PATH + " setting available");
}
}
/**
* Resolve the custom path for a index's shard. Uses the {@code IndexMetaData.SETTING_DATA_PATH}
* setting to determine the root path for the index.
*
* @param indexSettings settings for the index
*/
private Path resolveIndexCustomLocation(IndexSettings indexSettings) {
return resolveBaseCustomLocation(indexSettings).resolve(indexSettings.getUUID());
}
/**
* Resolve the custom path for a index's shard. Uses the {@code IndexMetaData.SETTING_DATA_PATH}
* setting to determine the root path for the index.
*
* @param indexSettings settings for the index
* @param shardId shard to resolve the path to
*/
public Path resolveCustomLocation(IndexSettings indexSettings, final ShardId shardId) {
return resolveIndexCustomLocation(indexSettings).resolve(Integer.toString(shardId.id()));
}
/** Returns the {@code NodePath.path} for this shard. */
public static Path shardStatePathToDataPath(Path shardPath) {
int count = shardPath.getNameCount();
// Sanity check:
assert Integer.parseInt(shardPath.getName(count - 1).toString()) >= 0;
assert "indices".equals(shardPath.getName(count - 3).toString());
return shardPath.getParent().getParent().getParent();
}
/**
* This is a best effort to ensure that we actually have write permissions to write in all our
* data directories. This prevents disasters if nodes are started under the wrong username etc.
*/
private void assertCanWrite() throws IOException {
for (Path path : nodeDataPaths()) { // check node-paths are writable
tryWriteTempFile(path);
}
for (String indexFolderName : this.availableIndexFolders()) {
for (Path indexPath :
this.resolveIndexFolder(indexFolderName)) { // check index paths are writable
Path indexStatePath = indexPath.resolve(MetaDataStateFormat.STATE_DIR_NAME);
tryWriteTempFile(indexStatePath);
tryWriteTempFile(indexPath);
try (DirectoryStream<Path> stream = Files.newDirectoryStream(indexPath)) {
for (Path shardPath : stream) {
String fileName = shardPath.getFileName().toString();
if (Files.isDirectory(shardPath) && fileName.chars().allMatch(Character::isDigit)) {
Path indexDir = shardPath.resolve(ShardPath.INDEX_FOLDER_NAME);
Path statePath = shardPath.resolve(MetaDataStateFormat.STATE_DIR_NAME);
Path translogDir = shardPath.resolve(ShardPath.TRANSLOG_FOLDER_NAME);
tryWriteTempFile(indexDir);
tryWriteTempFile(translogDir);
tryWriteTempFile(statePath);
tryWriteTempFile(shardPath);
}
}
}
}
}
}
private static void tryWriteTempFile(Path path) throws IOException {
if (Files.exists(path)) {
Path resolve = path.resolve(".es_temp_file");
try {
Files.createFile(resolve);
Files.deleteIfExists(resolve);
} catch (IOException ex) {
throw new IOException(
"failed to write in data directory [" + path + "] write permission is required", ex);
}
}
}
}
/**
* IndexModule represents the central extension point for index level custom implementations like:
*
* <ul>
* <li>{@link SimilarityProvider} - New {@link SimilarityProvider} implementations can be
* registered through {@link #addSimilarity(String, BiFunction)} while existing Providers can
* be referenced through Settings under the {@link IndexModule#SIMILARITY_SETTINGS_PREFIX}
* prefix along with the "type" value. For example, to reference the {@link
* BM25SimilarityProvider}, the configuration <tt>"index.similarity.my_similarity.type :
* "BM25"</tt> can be used.
* <li>{@link IndexStore} - Custom {@link IndexStore} instances can be registered via {@link
* #addIndexStore(String, BiFunction)}
* <li>{@link IndexEventListener} - Custom {@link IndexEventListener} instances can be registered
* via {@link #addIndexEventListener(IndexEventListener)}
* <li>Settings update listener - Custom settings update listener can be registered via {@link
* #addSettingsUpdateConsumer(Setting, Consumer)}
* </ul>
*/
public final class IndexModule {
public static final Setting<String> INDEX_STORE_TYPE_SETTING =
new Setting<>("index.store.type", "", Function.identity(), false, Setting.Scope.INDEX);
public static final String SIMILARITY_SETTINGS_PREFIX = "index.similarity";
public static final String INDEX_QUERY_CACHE = "index";
public static final String NONE_QUERY_CACHE = "none";
public static final Setting<String> INDEX_QUERY_CACHE_TYPE_SETTING =
new Setting<>(
"index.queries.cache.type",
INDEX_QUERY_CACHE,
Function.identity(),
false,
Setting.Scope.INDEX);
// for test purposes only
public static final Setting<Boolean> INDEX_QUERY_CACHE_EVERYTHING_SETTING =
Setting.boolSetting("index.queries.cache.everything", false, false, Setting.Scope.INDEX);
private final IndexSettings indexSettings;
private final IndexStoreConfig indexStoreConfig;
private final AnalysisRegistry analysisRegistry;
// pkg private so tests can mock
final SetOnce<EngineFactory> engineFactory = new SetOnce<>();
private SetOnce<IndexSearcherWrapperFactory> indexSearcherWrapper = new SetOnce<>();
private final Set<IndexEventListener> indexEventListeners = new HashSet<>();
private IndexEventListener listener;
private final Map<String, BiFunction<String, Settings, SimilarityProvider>> similarities =
new HashMap<>();
private final Map<String, BiFunction<IndexSettings, IndexStoreConfig, IndexStore>> storeTypes =
new HashMap<>();
private final Map<String, BiFunction<IndexSettings, IndicesQueryCache, QueryCache>> queryCaches =
new HashMap<>();
public IndexModule(
IndexSettings indexSettings,
IndexStoreConfig indexStoreConfig,
AnalysisRegistry analysisRegistry) {
this.indexStoreConfig = indexStoreConfig;
this.indexSettings = indexSettings;
this.analysisRegistry = analysisRegistry;
registerQueryCache(INDEX_QUERY_CACHE, IndexQueryCache::new);
registerQueryCache(NONE_QUERY_CACHE, (a, b) -> new NoneQueryCache(a));
}
/** Adds a Setting and it's consumer for this index. */
public <T> void addSettingsUpdateConsumer(Setting<T> setting, Consumer<T> consumer) {
if (setting == null) {
throw new IllegalArgumentException("setting must not be null");
}
indexSettings.getScopedSettings().addSettingsUpdateConsumer(setting, consumer);
}
/** Returns the index {@link Settings} for this index */
public Settings getSettings() {
return indexSettings.getSettings();
}
/** Returns the index this module is associated with */
public Index getIndex() {
return indexSettings.getIndex();
}
/**
* Adds an {@link IndexEventListener} for this index. All listeners added here are maintained for
* the entire index lifecycle on this node. Once an index is closed or deleted these listeners go
* out of scope.
*
* <p>Note: an index might be created on a node multiple times. For instance if the last shard
* from an index is relocated to another node the internal representation will be destroyed which
* includes the registered listeners. Once the node holds at least one shard of an index all
* modules are reloaded and listeners are registered again. Listeners can't be unregistered they
* will stay alive for the entire time the index is allocated on a node.
*/
public void addIndexEventListener(IndexEventListener listener) {
if (this.listener != null) {
throw new IllegalStateException("can't add listener after listeners are frozen");
}
if (listener == null) {
throw new IllegalArgumentException("listener must not be null");
}
if (indexEventListeners.contains(listener)) {
throw new IllegalArgumentException("listener already added");
}
this.indexEventListeners.add(listener);
}
/**
* Adds an {@link IndexStore} type to this index module. Typically stores are registered with a
* refrence to it's constructor:
*
* <pre>
* indexModule.addIndexStore("my_store_type", MyStore::new);
* </pre>
*
* @param type the type to register
* @param provider the instance provider / factory method
*/
public void addIndexStore(
String type, BiFunction<IndexSettings, IndexStoreConfig, IndexStore> provider) {
if (storeTypes.containsKey(type)) {
throw new IllegalArgumentException("key [" + type + "] already registerd");
}
storeTypes.put(type, provider);
}
/**
* Registers the given {@link SimilarityProvider} with the given name
*
* @param name Name of the SimilarityProvider
* @param similarity SimilarityProvider to register
*/
public void addSimilarity(
String name, BiFunction<String, Settings, SimilarityProvider> similarity) {
if (similarities.containsKey(name) || SimilarityService.BUILT_IN.containsKey(name)) {
throw new IllegalArgumentException(
"similarity for name: [" + name + " is already registered");
}
similarities.put(name, similarity);
}
/**
* Registers a {@link QueryCache} provider for a given name
*
* @param name the providers / caches name
* @param provider the provider instance
*/
public void registerQueryCache(
String name, BiFunction<IndexSettings, IndicesQueryCache, QueryCache> provider) {
if (provider == null) {
throw new IllegalArgumentException("provider must not be null");
}
if (queryCaches.containsKey(name)) {
throw new IllegalArgumentException(
"Can't register the same [query_cache] more than once for [" + name + "]");
}
queryCaches.put(name, provider);
}
/**
* Sets a {@link org.elasticsearch.index.IndexModule.IndexSearcherWrapperFactory} that is called
* once the IndexService is fully constructed. Note: this method can only be called once per
* index. Multiple wrappers are not supported.
*/
public void setSearcherWrapper(IndexSearcherWrapperFactory indexSearcherWrapperFactory) {
this.indexSearcherWrapper.set(indexSearcherWrapperFactory);
}
public IndexEventListener freeze() {
// TODO somehow we need to make this pkg private...
if (listener == null) {
listener = new CompositeIndexEventListener(indexSettings, indexEventListeners);
}
return listener;
}
private static boolean isBuiltinType(String storeType) {
for (Type type : Type.values()) {
if (type.match(storeType)) {
return true;
}
}
return false;
}
public enum Type {
NIOFS,
MMAPFS,
SIMPLEFS,
FS,
DEFAULT;
public String getSettingsKey() {
return this.name().toLowerCase(Locale.ROOT);
}
/** Returns true iff this settings matches the type. */
public boolean match(String setting) {
return getSettingsKey().equals(setting);
}
}
/** Factory for creating new {@link IndexSearcherWrapper} instances */
public interface IndexSearcherWrapperFactory {
/** Returns a new IndexSearcherWrapper. This method is called once per index per node */
IndexSearcherWrapper newWrapper(final IndexService indexService);
}
public IndexService newIndexService(
NodeEnvironment environment,
IndexService.ShardStoreDeleter shardStoreDeleter,
NodeServicesProvider servicesProvider,
MapperRegistry mapperRegistry,
IndexingOperationListener... listeners)
throws IOException {
IndexSearcherWrapperFactory searcherWrapperFactory =
indexSearcherWrapper.get() == null ? (shard) -> null : indexSearcherWrapper.get();
IndexEventListener eventListener = freeze();
final String storeType = indexSettings.getValue(INDEX_STORE_TYPE_SETTING);
final IndexStore store;
if (Strings.isEmpty(storeType) || isBuiltinType(storeType)) {
store = new IndexStore(indexSettings, indexStoreConfig);
} else {
BiFunction<IndexSettings, IndexStoreConfig, IndexStore> factory = storeTypes.get(storeType);
if (factory == null) {
throw new IllegalArgumentException("Unknown store type [" + storeType + "]");
}
store = factory.apply(indexSettings, indexStoreConfig);
if (store == null) {
throw new IllegalStateException("store must not be null");
}
}
indexSettings
.getScopedSettings()
.addSettingsUpdateConsumer(
IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING, store::setMaxRate);
indexSettings
.getScopedSettings()
.addSettingsUpdateConsumer(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING, store::setType);
final String queryCacheType = indexSettings.getValue(INDEX_QUERY_CACHE_TYPE_SETTING);
final BiFunction<IndexSettings, IndicesQueryCache, QueryCache> queryCacheProvider =
queryCaches.get(queryCacheType);
final QueryCache queryCache =
queryCacheProvider.apply(indexSettings, servicesProvider.getIndicesQueryCache());
return new IndexService(
indexSettings,
environment,
new SimilarityService(indexSettings, similarities),
shardStoreDeleter,
analysisRegistry,
engineFactory.get(),
servicesProvider,
queryCache,
store,
eventListener,
searcherWrapperFactory,
mapperRegistry,
listeners);
}
}
