Пример #1
0
  /*
   * Finds all mappings for types and concrete indices. Types are expanded to
   * include all types that match the glob patterns in the types array. Empty
   * types array, null or {"_all"} will be expanded to all types available for
   * the given indices.
   */
  public ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> findMappings(
      String[] concreteIndices, final String[] types) {
    assert types != null;
    assert concreteIndices != null;
    if (concreteIndices.length == 0) {
      return ImmutableOpenMap.of();
    }

    ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder =
        ImmutableOpenMap.builder();
    Iterable<String> intersection =
        HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys());
    for (String index : intersection) {
      IndexMetaData indexMetaData = indices.get(index);
      ImmutableOpenMap.Builder<String, MappingMetaData> filteredMappings;
      if (isAllTypes(types)) {
        indexMapBuilder.put(
            index, indexMetaData.getMappings()); // No types specified means get it all

      } else {
        filteredMappings = ImmutableOpenMap.builder();
        for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.getMappings()) {
          if (Regex.simpleMatch(types, cursor.key)) {
            filteredMappings.put(cursor.key, cursor.value);
          }
        }
        if (!filteredMappings.isEmpty()) {
          indexMapBuilder.put(index, filteredMappings.build());
        }
      }
    }
    return indexMapBuilder.build();
  }
Пример #2
0
 public Builder put(IndexMetaData indexMetaData, boolean incrementVersion) {
   if (indices.get(indexMetaData.getIndex().getName()) == indexMetaData) {
     return this;
   }
   // if we put a new index metadata, increment its version
   if (incrementVersion) {
     indexMetaData =
         IndexMetaData.builder(indexMetaData).version(indexMetaData.getVersion() + 1).build();
   }
   indices.put(indexMetaData.getIndex().getName(), indexMetaData);
   return this;
 }
Пример #3
0
 protected void addMappers(
     Collection<ObjectMapper> objectMappers, Collection<FieldMapper> fieldMappers) {
   assert mappingLock.isWriteLockedByCurrentThread();
   ImmutableOpenMap.Builder<String, ObjectMapper> fullPathObjectMappers =
       ImmutableOpenMap.builder(this.fullPathObjectMappers);
   for (ObjectMapper objectMapper : objectMappers) {
     fullPathObjectMappers.put(objectMapper.fullPath(), objectMapper);
     if (objectMapper.nested().isNested()) {
       hasNested = true;
     }
   }
   this.fullPathObjectMappers = fullPathObjectMappers.build();
   this.fieldTypes = this.fieldTypes.copyAndAddAll(fieldMappers);
 }
 private MetaData buildMetaDataForVersion(MetaData metaData, long version) {
   ImmutableOpenMap.Builder<String, IndexMetaData> indices =
       ImmutableOpenMap.builder(metaData.indices());
   indices.put(
       "test" + version,
       IndexMetaData.builder("test" + version)
           .settings(
               Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
           .numberOfShards((int) version)
           .numberOfReplicas(0)
           .build());
   return MetaData.builder(metaData)
       .transientSettings(Settings.builder().put("test", version).build())
       .indices(indices.build())
       .build();
 }
Пример #5
0
 public Builder put(IndexMetaData.Builder indexMetaDataBuilder) {
   // we know its a new one, increment the version and store
   indexMetaDataBuilder.version(indexMetaDataBuilder.version() + 1);
   IndexMetaData indexMetaData = indexMetaDataBuilder.build();
   indices.put(indexMetaData.getIndex().getName(), indexMetaData);
   return this;
 }
 @Override
 public void readFrom(StreamInput in) throws IOException {
   super.readFrom(in);
   int size = in.readVInt();
   ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder =
       ImmutableOpenMap.builder();
   for (int i = 0; i < size; i++) {
     String key = in.readString();
     int valueSize = in.readVInt();
     ImmutableOpenMap.Builder<String, MappingMetaData> typeMapBuilder = ImmutableOpenMap.builder();
     for (int j = 0; j < valueSize; j++) {
       typeMapBuilder.put(in.readString(), new MappingMetaData(in));
     }
     indexMapBuilder.put(key, typeMapBuilder.build());
   }
   mappings = indexMapBuilder.build();
 }
Пример #7
0
 StoreDirectory(Distributor distributor) throws IOException {
   this.distributor = distributor;
   synchronized (mutex) {
     ImmutableOpenMap.Builder<String, StoreFileMetaData> builder = ImmutableOpenMap.builder();
     Map<String, String> checksums =
         readChecksums(distributor.all(), new HashMap<String, String>());
     for (Directory delegate : distributor.all()) {
       for (String file : delegate.listAll()) {
         String checksum = checksums.get(file);
         builder.put(
             file, new StoreFileMetaData(file, delegate.fileLength(file), checksum, delegate));
       }
     }
     filesMetadata = builder.build();
     files = filesMetadata.keys().toArray(String.class);
   }
 }
Пример #8
0
 private void addObjectMappers(ObjectMapper[] objectMappers) {
   synchronized (mappersMutex) {
     ImmutableOpenMap.Builder<String, ObjectMappers> fullPathObjectMappers =
         ImmutableOpenMap.builder(this.fullPathObjectMappers);
     for (ObjectMapper objectMapper : objectMappers) {
       ObjectMappers mappers = fullPathObjectMappers.get(objectMapper.fullPath());
       if (mappers == null) {
         mappers = new ObjectMappers(objectMapper);
       } else {
         mappers = mappers.concat(objectMapper);
       }
       fullPathObjectMappers.put(objectMapper.fullPath(), mappers);
       // update the hasNested flag
       if (objectMapper.nested().isNested()) {
         hasNested = true;
       }
     }
     this.fullPathObjectMappers = fullPathObjectMappers.build();
   }
 }
Пример #9
0
  private void removeObjectAndFieldMappers(DocumentMapper docMapper) {
    synchronized (mappersMutex) {
      fieldMappers.removeMappers(docMapper.mappers());

      ImmutableOpenMap.Builder<String, ObjectMappers> fullPathObjectMappers =
          ImmutableOpenMap.builder(this.fullPathObjectMappers);
      for (ObjectMapper mapper : docMapper.objectMappers().values()) {
        ObjectMappers mappers = fullPathObjectMappers.get(mapper.fullPath());
        if (mappers != null) {
          mappers = mappers.remove(mapper);
          if (mappers.isEmpty()) {
            fullPathObjectMappers.remove(mapper.fullPath());
          } else {
            fullPathObjectMappers.put(mapper.fullPath(), mappers);
          }
        }
      }

      this.fullPathObjectMappers = fullPathObjectMappers.build();
    }
  }
Пример #10
0
 public Builder putCustom(String type, Custom custom) {
   customs.put(type, custom);
   return this;
 }
Пример #11
0
 public Builder removeCustom(String type) {
   customs.remove(type);
   return this;
 }
Пример #12
0
 public Builder removeTemplate(String templateName) {
   templates.remove(templateName);
   return this;
 }
Пример #13
0
 public Custom getCustom(String type) {
   return customs.get(type);
 }
Пример #14
0
 public Builder removeAllIndices() {
   indices.clear();
   return this;
 }
Пример #15
0
 public Builder put(IndexTemplateMetaData template) {
   templates.put(template.name(), template);
   return this;
 }
Пример #16
0
 public Builder putMapping(MappingMetaData mappingMd) {
   mappings.put(mappingMd.type(), mappingMd);
   return this;
 }
Пример #17
0
 public Builder remove(String index) {
   indices.remove(index);
   return this;
 }
Пример #18
0
 public Builder putAlias(AliasMetaData.Builder aliasMetaData) {
   aliases.put(aliasMetaData.alias(), aliasMetaData.build());
   return this;
 }
Пример #19
0
 public Builder removeAlias(String alias) {
   aliases.remove(alias);
   return this;
 }
Пример #20
0
 public Builder removeAllAliases() {
   aliases.clear();
   return this;
 }
Пример #21
0
    public MetaData build() {
      // TODO: We should move these datastructures to IndexNameExpressionResolver, this will give
      // the following benefits:
      // 1) The datastructures will only be rebuilded when needed. Now during serializing we rebuild
      // these datastructures
      //    while these datastructures aren't even used.
      // 2) The aliasAndIndexLookup can be updated instead of rebuilding it all the time.

      // build all concrete indices arrays:
      // TODO: I think we can remove these arrays. it isn't worth the effort, for operations on all
      // indices.
      // When doing an operation across all indices, most of the time is spent on actually going to
      // all shards and
      // do the required operations, the bottleneck isn't resolving expressions into concrete
      // indices.
      List<String> allIndicesLst = new ArrayList<>();
      for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
        allIndicesLst.add(cursor.value.getIndex().getName());
      }
      String[] allIndices = allIndicesLst.toArray(new String[allIndicesLst.size()]);

      List<String> allOpenIndicesLst = new ArrayList<>();
      List<String> allClosedIndicesLst = new ArrayList<>();
      for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
        IndexMetaData indexMetaData = cursor.value;
        if (indexMetaData.getState() == IndexMetaData.State.OPEN) {
          allOpenIndicesLst.add(indexMetaData.getIndex().getName());
        } else if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
          allClosedIndicesLst.add(indexMetaData.getIndex().getName());
        }
      }
      String[] allOpenIndices = allOpenIndicesLst.toArray(new String[allOpenIndicesLst.size()]);
      String[] allClosedIndices =
          allClosedIndicesLst.toArray(new String[allClosedIndicesLst.size()]);

      // build all indices map
      SortedMap<String, AliasOrIndex> aliasAndIndexLookup = new TreeMap<>();
      for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
        IndexMetaData indexMetaData = cursor.value;
        aliasAndIndexLookup.put(
            indexMetaData.getIndex().getName(), new AliasOrIndex.Index(indexMetaData));

        for (ObjectObjectCursor<String, AliasMetaData> aliasCursor : indexMetaData.getAliases()) {
          AliasMetaData aliasMetaData = aliasCursor.value;
          AliasOrIndex aliasOrIndex = aliasAndIndexLookup.get(aliasMetaData.getAlias());
          if (aliasOrIndex == null) {
            aliasOrIndex = new AliasOrIndex.Alias(aliasMetaData, indexMetaData);
            aliasAndIndexLookup.put(aliasMetaData.getAlias(), aliasOrIndex);
          } else if (aliasOrIndex instanceof AliasOrIndex.Alias) {
            AliasOrIndex.Alias alias = (AliasOrIndex.Alias) aliasOrIndex;
            alias.addIndex(indexMetaData);
          } else if (aliasOrIndex instanceof AliasOrIndex.Index) {
            AliasOrIndex.Index index = (AliasOrIndex.Index) aliasOrIndex;
            throw new IllegalStateException(
                "index and alias names need to be unique, but alias ["
                    + aliasMetaData.getAlias()
                    + "] and index "
                    + index.getIndex().getIndex()
                    + " have the same name");
          } else {
            throw new IllegalStateException(
                "unexpected alias [" + aliasMetaData.getAlias() + "][" + aliasOrIndex + "]");
          }
        }
      }
      aliasAndIndexLookup = Collections.unmodifiableSortedMap(aliasAndIndexLookup);
      return new MetaData(
          clusterUUID,
          version,
          transientSettings,
          persistentSettings,
          indices.build(),
          templates.build(),
          customs.build(),
          allIndices,
          allOpenIndices,
          allClosedIndices,
          aliasAndIndexLookup);
    }
Пример #22
0
    public IndexMetaData build() {
      ImmutableOpenMap.Builder<String, AliasMetaData> tmpAliases = aliases;
      Settings tmpSettings = settings;

      // update default mapping on the MappingMetaData
      if (mappings.containsKey(MapperService.DEFAULT_MAPPING)) {
        MappingMetaData defaultMapping = mappings.get(MapperService.DEFAULT_MAPPING);
        for (ObjectCursor<MappingMetaData> cursor : mappings.values()) {
          cursor.value.updateDefaultMapping(defaultMapping);
        }
      }

      Integer maybeNumberOfShards = settings.getAsInt(SETTING_NUMBER_OF_SHARDS, null);
      if (maybeNumberOfShards == null) {
        throw new IllegalArgumentException("must specify numberOfShards for index [" + index + "]");
      }
      int numberOfShards = maybeNumberOfShards;
      if (numberOfShards <= 0) {
        throw new IllegalArgumentException(
            "must specify positive number of shards for index [" + index + "]");
      }

      Integer maybeNumberOfReplicas = settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, null);
      if (maybeNumberOfReplicas == null) {
        throw new IllegalArgumentException(
            "must specify numberOfReplicas for index [" + index + "]");
      }
      int numberOfReplicas = maybeNumberOfReplicas;
      if (numberOfReplicas < 0) {
        throw new IllegalArgumentException(
            "must specify non-negative number of shards for index [" + index + "]");
      }

      // fill missing slots in activeAllocationIds with empty set if needed and make all entries
      // immutable
      ImmutableOpenIntMap.Builder<Set<String>> filledActiveAllocationIds =
          ImmutableOpenIntMap.builder();
      for (int i = 0; i < numberOfShards; i++) {
        if (activeAllocationIds.containsKey(i)) {
          filledActiveAllocationIds.put(
              i, Collections.unmodifiableSet(new HashSet<>(activeAllocationIds.get(i))));
        } else {
          filledActiveAllocationIds.put(i, Collections.emptySet());
        }
      }
      final Map<String, String> requireMap =
          INDEX_ROUTING_REQUIRE_GROUP_SETTING.get(settings).getAsMap();
      final DiscoveryNodeFilters requireFilters;
      if (requireMap.isEmpty()) {
        requireFilters = null;
      } else {
        requireFilters = DiscoveryNodeFilters.buildFromKeyValue(AND, requireMap);
      }
      Map<String, String> includeMap = INDEX_ROUTING_INCLUDE_GROUP_SETTING.get(settings).getAsMap();
      final DiscoveryNodeFilters includeFilters;
      if (includeMap.isEmpty()) {
        includeFilters = null;
      } else {
        includeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, includeMap);
      }
      Map<String, String> excludeMap = INDEX_ROUTING_EXCLUDE_GROUP_SETTING.get(settings).getAsMap();
      final DiscoveryNodeFilters excludeFilters;
      if (excludeMap.isEmpty()) {
        excludeFilters = null;
      } else {
        excludeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, excludeMap);
      }
      Version indexCreatedVersion = Version.indexCreated(settings);
      Version indexUpgradedVersion =
          settings.getAsVersion(IndexMetaData.SETTING_VERSION_UPGRADED, indexCreatedVersion);
      String stringLuceneVersion = settings.get(SETTING_VERSION_MINIMUM_COMPATIBLE);
      final org.apache.lucene.util.Version minimumCompatibleLuceneVersion;
      if (stringLuceneVersion != null) {
        try {
          minimumCompatibleLuceneVersion =
              org.apache.lucene.util.Version.parse(stringLuceneVersion);
        } catch (ParseException ex) {
          throw new IllegalStateException(
              "Cannot parse lucene version ["
                  + stringLuceneVersion
                  + "] in the ["
                  + SETTING_VERSION_MINIMUM_COMPATIBLE
                  + "] setting",
              ex);
        }
      } else {
        minimumCompatibleLuceneVersion = null;
      }

      final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE);
      return new IndexMetaData(
          new Index(index, uuid),
          version,
          state,
          numberOfShards,
          numberOfReplicas,
          tmpSettings,
          mappings.build(),
          tmpAliases.build(),
          customs.build(),
          filledActiveAllocationIds.build(),
          requireFilters,
          includeFilters,
          excludeFilters,
          indexCreatedVersion,
          indexUpgradedVersion,
          minimumCompatibleLuceneVersion);
    }
Пример #23
0
 public IndexMetaData get(String index) {
   return indices.get(index);
 }
Пример #24
0
 public MappingMetaData mapping(String type) {
   return mappings.get(type);
 }