Esempio n. 1
0
  @Override
  public boolean equals(Object o) {
    if (this == o) {
      return true;
    }
    if (o == null || getClass() != o.getClass()) {
      return false;
    }

    IndexMetaData that = (IndexMetaData) o;

    if (!aliases.equals(that.aliases)) {
      return false;
    }
    if (!index.equals(that.index)) {
      return false;
    }
    if (!mappings.equals(that.mappings)) {
      return false;
    }
    if (!settings.equals(that.settings)) {
      return false;
    }
    if (state != that.state) {
      return false;
    }
    if (!customs.equals(that.customs)) {
      return false;
    }
    if (!activeAllocationIds.equals(that.activeAllocationIds)) {
      return false;
    }
    return true;
  }
Esempio n. 2
0
 @Override
 public void writeTo(StreamOutput out) throws IOException {
   out.writeString(index.getName()); // uuid will come as part of settings
   out.writeLong(version);
   out.writeByte(state.id());
   writeSettingsToStream(settings, out);
   out.writeVInt(mappings.size());
   for (ObjectCursor<MappingMetaData> cursor : mappings.values()) {
     cursor.value.writeTo(out);
   }
   out.writeVInt(aliases.size());
   for (ObjectCursor<AliasMetaData> cursor : aliases.values()) {
     cursor.value.writeTo(out);
   }
   out.writeVInt(customs.size());
   for (ObjectObjectCursor<String, Custom> cursor : customs) {
     out.writeString(cursor.key);
     cursor.value.writeTo(out);
   }
   out.writeVInt(activeAllocationIds.size());
   for (IntObjectCursor<Set<String>> cursor : activeAllocationIds) {
     out.writeVInt(cursor.key);
     DiffableUtils.StringSetValueSerializer.getInstance().write(cursor.value, out);
   }
 }
Esempio n. 3
0
  /*
   * Finds all mappings for types and concrete indices. Types are expanded to
   * include all types that match the glob patterns in the types array. Empty
   * types array, null or {"_all"} will be expanded to all types available for
   * the given indices.
   */
  public ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> findMappings(
      String[] concreteIndices, final String[] types) {
    assert types != null;
    assert concreteIndices != null;
    if (concreteIndices.length == 0) {
      return ImmutableOpenMap.of();
    }

    ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder =
        ImmutableOpenMap.builder();
    Iterable<String> intersection =
        HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys());
    for (String index : intersection) {
      IndexMetaData indexMetaData = indices.get(index);
      ImmutableOpenMap.Builder<String, MappingMetaData> filteredMappings;
      if (isAllTypes(types)) {
        indexMapBuilder.put(
            index, indexMetaData.getMappings()); // No types specified means get it all

      } else {
        filteredMappings = ImmutableOpenMap.builder();
        for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.getMappings()) {
          if (Regex.simpleMatch(types, cursor.key)) {
            filteredMappings.put(cursor.key, cursor.value);
          }
        }
        if (!filteredMappings.isEmpty()) {
          indexMapBuilder.put(index, filteredMappings.build());
        }
      }
    }
    return indexMapBuilder.build();
  }
Esempio n. 4
0
 public Builder(String index) {
   this.index = index;
   this.mappings = ImmutableOpenMap.builder();
   this.aliases = ImmutableOpenMap.builder();
   this.customs = ImmutableOpenMap.builder();
   this.activeAllocationIds = ImmutableOpenIntMap.builder();
 }
    @Override
    public ClusterState execute(ClusterState currentState) throws Exception {
      if (cancellableThreads.isCancelled()
          == false) { // no need to run this if recovery is canceled
        IndexMetaData indexMetaData =
            clusterService.state().metaData().getIndices().get(indexService.index().getName());
        ImmutableOpenMap<String, MappingMetaData> metaDataMappings = null;
        if (indexMetaData != null) {
          metaDataMappings = indexMetaData.getMappings();
        }
        // default mapping should not be sent back, it can only be updated by put mapping API, and
        // its
        // a full in place replace, we don't want to override a potential update coming into it
        for (DocumentMapper documentMapper : indexService.mapperService().docMappers(false)) {

          MappingMetaData mappingMetaData =
              metaDataMappings == null ? null : metaDataMappings.get(documentMapper.type());
          if (mappingMetaData == null
              || !documentMapper.refreshSource().equals(mappingMetaData.source())) {
            // not on master yet in the right form
            documentMappersToUpdate.add(documentMapper);
          }
        }
      }
      return currentState;
    }
Esempio n. 6
0
 /**
  * Sometimes, the default mapping exists and an actual mapping is not created yet (introduced), in
  * this case, we want to return the default mapping in case it has some default mapping
  * definitions.
  *
  * <p>Note, once the mapping type is introduced, the default mapping is applied on the actual
  * typed MappingMetaData, setting its routing, timestamp, and so on if needed.
  */
 @Nullable
 public MappingMetaData mappingOrDefault(String mappingType) {
   MappingMetaData mapping = mappings.get(mappingType);
   if (mapping != null) {
     return mapping;
   }
   return mappings.get(MapperService.DEFAULT_MAPPING);
 }
Esempio n. 7
0
 @Override
 public boolean hasIndex(@NotNull ValueTable valueTable) {
   ClusterStateResponse resp =
       opalSearchService.getClient().admin().cluster().prepareState().execute().actionGet();
   ImmutableOpenMap<String, MappingMetaData> mappings =
       resp.getState().metaData().index(getName()).mappings();
   return mappings.containsKey(getIndex(valueTable).getIndexName());
 }
Esempio n. 8
0
 public Builder(MetaData metaData) {
   this.clusterUUID = metaData.clusterUUID;
   this.transientSettings = metaData.transientSettings;
   this.persistentSettings = metaData.persistentSettings;
   this.version = metaData.version;
   this.indices = ImmutableOpenMap.builder(metaData.indices);
   this.templates = ImmutableOpenMap.builder(metaData.templates);
   this.customs = ImmutableOpenMap.builder(metaData.customs);
 }
Esempio n. 9
0
 @Override
 public int hashCode() {
   int result = index.hashCode();
   result = 31 * result + state.hashCode();
   result = 31 * result + aliases.hashCode();
   result = 31 * result + settings.hashCode();
   result = 31 * result + mappings.hashCode();
   result = 31 * result + activeAllocationIds.hashCode();
   return result;
 }
Esempio n. 10
0
 public Builder(IndexMetaData indexMetaData) {
   this.index = indexMetaData.getIndex().getName();
   this.state = indexMetaData.state;
   this.version = indexMetaData.version;
   this.settings = indexMetaData.getSettings();
   this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings);
   this.aliases = ImmutableOpenMap.builder(indexMetaData.aliases);
   this.customs = ImmutableOpenMap.builder(indexMetaData.customs);
   this.activeAllocationIds = ImmutableOpenIntMap.builder(indexMetaData.activeAllocationIds);
 }
Esempio n. 11
0
 public void writeChecksum(String name, String checksum) throws IOException {
   ensureOpen();
   // update the metadata to include the checksum and write a new checksums file
   synchronized (mutex) {
     StoreFileMetaData metaData = filesMetadata.get(name);
     metaData =
         new StoreFileMetaData(metaData.name(), metaData.length(), checksum, metaData.directory());
     filesMetadata = ImmutableOpenMap.builder(filesMetadata).fPut(name, metaData).build();
     writeChecksums();
   }
 }
 private static void assertMappingsHaveField(
     GetMappingsResponse mappings, String index, String type, String field) throws IOException {
   ImmutableOpenMap<String, MappingMetaData> indexMappings = mappings.getMappings().get("index");
   assertNotNull(indexMappings);
   MappingMetaData typeMappings = indexMappings.get(type);
   assertNotNull(typeMappings);
   Map<String, Object> typeMappingsMap = typeMappings.getSourceAsMap();
   Map<String, Object> properties = (Map<String, Object>) typeMappingsMap.get("properties");
   assertTrue(
       "Could not find [" + field + "] in " + typeMappingsMap.toString(),
       properties.containsKey(field));
 }
 public void assertVersionCreated(Version version, String... indices) {
   GetSettingsResponse getSettingsResponse =
       client().admin().indices().prepareGetSettings(indices).get();
   ImmutableOpenMap<String, Settings> indexToSettings = getSettingsResponse.getIndexToSettings();
   for (String index : indices) {
     Settings settings = indexToSettings.get(index);
     assertThat(
         settings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null), notNullValue());
     assertThat(
         settings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null), equalTo(version));
   }
 }
 private static void assertContext(ActionRequest<?> request, Map<String, String> context) {
   if (context.size() == 0) {
     assertThat(request.isContextEmpty(), is(true));
   } else {
     ImmutableOpenMap map = request.getContext();
     assertThat(map, notNullValue());
     assertThat(map.size(), equalTo(context.size()));
     for (Object key : map.keys()) {
       assertThat(context.get(key), equalTo(request.getFromContext(key)));
     }
   }
 }
Esempio n. 15
0
 public void writeChecksums(Map<String, String> checksums) throws IOException {
   ensureOpen();
   // update the metadata to include the checksum and write a new checksums file
   synchronized (mutex) {
     for (Map.Entry<String, String> entry : checksums.entrySet()) {
       StoreFileMetaData metaData = filesMetadata.get(entry.getKey());
       metaData =
           new StoreFileMetaData(
               metaData.name(), metaData.length(), entry.getValue(), metaData.directory());
       filesMetadata =
           ImmutableOpenMap.builder(filesMetadata).fPut(entry.getKey(), metaData).build();
     }
     writeChecksums();
   }
 }
Esempio n. 16
0
 public void renameFile(String from, String to) throws IOException {
   ensureOpen();
   synchronized (mutex) {
     StoreFileMetaData fromMetaData = filesMetadata.get(from); // we should always find this one
     if (fromMetaData == null) {
       throw new FileNotFoundException(from);
     }
     directoryService.renameFile(fromMetaData.directory(), from, to);
     StoreFileMetaData toMetaData =
         new StoreFileMetaData(
             to, fromMetaData.length(), fromMetaData.checksum(), fromMetaData.directory());
     filesMetadata =
         ImmutableOpenMap.builder(filesMetadata).fRemove(from).fPut(to, toMetaData).build();
     files = filesMetadata.keys().toArray(String.class);
   }
 }
 @Override
 public void readFrom(StreamInput in) throws IOException {
   super.readFrom(in);
   int size = in.readVInt();
   ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder =
       ImmutableOpenMap.builder();
   for (int i = 0; i < size; i++) {
     String key = in.readString();
     int valueSize = in.readVInt();
     ImmutableOpenMap.Builder<String, MappingMetaData> typeMapBuilder = ImmutableOpenMap.builder();
     for (int j = 0; j < valueSize; j++) {
       typeMapBuilder.put(in.readString(), new MappingMetaData(in));
     }
     indexMapBuilder.put(key, typeMapBuilder.build());
   }
   mappings = indexMapBuilder.build();
 }
Esempio n. 18
0
 /** Opened an index input in raw form, no decompression for example. */
 public IndexInput openInputRaw(String name, IOContext context) throws IOException {
   ensureOpen();
   StoreFileMetaData metaData = filesMetadata.get(name);
   if (metaData == null) {
     throw new FileNotFoundException(name);
   }
   return metaData.directory().openInput(name, context);
 }
Esempio n. 19
0
 public ClusterBlocks build() {
   // We copy the block sets here in case of the builder is modified after build is called
   ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder =
       ImmutableOpenMap.builder(indices.size());
   for (Map.Entry<String, Set<ClusterBlock>> entry : indices.entrySet()) {
     indicesBuilder.put(entry.getKey(), unmodifiableSet(new HashSet<>(entry.getValue())));
   }
   return new ClusterBlocks(unmodifiableSet(new HashSet<>(global)), indicesBuilder.build());
 }
Esempio n. 20
0
 /**
  * @param concreteIndex The concrete index to check if routing is required
  * @param type The type to check if routing is required
  * @return Whether routing is required according to the mapping for the specified index and type
  */
 public boolean routingRequired(String concreteIndex, String type) {
   IndexMetaData indexMetaData = indices.get(concreteIndex);
   if (indexMetaData != null) {
     MappingMetaData mappingMetaData = indexMetaData.getMappings().get(type);
     if (mappingMetaData != null) {
       return mappingMetaData.routing().required();
     }
   }
   return false;
 }
Esempio n. 21
0
 @Override
 public ClusterBlocks readFrom(StreamInput in) throws IOException {
   Set<ClusterBlock> global = readBlockSet(in);
   int size = in.readVInt();
   ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder =
       ImmutableOpenMap.builder(size);
   for (int j = 0; j < size; j++) {
     indicesBuilder.put(in.readString().intern(), readBlockSet(in));
   }
   return new ClusterBlocks(global, indicesBuilder.build());
 }
Esempio n. 22
0
 @Override
 public void writeTo(StreamOutput out) throws IOException {
   out.writeLong(version);
   out.writeString(clusterUUID);
   writeSettingsToStream(transientSettings, out);
   writeSettingsToStream(persistentSettings, out);
   out.writeVInt(indices.size());
   for (IndexMetaData indexMetaData : this) {
     indexMetaData.writeTo(out);
   }
   out.writeVInt(templates.size());
   for (ObjectCursor<IndexTemplateMetaData> cursor : templates.values()) {
     cursor.value.writeTo(out);
   }
   out.writeVInt(customs.size());
   for (ObjectObjectCursor<String, Custom> cursor : customs) {
     out.writeString(cursor.key);
     cursor.value.writeTo(out);
   }
 }
 @Override
 public void deleteType(String index, String type) {
   ImmutableOpenMap<String, MappingMetaData> mappings =
       client
           .admin()
           .cluster()
           .prepareState()
           .execute()
           .actionGet()
           .getState()
           .metaData()
           .index(index)
           .mappings();
   if (mappings.containsKey(type)) {
     client
         .admin()
         .indices()
         .deleteMapping(new DeleteMappingRequest(index).types(type))
         .actionGet();
   }
 }
public class GetMappingsResponse extends ActionResponse {

  private ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings =
      ImmutableOpenMap.of();

  GetMappingsResponse(
      ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings) {
    this.mappings = mappings;
  }

  GetMappingsResponse() {}

  public ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings() {
    return mappings;
  }

  public ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> getMappings() {
    return mappings();
  }

  @Override
  public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    int size = in.readVInt();
    ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder =
        ImmutableOpenMap.builder();
    for (int i = 0; i < size; i++) {
      String key = in.readString();
      int valueSize = in.readVInt();
      ImmutableOpenMap.Builder<String, MappingMetaData> typeMapBuilder = ImmutableOpenMap.builder();
      for (int j = 0; j < valueSize; j++) {
        typeMapBuilder.put(in.readString(), new MappingMetaData(in));
      }
      indexMapBuilder.put(key, typeMapBuilder.build());
    }
    mappings = indexMapBuilder.build();
  }

  @Override
  public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    out.writeVInt(mappings.size());
    for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> indexEntry :
        mappings) {
      out.writeString(indexEntry.key);
      out.writeVInt(indexEntry.value.size());
      for (ObjectObjectCursor<String, MappingMetaData> typeEntry : indexEntry.value) {
        out.writeString(typeEntry.key);
        typeEntry.value.writeTo(out);
      }
    }
  }
}
Esempio n. 25
0
 public StoreFileMetaData metaData(String name) throws IOException {
   ensureOpen();
   StoreFileMetaData md = filesMetadata.get(name);
   if (md == null) {
     return null;
   }
   // IndexOutput not closed, does not exists
   if (md.length() == -1) {
     return null;
   }
   return md;
 }
Esempio n. 26
0
 synchronized void closeInternal() throws IOException {
   if (isOpen) {
     isOpen = false;
     for (Directory delegate : distributor.all()) {
       delegate.close();
     }
     synchronized (mutex) {
       filesMetadata = ImmutableOpenMap.of();
       files = Strings.EMPTY_ARRAY;
     }
   }
 }
 protected void addMappers(
     Collection<ObjectMapper> objectMappers, Collection<FieldMapper> fieldMappers) {
   assert mappingLock.isWriteLockedByCurrentThread();
   ImmutableOpenMap.Builder<String, ObjectMapper> fullPathObjectMappers =
       ImmutableOpenMap.builder(this.fullPathObjectMappers);
   for (ObjectMapper objectMapper : objectMappers) {
     fullPathObjectMappers.put(objectMapper.fullPath(), objectMapper);
     if (objectMapper.nested().isNested()) {
       hasNested = true;
     }
   }
   this.fullPathObjectMappers = fullPathObjectMappers.build();
   this.fieldTypes = this.fieldTypes.copyAndAddAll(fieldMappers);
 }
Esempio n. 28
0
  /**
   * Finds the specific index aliases that match with the specified aliases directly or partially
   * via wildcards and that point to the specified concrete indices or match partially with the
   * indices via wildcards.
   *
   * @param aliases The names of the index aliases to find
   * @param concreteIndices The concrete indexes the index aliases must point to order to be
   *     returned.
   * @return the found index aliases grouped by index
   */
  public ImmutableOpenMap<String, List<AliasMetaData>> findAliases(
      final String[] aliases, String[] concreteIndices) {
    assert aliases != null;
    assert concreteIndices != null;
    if (concreteIndices.length == 0) {
      return ImmutableOpenMap.of();
    }

    boolean matchAllAliases = matchAllAliases(aliases);
    ImmutableOpenMap.Builder<String, List<AliasMetaData>> mapBuilder = ImmutableOpenMap.builder();
    Iterable<String> intersection =
        HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys());
    for (String index : intersection) {
      IndexMetaData indexMetaData = indices.get(index);
      List<AliasMetaData> filteredValues = new ArrayList<>();
      for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
        AliasMetaData value = cursor.value;
        if (matchAllAliases || Regex.simpleMatch(aliases, value.alias())) {
          filteredValues.add(value);
        }
      }

      if (!filteredValues.isEmpty()) {
        // Make the list order deterministic
        CollectionUtil.timSort(
            filteredValues,
            new Comparator<AliasMetaData>() {
              @Override
              public int compare(AliasMetaData o1, AliasMetaData o2) {
                return o1.alias().compareTo(o2.alias());
              }
            });
        mapBuilder.put(index, Collections.unmodifiableList(filteredValues));
      }
    }
    return mapBuilder.build();
  }
Esempio n. 29
0
 /**
  * Returns the filter associated with listed filtering aliases.
  *
  * <p>The list of filtering aliases should be obtained by calling MetaData.filteringAliases.
  * Returns <tt>null</tt> if no filtering is required.
  */
 public Query aliasFilter(QueryShardContext context, String... aliasNames) {
   if (aliasNames == null || aliasNames.length == 0) {
     return null;
   }
   final ImmutableOpenMap<String, AliasMetaData> aliases =
       indexSettings.getIndexMetaData().getAliases();
   if (aliasNames.length == 1) {
     AliasMetaData alias = aliases.get(aliasNames[0]);
     if (alias == null) {
       // This shouldn't happen unless alias disappeared after filteringAliases was called.
       throw new InvalidAliasNameException(
           index(), aliasNames[0], "Unknown alias name was passed to alias Filter");
     }
     return parse(alias, context);
   } else {
     // we need to bench here a bit, to see maybe it makes sense to use OrFilter
     BooleanQuery.Builder combined = new BooleanQuery.Builder();
     for (String aliasName : aliasNames) {
       AliasMetaData alias = aliases.get(aliasName);
       if (alias == null) {
         // This shouldn't happen unless alias disappeared after filteringAliases was called.
         throw new InvalidAliasNameException(
             indexSettings.getIndex(),
             aliasNames[0],
             "Unknown alias name was passed to alias Filter");
       }
       Query parsedFilter = parse(alias, context);
       if (parsedFilter != null) {
         combined.add(parsedFilter, BooleanClause.Occur.SHOULD);
       } else {
         // The filter might be null only if filter was removed after filteringAliases was called
         return null;
       }
     }
     return combined.build();
   }
 }
  /**
   * Until we can verify that all instances have moved over to our new mapping, we need to handcheck
   * all index mappings to make sure they have doc.associations.assoc_index as a doc value
   *
   * @param communityIdStrs
   * @return
   */
  @SuppressWarnings("unchecked")
  private static boolean validateAssociationMapping(String[] communityIdStrs) {
    // get all index mappings associated with these commids
    String[] mappings = new String[communityIdStrs.length];
    StringBuilder sb = new StringBuilder(", ");
    for (int i = 0; i < communityIdStrs.length; i++) {
      String s = communityIdStrs[i];
      mappings[i] = "doc_" + s + "*";
      sb.append("doc_").append(s).append("*, ");
    }
    ElasticSearchManager esm = ElasticSearchManager.getIndex(sb.substring(2, sb.length()));
    GetMappingsResponse response =
        esm.getRawClient().admin().indices().prepareGetMappings(mappings).get();
    for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> mapping :
        response.getMappings()) {
      ImmutableOpenMap<String, MappingMetaData> mappingVal = mapping.value;
      MappingMetaData mapping_meta = mappingVal.get("document_index");
      try {
        Map<String, Object> map = mapping_meta.getSourceAsMap();
        Map<String, Object> props = (Map<String, Object>) map.get("properties");
        Map<String, Object> assocs = (Map<String, Object>) props.get(DocumentPojo.associations_);
        Map<String, Object> assocs_props = (Map<String, Object>) assocs.get("properties");
        Map<String, Object> assoc_index =
            (Map<String, Object>) assocs_props.get(AssociationPojo.assoc_index_);
        if (!assoc_index.containsKey("doc_values") || !((Boolean) assoc_index.get("doc_values"))) {
          // doc values doesn't exist in mapping or was false
          return false;
        }

      } catch (Exception ex) {
        // failed somehow
        return false;
      }
    }
    // if we fell through, all the checked indexes had the doc_value field set
    return true;
  }