Example #1
0
  /*
   * Finds all mappings for types and concrete indices. Types are expanded to
   * include all types that match the glob patterns in the types array. Empty
   * types array, null or {"_all"} will be expanded to all types available for
   * the given indices.
   */
  public ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> findMappings(
      String[] concreteIndices, final String[] types) {
    assert types != null;
    assert concreteIndices != null;
    if (concreteIndices.length == 0) {
      return ImmutableOpenMap.of();
    }

    ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder =
        ImmutableOpenMap.builder();
    Iterable<String> intersection =
        HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys());
    for (String index : intersection) {
      IndexMetaData indexMetaData = indices.get(index);
      ImmutableOpenMap.Builder<String, MappingMetaData> filteredMappings;
      if (isAllTypes(types)) {
        indexMapBuilder.put(
            index, indexMetaData.getMappings()); // No types specified means get it all

      } else {
        filteredMappings = ImmutableOpenMap.builder();
        for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.getMappings()) {
          if (Regex.simpleMatch(types, cursor.key)) {
            filteredMappings.put(cursor.key, cursor.value);
          }
        }
        if (!filteredMappings.isEmpty()) {
          indexMapBuilder.put(index, filteredMappings.build());
        }
      }
    }
    return indexMapBuilder.build();
  }
 public Builder(String index) {
   this.index = index;
   this.mappings = ImmutableOpenMap.builder();
   this.aliases = ImmutableOpenMap.builder();
   this.customs = ImmutableOpenMap.builder();
   this.activeAllocationIds = ImmutableOpenIntMap.builder();
 }
Example #3
0
 public Builder(MetaData metaData) {
   this.clusterUUID = metaData.clusterUUID;
   this.transientSettings = metaData.transientSettings;
   this.persistentSettings = metaData.persistentSettings;
   this.version = metaData.version;
   this.indices = ImmutableOpenMap.builder(metaData.indices);
   this.templates = ImmutableOpenMap.builder(metaData.templates);
   this.customs = ImmutableOpenMap.builder(metaData.customs);
 }
 public Builder(IndexMetaData indexMetaData) {
   this.index = indexMetaData.getIndex().getName();
   this.state = indexMetaData.state;
   this.version = indexMetaData.version;
   this.settings = indexMetaData.getSettings();
   this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings);
   this.aliases = ImmutableOpenMap.builder(indexMetaData.aliases);
   this.customs = ImmutableOpenMap.builder(indexMetaData.customs);
   this.activeAllocationIds = ImmutableOpenIntMap.builder(indexMetaData.activeAllocationIds);
 }
 @Override
 public void readFrom(StreamInput in) throws IOException {
   super.readFrom(in);
   int size = in.readVInt();
   ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder =
       ImmutableOpenMap.builder();
   for (int i = 0; i < size; i++) {
     String key = in.readString();
     int valueSize = in.readVInt();
     ImmutableOpenMap.Builder<String, MappingMetaData> typeMapBuilder = ImmutableOpenMap.builder();
     for (int j = 0; j < valueSize; j++) {
       typeMapBuilder.put(in.readString(), new MappingMetaData(in));
     }
     indexMapBuilder.put(key, typeMapBuilder.build());
   }
   mappings = indexMapBuilder.build();
 }
 public ClusterBlocks build() {
   // We copy the block sets here in case of the builder is modified after build is called
   ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder =
       ImmutableOpenMap.builder(indices.size());
   for (Map.Entry<String, Set<ClusterBlock>> entry : indices.entrySet()) {
     indicesBuilder.put(entry.getKey(), unmodifiableSet(new HashSet<>(entry.getValue())));
   }
   return new ClusterBlocks(unmodifiableSet(new HashSet<>(global)), indicesBuilder.build());
 }
Example #7
0
 public void writeChecksum(String name, String checksum) throws IOException {
   ensureOpen();
   // update the metadata to include the checksum and write a new checksums file
   synchronized (mutex) {
     StoreFileMetaData metaData = filesMetadata.get(name);
     metaData =
         new StoreFileMetaData(metaData.name(), metaData.length(), checksum, metaData.directory());
     filesMetadata = ImmutableOpenMap.builder(filesMetadata).fPut(name, metaData).build();
     writeChecksums();
   }
 }
 @Override
 public ClusterBlocks readFrom(StreamInput in) throws IOException {
   Set<ClusterBlock> global = readBlockSet(in);
   int size = in.readVInt();
   ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder =
       ImmutableOpenMap.builder(size);
   for (int j = 0; j < size; j++) {
     indicesBuilder.put(in.readString().intern(), readBlockSet(in));
   }
   return new ClusterBlocks(global, indicesBuilder.build());
 }
 protected void addMappers(
     Collection<ObjectMapper> objectMappers, Collection<FieldMapper> fieldMappers) {
   assert mappingLock.isWriteLockedByCurrentThread();
   ImmutableOpenMap.Builder<String, ObjectMapper> fullPathObjectMappers =
       ImmutableOpenMap.builder(this.fullPathObjectMappers);
   for (ObjectMapper objectMapper : objectMappers) {
     fullPathObjectMappers.put(objectMapper.fullPath(), objectMapper);
     if (objectMapper.nested().isNested()) {
       hasNested = true;
     }
   }
   this.fullPathObjectMappers = fullPathObjectMappers.build();
   this.fieldTypes = this.fieldTypes.copyAndAddAll(fieldMappers);
 }
Example #10
0
 public void writeChecksums(Map<String, String> checksums) throws IOException {
   ensureOpen();
   // update the metadata to include the checksum and write a new checksums file
   synchronized (mutex) {
     for (Map.Entry<String, String> entry : checksums.entrySet()) {
       StoreFileMetaData metaData = filesMetadata.get(entry.getKey());
       metaData =
           new StoreFileMetaData(
               metaData.name(), metaData.length(), entry.getValue(), metaData.directory());
       filesMetadata =
           ImmutableOpenMap.builder(filesMetadata).fPut(entry.getKey(), metaData).build();
     }
     writeChecksums();
   }
 }
 private MetaData buildMetaDataForVersion(MetaData metaData, long version) {
   ImmutableOpenMap.Builder<String, IndexMetaData> indices =
       ImmutableOpenMap.builder(metaData.indices());
   indices.put(
       "test" + version,
       IndexMetaData.builder("test" + version)
           .settings(
               Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
           .numberOfShards((int) version)
           .numberOfReplicas(0)
           .build());
   return MetaData.builder(metaData)
       .transientSettings(Settings.builder().put("test", version).build())
       .indices(indices.build())
       .build();
 }
Example #12
0
 public void renameFile(String from, String to) throws IOException {
   ensureOpen();
   synchronized (mutex) {
     StoreFileMetaData fromMetaData = filesMetadata.get(from); // we should always find this one
     if (fromMetaData == null) {
       throw new FileNotFoundException(from);
     }
     directoryService.renameFile(fromMetaData.directory(), from, to);
     StoreFileMetaData toMetaData =
         new StoreFileMetaData(
             to, fromMetaData.length(), fromMetaData.checksum(), fromMetaData.directory());
     filesMetadata =
         ImmutableOpenMap.builder(filesMetadata).fRemove(from).fPut(to, toMetaData).build();
     files = filesMetadata.keys().toArray(String.class);
   }
 }
Example #13
0
 public void deleteFileChecksum(String name) throws IOException {
   ensureOpen();
   StoreFileMetaData metaData = filesMetadata.get(name);
   if (metaData != null) {
     try {
       metaData.directory().deleteFile(name);
     } catch (IOException e) {
       if (metaData.directory().fileExists(name)) {
         throw e;
       }
     }
   }
   synchronized (mutex) {
     filesMetadata = ImmutableOpenMap.builder(filesMetadata).fRemove(name).build();
     files = filesMetadata.keys().toArray(String.class);
   }
 }
Example #14
0
 StoreDirectory(Distributor distributor) throws IOException {
   this.distributor = distributor;
   synchronized (mutex) {
     ImmutableOpenMap.Builder<String, StoreFileMetaData> builder = ImmutableOpenMap.builder();
     Map<String, String> checksums =
         readChecksums(distributor.all(), new HashMap<String, String>());
     for (Directory delegate : distributor.all()) {
       for (String file : delegate.listAll()) {
         String checksum = checksums.get(file);
         builder.put(
             file, new StoreFileMetaData(file, delegate.fileLength(file), checksum, delegate));
       }
     }
     filesMetadata = builder.build();
     files = filesMetadata.keys().toArray(String.class);
   }
 }
Example #15
0
 private void addObjectMappers(ObjectMapper[] objectMappers) {
   synchronized (mappersMutex) {
     ImmutableOpenMap.Builder<String, ObjectMappers> fullPathObjectMappers =
         ImmutableOpenMap.builder(this.fullPathObjectMappers);
     for (ObjectMapper objectMapper : objectMappers) {
       ObjectMappers mappers = fullPathObjectMappers.get(objectMapper.fullPath());
       if (mappers == null) {
         mappers = new ObjectMappers(objectMapper);
       } else {
         mappers = mappers.concat(objectMapper);
       }
       fullPathObjectMappers.put(objectMapper.fullPath(), mappers);
       // update the hasNested flag
       if (objectMapper.nested().isNested()) {
         hasNested = true;
       }
     }
     this.fullPathObjectMappers = fullPathObjectMappers.build();
   }
 }
Example #16
0
  private void removeObjectAndFieldMappers(DocumentMapper docMapper) {
    synchronized (mappersMutex) {
      fieldMappers.removeMappers(docMapper.mappers());

      ImmutableOpenMap.Builder<String, ObjectMappers> fullPathObjectMappers =
          ImmutableOpenMap.builder(this.fullPathObjectMappers);
      for (ObjectMapper mapper : docMapper.objectMappers().values()) {
        ObjectMappers mappers = fullPathObjectMappers.get(mapper.fullPath());
        if (mappers != null) {
          mappers = mappers.remove(mapper);
          if (mappers.isEmpty()) {
            fullPathObjectMappers.remove(mapper.fullPath());
          } else {
            fullPathObjectMappers.put(mapper.fullPath(), mappers);
          }
        }
      }

      this.fullPathObjectMappers = fullPathObjectMappers.build();
    }
  }
Example #17
0
 @Override
 public void close() throws IOException {
   out.close();
   String checksum = null;
   IndexOutput underlying = out;
   // TODO: cut over to lucene's CRC
   // *WARNING*: lucene has classes in same o.a.l.store package with very similar names,
   // but using CRC, not Adler!
   if (underlying instanceof BufferedChecksumIndexOutput) {
     Checksum digest = ((BufferedChecksumIndexOutput) underlying).digest();
     assert digest instanceof Adler32;
     checksum = Long.toString(digest.getValue(), Character.MAX_RADIX);
   }
   synchronized (mutex) {
     StoreFileMetaData md =
         new StoreFileMetaData(
             name, metaData.directory().fileLength(name), checksum, metaData.directory());
     filesMetadata = ImmutableOpenMap.builder(filesMetadata).fPut(name, md).build();
     files = filesMetadata.keys().toArray(String.class);
   }
 }
Example #18
0
    public IndexOutput createOutput(String name, IOContext context, boolean raw)
        throws IOException {
      ensureOpen();
      Directory directory;
      // we want to write the segments gen file to the same directory *all* the time
      // to make sure we don't create multiple copies of it
      if (isChecksum(name) || IndexFileNames.SEGMENTS_GEN.equals(name)) {
        directory = distributor.primary();
      } else {
        directory = distributor.any();
      }
      IndexOutput out = directory.createOutput(name, context);
      boolean success = false;
      try {
        synchronized (mutex) {
          StoreFileMetaData metaData = new StoreFileMetaData(name, -1, null, directory);
          filesMetadata = ImmutableOpenMap.builder(filesMetadata).fPut(name, metaData).build();
          files = filesMetadata.keys().toArray(String.class);
          boolean computeChecksum = !raw;
          if (computeChecksum) {
            // don't compute checksum for segment based files
            if (IndexFileNames.SEGMENTS_GEN.equals(name)
                || name.startsWith(IndexFileNames.SEGMENTS)) {
              computeChecksum = false;
            }
          }
          if (computeChecksum) {
            out = new BufferedChecksumIndexOutput(out, new Adler32());
          }

          final StoreIndexOutput storeIndexOutput = new StoreIndexOutput(metaData, out, name);
          success = true;
          return storeIndexOutput;
        }
      } finally {
        if (!success) {
          IOUtils.closeWhileHandlingException(out);
        }
      }
    }
Example #19
0
  ClusterBlocks(
      Set<ClusterBlock> global, ImmutableOpenMap<String, Set<ClusterBlock>> indicesBlocks) {
    this.global = global;
    this.indicesBlocks = indicesBlocks;

    levelHolders = new ImmutableLevelHolder[ClusterBlockLevel.values().length];
    for (final ClusterBlockLevel level : ClusterBlockLevel.values()) {
      Predicate<ClusterBlock> containsLevel = block -> block.contains(level);
      Set<ClusterBlock> newGlobal =
          unmodifiableSet(global.stream().filter(containsLevel).collect(toSet()));

      ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder =
          ImmutableOpenMap.builder();
      for (ObjectObjectCursor<String, Set<ClusterBlock>> entry : indicesBlocks) {
        indicesBuilder.put(
            entry.key,
            unmodifiableSet(entry.value.stream().filter(containsLevel).collect(toSet())));
      }

      levelHolders[level.id()] = new ImmutableLevelHolder(newGlobal, indicesBuilder.build());
    }
  }
Example #20
0
 @Override
 public void deleteFile(String name) throws IOException {
   ensureOpen();
   // we don't allow to delete the checksums files, only using the deleteChecksum method
   if (isChecksum(name)) {
     return;
   }
   StoreFileMetaData metaData = filesMetadata.get(name);
   if (metaData != null) {
     try {
       metaData.directory().deleteFile(name);
     } catch (IOException e) {
       if (metaData.directory().fileExists(name)) {
         throw e;
       }
     }
   }
   synchronized (mutex) {
     filesMetadata = ImmutableOpenMap.builder(filesMetadata).fRemove(name).build();
     files = filesMetadata.keys().toArray(String.class);
   }
 }
Example #21
0
  /**
   * Finds the specific index aliases that match with the specified aliases directly or partially
   * via wildcards and that point to the specified concrete indices or match partially with the
   * indices via wildcards.
   *
   * @param aliases The names of the index aliases to find
   * @param concreteIndices The concrete indexes the index aliases must point to order to be
   *     returned.
   * @return the found index aliases grouped by index
   */
  public ImmutableOpenMap<String, List<AliasMetaData>> findAliases(
      final String[] aliases, String[] concreteIndices) {
    assert aliases != null;
    assert concreteIndices != null;
    if (concreteIndices.length == 0) {
      return ImmutableOpenMap.of();
    }

    boolean matchAllAliases = matchAllAliases(aliases);
    ImmutableOpenMap.Builder<String, List<AliasMetaData>> mapBuilder = ImmutableOpenMap.builder();
    Iterable<String> intersection =
        HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys());
    for (String index : intersection) {
      IndexMetaData indexMetaData = indices.get(index);
      List<AliasMetaData> filteredValues = new ArrayList<>();
      for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
        AliasMetaData value = cursor.value;
        if (matchAllAliases || Regex.simpleMatch(aliases, value.alias())) {
          filteredValues.add(value);
        }
      }

      if (!filteredValues.isEmpty()) {
        // Make the list order deterministic
        CollectionUtil.timSort(
            filteredValues,
            new Comparator<AliasMetaData>() {
              @Override
              public int compare(AliasMetaData o1, AliasMetaData o2) {
                return o1.alias().compareTo(o2.alias());
              }
            });
        mapBuilder.put(index, Collections.unmodifiableList(filteredValues));
      }
    }
    return mapBuilder.build();
  }
Example #22
0
 public Builder() {
   clusterUUID = "_na_";
   indices = ImmutableOpenMap.builder();
   templates = ImmutableOpenMap.builder();
   customs = ImmutableOpenMap.builder();
 }
Example #23
0
 @Override
 protected ImmutableOpenMap<Integer, V> createMap(Map values) {
   return ImmutableOpenMap.<Integer, V>builder().putAll(values).build();
 }