/*
* Finds all mappings for types and concrete indices. Types are expanded to
* include all types that match the glob patterns in the types array. Empty
* types array, null or {"_all"} will be expanded to all types available for
* the given indices.
*/
public ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> findMappings(
String[] concreteIndices, final String[] types) {
assert types != null;
assert concreteIndices != null;
if (concreteIndices.length == 0) {
return ImmutableOpenMap.of();
}
ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder =
ImmutableOpenMap.builder();
Iterable<String> intersection =
HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys());
for (String index : intersection) {
IndexMetaData indexMetaData = indices.get(index);
ImmutableOpenMap.Builder<String, MappingMetaData> filteredMappings;
if (isAllTypes(types)) {
indexMapBuilder.put(
index, indexMetaData.getMappings()); // No types specified means get it all
} else {
filteredMappings = ImmutableOpenMap.builder();
for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.getMappings()) {
if (Regex.simpleMatch(types, cursor.key)) {
filteredMappings.put(cursor.key, cursor.value);
}
}
if (!filteredMappings.isEmpty()) {
indexMapBuilder.put(index, filteredMappings.build());
}
}
}
return indexMapBuilder.build();
}
public Builder(String index) {
this.index = index;
this.mappings = ImmutableOpenMap.builder();
this.aliases = ImmutableOpenMap.builder();
this.customs = ImmutableOpenMap.builder();
this.activeAllocationIds = ImmutableOpenIntMap.builder();
}
public Builder(MetaData metaData) {
this.clusterUUID = metaData.clusterUUID;
this.transientSettings = metaData.transientSettings;
this.persistentSettings = metaData.persistentSettings;
this.version = metaData.version;
this.indices = ImmutableOpenMap.builder(metaData.indices);
this.templates = ImmutableOpenMap.builder(metaData.templates);
this.customs = ImmutableOpenMap.builder(metaData.customs);
}
public Builder(IndexMetaData indexMetaData) {
this.index = indexMetaData.getIndex().getName();
this.state = indexMetaData.state;
this.version = indexMetaData.version;
this.settings = indexMetaData.getSettings();
this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings);
this.aliases = ImmutableOpenMap.builder(indexMetaData.aliases);
this.customs = ImmutableOpenMap.builder(indexMetaData.customs);
this.activeAllocationIds = ImmutableOpenIntMap.builder(indexMetaData.activeAllocationIds);
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
int size = in.readVInt();
ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder =
ImmutableOpenMap.builder();
for (int i = 0; i < size; i++) {
String key = in.readString();
int valueSize = in.readVInt();
ImmutableOpenMap.Builder<String, MappingMetaData> typeMapBuilder = ImmutableOpenMap.builder();
for (int j = 0; j < valueSize; j++) {
typeMapBuilder.put(in.readString(), new MappingMetaData(in));
}
indexMapBuilder.put(key, typeMapBuilder.build());
}
mappings = indexMapBuilder.build();
}
public ClusterBlocks build() {
// We copy the block sets here in case of the builder is modified after build is called
ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder =
ImmutableOpenMap.builder(indices.size());
for (Map.Entry<String, Set<ClusterBlock>> entry : indices.entrySet()) {
indicesBuilder.put(entry.getKey(), unmodifiableSet(new HashSet<>(entry.getValue())));
}
return new ClusterBlocks(unmodifiableSet(new HashSet<>(global)), indicesBuilder.build());
}
public void writeChecksum(String name, String checksum) throws IOException {
ensureOpen();
// update the metadata to include the checksum and write a new checksums file
synchronized (mutex) {
StoreFileMetaData metaData = filesMetadata.get(name);
metaData =
new StoreFileMetaData(metaData.name(), metaData.length(), checksum, metaData.directory());
filesMetadata = ImmutableOpenMap.builder(filesMetadata).fPut(name, metaData).build();
writeChecksums();
}
}
@Override
public ClusterBlocks readFrom(StreamInput in) throws IOException {
Set<ClusterBlock> global = readBlockSet(in);
int size = in.readVInt();
ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder =
ImmutableOpenMap.builder(size);
for (int j = 0; j < size; j++) {
indicesBuilder.put(in.readString().intern(), readBlockSet(in));
}
return new ClusterBlocks(global, indicesBuilder.build());
}
protected void addMappers(
Collection<ObjectMapper> objectMappers, Collection<FieldMapper> fieldMappers) {
assert mappingLock.isWriteLockedByCurrentThread();
ImmutableOpenMap.Builder<String, ObjectMapper> fullPathObjectMappers =
ImmutableOpenMap.builder(this.fullPathObjectMappers);
for (ObjectMapper objectMapper : objectMappers) {
fullPathObjectMappers.put(objectMapper.fullPath(), objectMapper);
if (objectMapper.nested().isNested()) {
hasNested = true;
}
}
this.fullPathObjectMappers = fullPathObjectMappers.build();
this.fieldTypes = this.fieldTypes.copyAndAddAll(fieldMappers);
}
public void writeChecksums(Map<String, String> checksums) throws IOException {
ensureOpen();
// update the metadata to include the checksum and write a new checksums file
synchronized (mutex) {
for (Map.Entry<String, String> entry : checksums.entrySet()) {
StoreFileMetaData metaData = filesMetadata.get(entry.getKey());
metaData =
new StoreFileMetaData(
metaData.name(), metaData.length(), entry.getValue(), metaData.directory());
filesMetadata =
ImmutableOpenMap.builder(filesMetadata).fPut(entry.getKey(), metaData).build();
}
writeChecksums();
}
}
private MetaData buildMetaDataForVersion(MetaData metaData, long version) {
ImmutableOpenMap.Builder<String, IndexMetaData> indices =
ImmutableOpenMap.builder(metaData.indices());
indices.put(
"test" + version,
IndexMetaData.builder("test" + version)
.settings(
Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
.numberOfShards((int) version)
.numberOfReplicas(0)
.build());
return MetaData.builder(metaData)
.transientSettings(Settings.builder().put("test", version).build())
.indices(indices.build())
.build();
}
public void renameFile(String from, String to) throws IOException {
ensureOpen();
synchronized (mutex) {
StoreFileMetaData fromMetaData = filesMetadata.get(from); // we should always find this one
if (fromMetaData == null) {
throw new FileNotFoundException(from);
}
directoryService.renameFile(fromMetaData.directory(), from, to);
StoreFileMetaData toMetaData =
new StoreFileMetaData(
to, fromMetaData.length(), fromMetaData.checksum(), fromMetaData.directory());
filesMetadata =
ImmutableOpenMap.builder(filesMetadata).fRemove(from).fPut(to, toMetaData).build();
files = filesMetadata.keys().toArray(String.class);
}
}
public void deleteFileChecksum(String name) throws IOException {
ensureOpen();
StoreFileMetaData metaData = filesMetadata.get(name);
if (metaData != null) {
try {
metaData.directory().deleteFile(name);
} catch (IOException e) {
if (metaData.directory().fileExists(name)) {
throw e;
}
}
}
synchronized (mutex) {
filesMetadata = ImmutableOpenMap.builder(filesMetadata).fRemove(name).build();
files = filesMetadata.keys().toArray(String.class);
}
}
StoreDirectory(Distributor distributor) throws IOException {
this.distributor = distributor;
synchronized (mutex) {
ImmutableOpenMap.Builder<String, StoreFileMetaData> builder = ImmutableOpenMap.builder();
Map<String, String> checksums =
readChecksums(distributor.all(), new HashMap<String, String>());
for (Directory delegate : distributor.all()) {
for (String file : delegate.listAll()) {
String checksum = checksums.get(file);
builder.put(
file, new StoreFileMetaData(file, delegate.fileLength(file), checksum, delegate));
}
}
filesMetadata = builder.build();
files = filesMetadata.keys().toArray(String.class);
}
}
private void addObjectMappers(ObjectMapper[] objectMappers) {
synchronized (mappersMutex) {
ImmutableOpenMap.Builder<String, ObjectMappers> fullPathObjectMappers =
ImmutableOpenMap.builder(this.fullPathObjectMappers);
for (ObjectMapper objectMapper : objectMappers) {
ObjectMappers mappers = fullPathObjectMappers.get(objectMapper.fullPath());
if (mappers == null) {
mappers = new ObjectMappers(objectMapper);
} else {
mappers = mappers.concat(objectMapper);
}
fullPathObjectMappers.put(objectMapper.fullPath(), mappers);
// update the hasNested flag
if (objectMapper.nested().isNested()) {
hasNested = true;
}
}
this.fullPathObjectMappers = fullPathObjectMappers.build();
}
}
private void removeObjectAndFieldMappers(DocumentMapper docMapper) {
synchronized (mappersMutex) {
fieldMappers.removeMappers(docMapper.mappers());
ImmutableOpenMap.Builder<String, ObjectMappers> fullPathObjectMappers =
ImmutableOpenMap.builder(this.fullPathObjectMappers);
for (ObjectMapper mapper : docMapper.objectMappers().values()) {
ObjectMappers mappers = fullPathObjectMappers.get(mapper.fullPath());
if (mappers != null) {
mappers = mappers.remove(mapper);
if (mappers.isEmpty()) {
fullPathObjectMappers.remove(mapper.fullPath());
} else {
fullPathObjectMappers.put(mapper.fullPath(), mappers);
}
}
}
this.fullPathObjectMappers = fullPathObjectMappers.build();
}
}
@Override
public void close() throws IOException {
out.close();
String checksum = null;
IndexOutput underlying = out;
// TODO: cut over to lucene's CRC
// *WARNING*: lucene has classes in same o.a.l.store package with very similar names,
// but using CRC, not Adler!
if (underlying instanceof BufferedChecksumIndexOutput) {
Checksum digest = ((BufferedChecksumIndexOutput) underlying).digest();
assert digest instanceof Adler32;
checksum = Long.toString(digest.getValue(), Character.MAX_RADIX);
}
synchronized (mutex) {
StoreFileMetaData md =
new StoreFileMetaData(
name, metaData.directory().fileLength(name), checksum, metaData.directory());
filesMetadata = ImmutableOpenMap.builder(filesMetadata).fPut(name, md).build();
files = filesMetadata.keys().toArray(String.class);
}
}
public IndexOutput createOutput(String name, IOContext context, boolean raw)
throws IOException {
ensureOpen();
Directory directory;
// we want to write the segments gen file to the same directory *all* the time
// to make sure we don't create multiple copies of it
if (isChecksum(name) || IndexFileNames.SEGMENTS_GEN.equals(name)) {
directory = distributor.primary();
} else {
directory = distributor.any();
}
IndexOutput out = directory.createOutput(name, context);
boolean success = false;
try {
synchronized (mutex) {
StoreFileMetaData metaData = new StoreFileMetaData(name, -1, null, directory);
filesMetadata = ImmutableOpenMap.builder(filesMetadata).fPut(name, metaData).build();
files = filesMetadata.keys().toArray(String.class);
boolean computeChecksum = !raw;
if (computeChecksum) {
// don't compute checksum for segment based files
if (IndexFileNames.SEGMENTS_GEN.equals(name)
|| name.startsWith(IndexFileNames.SEGMENTS)) {
computeChecksum = false;
}
}
if (computeChecksum) {
out = new BufferedChecksumIndexOutput(out, new Adler32());
}
final StoreIndexOutput storeIndexOutput = new StoreIndexOutput(metaData, out, name);
success = true;
return storeIndexOutput;
}
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(out);
}
}
}
ClusterBlocks(
Set<ClusterBlock> global, ImmutableOpenMap<String, Set<ClusterBlock>> indicesBlocks) {
this.global = global;
this.indicesBlocks = indicesBlocks;
levelHolders = new ImmutableLevelHolder[ClusterBlockLevel.values().length];
for (final ClusterBlockLevel level : ClusterBlockLevel.values()) {
Predicate<ClusterBlock> containsLevel = block -> block.contains(level);
Set<ClusterBlock> newGlobal =
unmodifiableSet(global.stream().filter(containsLevel).collect(toSet()));
ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder =
ImmutableOpenMap.builder();
for (ObjectObjectCursor<String, Set<ClusterBlock>> entry : indicesBlocks) {
indicesBuilder.put(
entry.key,
unmodifiableSet(entry.value.stream().filter(containsLevel).collect(toSet())));
}
levelHolders[level.id()] = new ImmutableLevelHolder(newGlobal, indicesBuilder.build());
}
}
@Override
public void deleteFile(String name) throws IOException {
ensureOpen();
// we don't allow to delete the checksums files, only using the deleteChecksum method
if (isChecksum(name)) {
return;
}
StoreFileMetaData metaData = filesMetadata.get(name);
if (metaData != null) {
try {
metaData.directory().deleteFile(name);
} catch (IOException e) {
if (metaData.directory().fileExists(name)) {
throw e;
}
}
}
synchronized (mutex) {
filesMetadata = ImmutableOpenMap.builder(filesMetadata).fRemove(name).build();
files = filesMetadata.keys().toArray(String.class);
}
}
/**
* Finds the specific index aliases that match with the specified aliases directly or partially
* via wildcards and that point to the specified concrete indices or match partially with the
* indices via wildcards.
*
* @param aliases The names of the index aliases to find
* @param concreteIndices The concrete indexes the index aliases must point to order to be
* returned.
* @return the found index aliases grouped by index
*/
public ImmutableOpenMap<String, List<AliasMetaData>> findAliases(
final String[] aliases, String[] concreteIndices) {
assert aliases != null;
assert concreteIndices != null;
if (concreteIndices.length == 0) {
return ImmutableOpenMap.of();
}
boolean matchAllAliases = matchAllAliases(aliases);
ImmutableOpenMap.Builder<String, List<AliasMetaData>> mapBuilder = ImmutableOpenMap.builder();
Iterable<String> intersection =
HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys());
for (String index : intersection) {
IndexMetaData indexMetaData = indices.get(index);
List<AliasMetaData> filteredValues = new ArrayList<>();
for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
AliasMetaData value = cursor.value;
if (matchAllAliases || Regex.simpleMatch(aliases, value.alias())) {
filteredValues.add(value);
}
}
if (!filteredValues.isEmpty()) {
// Make the list order deterministic
CollectionUtil.timSort(
filteredValues,
new Comparator<AliasMetaData>() {
@Override
public int compare(AliasMetaData o1, AliasMetaData o2) {
return o1.alias().compareTo(o2.alias());
}
});
mapBuilder.put(index, Collections.unmodifiableList(filteredValues));
}
}
return mapBuilder.build();
}
public Builder() {
clusterUUID = "_na_";
indices = ImmutableOpenMap.builder();
templates = ImmutableOpenMap.builder();
customs = ImmutableOpenMap.builder();
}
@Override
protected ImmutableOpenMap<Integer, V> createMap(Map values) {
return ImmutableOpenMap.<Integer, V>builder().putAll(values).build();
}
