@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
IndexMetaData that = (IndexMetaData) o;
if (!aliases.equals(that.aliases)) {
return false;
}
if (!index.equals(that.index)) {
return false;
}
if (!mappings.equals(that.mappings)) {
return false;
}
if (!settings.equals(that.settings)) {
return false;
}
if (state != that.state) {
return false;
}
if (!customs.equals(that.customs)) {
return false;
}
if (!activeAllocationIds.equals(that.activeAllocationIds)) {
return false;
}
return true;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(index.getName()); // uuid will come as part of settings
out.writeLong(version);
out.writeByte(state.id());
writeSettingsToStream(settings, out);
out.writeVInt(mappings.size());
for (ObjectCursor<MappingMetaData> cursor : mappings.values()) {
cursor.value.writeTo(out);
}
out.writeVInt(aliases.size());
for (ObjectCursor<AliasMetaData> cursor : aliases.values()) {
cursor.value.writeTo(out);
}
out.writeVInt(customs.size());
for (ObjectObjectCursor<String, Custom> cursor : customs) {
out.writeString(cursor.key);
cursor.value.writeTo(out);
}
out.writeVInt(activeAllocationIds.size());
for (IntObjectCursor<Set<String>> cursor : activeAllocationIds) {
out.writeVInt(cursor.key);
DiffableUtils.StringSetValueSerializer.getInstance().write(cursor.value, out);
}
}
/*
* Finds all mappings for types and concrete indices. Types are expanded to
* include all types that match the glob patterns in the types array. Empty
* types array, null or {"_all"} will be expanded to all types available for
* the given indices.
*/
public ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> findMappings(
String[] concreteIndices, final String[] types) {
assert types != null;
assert concreteIndices != null;
if (concreteIndices.length == 0) {
return ImmutableOpenMap.of();
}
ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder =
ImmutableOpenMap.builder();
Iterable<String> intersection =
HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys());
for (String index : intersection) {
IndexMetaData indexMetaData = indices.get(index);
ImmutableOpenMap.Builder<String, MappingMetaData> filteredMappings;
if (isAllTypes(types)) {
indexMapBuilder.put(
index, indexMetaData.getMappings()); // No types specified means get it all
} else {
filteredMappings = ImmutableOpenMap.builder();
for (ObjectObjectCursor<String, MappingMetaData> cursor : indexMetaData.getMappings()) {
if (Regex.simpleMatch(types, cursor.key)) {
filteredMappings.put(cursor.key, cursor.value);
}
}
if (!filteredMappings.isEmpty()) {
indexMapBuilder.put(index, filteredMappings.build());
}
}
}
return indexMapBuilder.build();
}
public Builder(String index) {
this.index = index;
this.mappings = ImmutableOpenMap.builder();
this.aliases = ImmutableOpenMap.builder();
this.customs = ImmutableOpenMap.builder();
this.activeAllocationIds = ImmutableOpenIntMap.builder();
}
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
if (cancellableThreads.isCancelled()
== false) { // no need to run this if recovery is canceled
IndexMetaData indexMetaData =
clusterService.state().metaData().getIndices().get(indexService.index().getName());
ImmutableOpenMap<String, MappingMetaData> metaDataMappings = null;
if (indexMetaData != null) {
metaDataMappings = indexMetaData.getMappings();
}
// default mapping should not be sent back, it can only be updated by put mapping API, and
// its
// a full in place replace, we don't want to override a potential update coming into it
for (DocumentMapper documentMapper : indexService.mapperService().docMappers(false)) {
MappingMetaData mappingMetaData =
metaDataMappings == null ? null : metaDataMappings.get(documentMapper.type());
if (mappingMetaData == null
|| !documentMapper.refreshSource().equals(mappingMetaData.source())) {
// not on master yet in the right form
documentMappersToUpdate.add(documentMapper);
}
}
}
return currentState;
}
/**
* Sometimes, the default mapping exists and an actual mapping is not created yet (introduced), in
* this case, we want to return the default mapping in case it has some default mapping
* definitions.
*
* <p>Note, once the mapping type is introduced, the default mapping is applied on the actual
* typed MappingMetaData, setting its routing, timestamp, and so on if needed.
*/
@Nullable
public MappingMetaData mappingOrDefault(String mappingType) {
MappingMetaData mapping = mappings.get(mappingType);
if (mapping != null) {
return mapping;
}
return mappings.get(MapperService.DEFAULT_MAPPING);
}
@Override
public boolean hasIndex(@NotNull ValueTable valueTable) {
ClusterStateResponse resp =
opalSearchService.getClient().admin().cluster().prepareState().execute().actionGet();
ImmutableOpenMap<String, MappingMetaData> mappings =
resp.getState().metaData().index(getName()).mappings();
return mappings.containsKey(getIndex(valueTable).getIndexName());
}
public Builder(MetaData metaData) {
this.clusterUUID = metaData.clusterUUID;
this.transientSettings = metaData.transientSettings;
this.persistentSettings = metaData.persistentSettings;
this.version = metaData.version;
this.indices = ImmutableOpenMap.builder(metaData.indices);
this.templates = ImmutableOpenMap.builder(metaData.templates);
this.customs = ImmutableOpenMap.builder(metaData.customs);
}
@Override
public int hashCode() {
int result = index.hashCode();
result = 31 * result + state.hashCode();
result = 31 * result + aliases.hashCode();
result = 31 * result + settings.hashCode();
result = 31 * result + mappings.hashCode();
result = 31 * result + activeAllocationIds.hashCode();
return result;
}
public Builder(IndexMetaData indexMetaData) {
this.index = indexMetaData.getIndex().getName();
this.state = indexMetaData.state;
this.version = indexMetaData.version;
this.settings = indexMetaData.getSettings();
this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings);
this.aliases = ImmutableOpenMap.builder(indexMetaData.aliases);
this.customs = ImmutableOpenMap.builder(indexMetaData.customs);
this.activeAllocationIds = ImmutableOpenIntMap.builder(indexMetaData.activeAllocationIds);
}
public void writeChecksum(String name, String checksum) throws IOException {
ensureOpen();
// update the metadata to include the checksum and write a new checksums file
synchronized (mutex) {
StoreFileMetaData metaData = filesMetadata.get(name);
metaData =
new StoreFileMetaData(metaData.name(), metaData.length(), checksum, metaData.directory());
filesMetadata = ImmutableOpenMap.builder(filesMetadata).fPut(name, metaData).build();
writeChecksums();
}
}
private static void assertMappingsHaveField(
GetMappingsResponse mappings, String index, String type, String field) throws IOException {
ImmutableOpenMap<String, MappingMetaData> indexMappings = mappings.getMappings().get("index");
assertNotNull(indexMappings);
MappingMetaData typeMappings = indexMappings.get(type);
assertNotNull(typeMappings);
Map<String, Object> typeMappingsMap = typeMappings.getSourceAsMap();
Map<String, Object> properties = (Map<String, Object>) typeMappingsMap.get("properties");
assertTrue(
"Could not find [" + field + "] in " + typeMappingsMap.toString(),
properties.containsKey(field));
}
public void assertVersionCreated(Version version, String... indices) {
GetSettingsResponse getSettingsResponse =
client().admin().indices().prepareGetSettings(indices).get();
ImmutableOpenMap<String, Settings> indexToSettings = getSettingsResponse.getIndexToSettings();
for (String index : indices) {
Settings settings = indexToSettings.get(index);
assertThat(
settings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null), notNullValue());
assertThat(
settings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null), equalTo(version));
}
}
private static void assertContext(ActionRequest<?> request, Map<String, String> context) {
if (context.size() == 0) {
assertThat(request.isContextEmpty(), is(true));
} else {
ImmutableOpenMap map = request.getContext();
assertThat(map, notNullValue());
assertThat(map.size(), equalTo(context.size()));
for (Object key : map.keys()) {
assertThat(context.get(key), equalTo(request.getFromContext(key)));
}
}
}
public void writeChecksums(Map<String, String> checksums) throws IOException {
ensureOpen();
// update the metadata to include the checksum and write a new checksums file
synchronized (mutex) {
for (Map.Entry<String, String> entry : checksums.entrySet()) {
StoreFileMetaData metaData = filesMetadata.get(entry.getKey());
metaData =
new StoreFileMetaData(
metaData.name(), metaData.length(), entry.getValue(), metaData.directory());
filesMetadata =
ImmutableOpenMap.builder(filesMetadata).fPut(entry.getKey(), metaData).build();
}
writeChecksums();
}
}
public void renameFile(String from, String to) throws IOException {
ensureOpen();
synchronized (mutex) {
StoreFileMetaData fromMetaData = filesMetadata.get(from); // we should always find this one
if (fromMetaData == null) {
throw new FileNotFoundException(from);
}
directoryService.renameFile(fromMetaData.directory(), from, to);
StoreFileMetaData toMetaData =
new StoreFileMetaData(
to, fromMetaData.length(), fromMetaData.checksum(), fromMetaData.directory());
filesMetadata =
ImmutableOpenMap.builder(filesMetadata).fRemove(from).fPut(to, toMetaData).build();
files = filesMetadata.keys().toArray(String.class);
}
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
int size = in.readVInt();
ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder =
ImmutableOpenMap.builder();
for (int i = 0; i < size; i++) {
String key = in.readString();
int valueSize = in.readVInt();
ImmutableOpenMap.Builder<String, MappingMetaData> typeMapBuilder = ImmutableOpenMap.builder();
for (int j = 0; j < valueSize; j++) {
typeMapBuilder.put(in.readString(), new MappingMetaData(in));
}
indexMapBuilder.put(key, typeMapBuilder.build());
}
mappings = indexMapBuilder.build();
}
/** Opened an index input in raw form, no decompression for example. */
public IndexInput openInputRaw(String name, IOContext context) throws IOException {
ensureOpen();
StoreFileMetaData metaData = filesMetadata.get(name);
if (metaData == null) {
throw new FileNotFoundException(name);
}
return metaData.directory().openInput(name, context);
}
public ClusterBlocks build() {
// We copy the block sets here in case of the builder is modified after build is called
ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder =
ImmutableOpenMap.builder(indices.size());
for (Map.Entry<String, Set<ClusterBlock>> entry : indices.entrySet()) {
indicesBuilder.put(entry.getKey(), unmodifiableSet(new HashSet<>(entry.getValue())));
}
return new ClusterBlocks(unmodifiableSet(new HashSet<>(global)), indicesBuilder.build());
}
/**
* @param concreteIndex The concrete index to check if routing is required
* @param type The type to check if routing is required
* @return Whether routing is required according to the mapping for the specified index and type
*/
public boolean routingRequired(String concreteIndex, String type) {
IndexMetaData indexMetaData = indices.get(concreteIndex);
if (indexMetaData != null) {
MappingMetaData mappingMetaData = indexMetaData.getMappings().get(type);
if (mappingMetaData != null) {
return mappingMetaData.routing().required();
}
}
return false;
}
@Override
public ClusterBlocks readFrom(StreamInput in) throws IOException {
Set<ClusterBlock> global = readBlockSet(in);
int size = in.readVInt();
ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder =
ImmutableOpenMap.builder(size);
for (int j = 0; j < size; j++) {
indicesBuilder.put(in.readString().intern(), readBlockSet(in));
}
return new ClusterBlocks(global, indicesBuilder.build());
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeLong(version);
out.writeString(clusterUUID);
writeSettingsToStream(transientSettings, out);
writeSettingsToStream(persistentSettings, out);
out.writeVInt(indices.size());
for (IndexMetaData indexMetaData : this) {
indexMetaData.writeTo(out);
}
out.writeVInt(templates.size());
for (ObjectCursor<IndexTemplateMetaData> cursor : templates.values()) {
cursor.value.writeTo(out);
}
out.writeVInt(customs.size());
for (ObjectObjectCursor<String, Custom> cursor : customs) {
out.writeString(cursor.key);
cursor.value.writeTo(out);
}
}
@Override
public void deleteType(String index, String type) {
ImmutableOpenMap<String, MappingMetaData> mappings =
client
.admin()
.cluster()
.prepareState()
.execute()
.actionGet()
.getState()
.metaData()
.index(index)
.mappings();
if (mappings.containsKey(type)) {
client
.admin()
.indices()
.deleteMapping(new DeleteMappingRequest(index).types(type))
.actionGet();
}
}
public class GetMappingsResponse extends ActionResponse {
private ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings =
ImmutableOpenMap.of();
GetMappingsResponse(
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings) {
this.mappings = mappings;
}
GetMappingsResponse() {}
public ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings() {
return mappings;
}
public ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> getMappings() {
return mappings();
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
int size = in.readVInt();
ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder =
ImmutableOpenMap.builder();
for (int i = 0; i < size; i++) {
String key = in.readString();
int valueSize = in.readVInt();
ImmutableOpenMap.Builder<String, MappingMetaData> typeMapBuilder = ImmutableOpenMap.builder();
for (int j = 0; j < valueSize; j++) {
typeMapBuilder.put(in.readString(), new MappingMetaData(in));
}
indexMapBuilder.put(key, typeMapBuilder.build());
}
mappings = indexMapBuilder.build();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeVInt(mappings.size());
for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> indexEntry :
mappings) {
out.writeString(indexEntry.key);
out.writeVInt(indexEntry.value.size());
for (ObjectObjectCursor<String, MappingMetaData> typeEntry : indexEntry.value) {
out.writeString(typeEntry.key);
typeEntry.value.writeTo(out);
}
}
}
}
public StoreFileMetaData metaData(String name) throws IOException {
ensureOpen();
StoreFileMetaData md = filesMetadata.get(name);
if (md == null) {
return null;
}
// IndexOutput not closed, does not exists
if (md.length() == -1) {
return null;
}
return md;
}
synchronized void closeInternal() throws IOException {
if (isOpen) {
isOpen = false;
for (Directory delegate : distributor.all()) {
delegate.close();
}
synchronized (mutex) {
filesMetadata = ImmutableOpenMap.of();
files = Strings.EMPTY_ARRAY;
}
}
}
protected void addMappers(
Collection<ObjectMapper> objectMappers, Collection<FieldMapper> fieldMappers) {
assert mappingLock.isWriteLockedByCurrentThread();
ImmutableOpenMap.Builder<String, ObjectMapper> fullPathObjectMappers =
ImmutableOpenMap.builder(this.fullPathObjectMappers);
for (ObjectMapper objectMapper : objectMappers) {
fullPathObjectMappers.put(objectMapper.fullPath(), objectMapper);
if (objectMapper.nested().isNested()) {
hasNested = true;
}
}
this.fullPathObjectMappers = fullPathObjectMappers.build();
this.fieldTypes = this.fieldTypes.copyAndAddAll(fieldMappers);
}
/**
* Finds the specific index aliases that match with the specified aliases directly or partially
* via wildcards and that point to the specified concrete indices or match partially with the
* indices via wildcards.
*
* @param aliases The names of the index aliases to find
* @param concreteIndices The concrete indexes the index aliases must point to order to be
* returned.
* @return the found index aliases grouped by index
*/
public ImmutableOpenMap<String, List<AliasMetaData>> findAliases(
final String[] aliases, String[] concreteIndices) {
assert aliases != null;
assert concreteIndices != null;
if (concreteIndices.length == 0) {
return ImmutableOpenMap.of();
}
boolean matchAllAliases = matchAllAliases(aliases);
ImmutableOpenMap.Builder<String, List<AliasMetaData>> mapBuilder = ImmutableOpenMap.builder();
Iterable<String> intersection =
HppcMaps.intersection(ObjectHashSet.from(concreteIndices), indices.keys());
for (String index : intersection) {
IndexMetaData indexMetaData = indices.get(index);
List<AliasMetaData> filteredValues = new ArrayList<>();
for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
AliasMetaData value = cursor.value;
if (matchAllAliases || Regex.simpleMatch(aliases, value.alias())) {
filteredValues.add(value);
}
}
if (!filteredValues.isEmpty()) {
// Make the list order deterministic
CollectionUtil.timSort(
filteredValues,
new Comparator<AliasMetaData>() {
@Override
public int compare(AliasMetaData o1, AliasMetaData o2) {
return o1.alias().compareTo(o2.alias());
}
});
mapBuilder.put(index, Collections.unmodifiableList(filteredValues));
}
}
return mapBuilder.build();
}
/**
* Returns the filter associated with listed filtering aliases.
*
* <p>The list of filtering aliases should be obtained by calling MetaData.filteringAliases.
* Returns <tt>null</tt> if no filtering is required.
*/
public Query aliasFilter(QueryShardContext context, String... aliasNames) {
if (aliasNames == null || aliasNames.length == 0) {
return null;
}
final ImmutableOpenMap<String, AliasMetaData> aliases =
indexSettings.getIndexMetaData().getAliases();
if (aliasNames.length == 1) {
AliasMetaData alias = aliases.get(aliasNames[0]);
if (alias == null) {
// This shouldn't happen unless alias disappeared after filteringAliases was called.
throw new InvalidAliasNameException(
index(), aliasNames[0], "Unknown alias name was passed to alias Filter");
}
return parse(alias, context);
} else {
// we need to bench here a bit, to see maybe it makes sense to use OrFilter
BooleanQuery.Builder combined = new BooleanQuery.Builder();
for (String aliasName : aliasNames) {
AliasMetaData alias = aliases.get(aliasName);
if (alias == null) {
// This shouldn't happen unless alias disappeared after filteringAliases was called.
throw new InvalidAliasNameException(
indexSettings.getIndex(),
aliasNames[0],
"Unknown alias name was passed to alias Filter");
}
Query parsedFilter = parse(alias, context);
if (parsedFilter != null) {
combined.add(parsedFilter, BooleanClause.Occur.SHOULD);
} else {
// The filter might be null only if filter was removed after filteringAliases was called
return null;
}
}
return combined.build();
}
}
/**
* Until we can verify that all instances have moved over to our new mapping, we need to handcheck
* all index mappings to make sure they have doc.associations.assoc_index as a doc value
*
* @param communityIdStrs
* @return
*/
@SuppressWarnings("unchecked")
private static boolean validateAssociationMapping(String[] communityIdStrs) {
// get all index mappings associated with these commids
String[] mappings = new String[communityIdStrs.length];
StringBuilder sb = new StringBuilder(", ");
for (int i = 0; i < communityIdStrs.length; i++) {
String s = communityIdStrs[i];
mappings[i] = "doc_" + s + "*";
sb.append("doc_").append(s).append("*, ");
}
ElasticSearchManager esm = ElasticSearchManager.getIndex(sb.substring(2, sb.length()));
GetMappingsResponse response =
esm.getRawClient().admin().indices().prepareGetMappings(mappings).get();
for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> mapping :
response.getMappings()) {
ImmutableOpenMap<String, MappingMetaData> mappingVal = mapping.value;
MappingMetaData mapping_meta = mappingVal.get("document_index");
try {
Map<String, Object> map = mapping_meta.getSourceAsMap();
Map<String, Object> props = (Map<String, Object>) map.get("properties");
Map<String, Object> assocs = (Map<String, Object>) props.get(DocumentPojo.associations_);
Map<String, Object> assocs_props = (Map<String, Object>) assocs.get("properties");
Map<String, Object> assoc_index =
(Map<String, Object>) assocs_props.get(AssociationPojo.assoc_index_);
if (!assoc_index.containsKey("doc_values") || !((Boolean) assoc_index.get("doc_values"))) {
// doc values doesn't exist in mapping or was false
return false;
}
} catch (Exception ex) {
// failed somehow
return false;
}
}
// if we fell through, all the checked indexes had the doc_value field set
return true;
}