private ImmutableList<ColumnIdent> getPrimaryKey() {
Map<String, Object> metaMap = getNested(defaultMappingMap, "_meta");
if (metaMap != null) {
ImmutableList.Builder<ColumnIdent> builder = ImmutableList.builder();
Object pKeys = metaMap.get("primary_keys");
if (pKeys != null) {
if (pKeys instanceof String) {
builder.add(ColumnIdent.fromPath((String) pKeys));
return builder.build();
} else if (pKeys instanceof Collection) {
Collection keys = (Collection) pKeys;
if (!keys.isEmpty()) {
for (Object pkey : keys) {
builder.add(ColumnIdent.fromPath(pkey.toString()));
}
return builder.build();
}
}
}
}
if (getCustomRoutingCol() == null && partitionedByList.isEmpty()) {
hasAutoGeneratedPrimaryKey = true;
return ImmutableList.of(ID_IDENT);
}
return ImmutableList.of();
}
private ColumnIdent getRoutingCol() {
ColumnIdent col = getCustomRoutingCol();
if (col != null) {
return col;
}
if (primaryKey.size() == 1) {
return primaryKey.get(0);
}
return ID_IDENT;
}
private ImmutableList<ColumnIdent> getPartitionedBy() {
ImmutableList.Builder<ColumnIdent> builder = ImmutableList.builder();
for (List<String> partitionedByInfo : partitionedByList) {
builder.add(ColumnIdent.fromPath(partitionedByInfo.get(0)));
}
return builder.build();
}
/**
* Returns true if the schema of this and <code>other</code> is the same, this includes the table
* name, as this is reflected in the ReferenceIdents of the columns.
*/
public boolean schemaEquals(DocIndexMetaData other) {
if (this == other) return true;
if (other == null) return false;
// TODO: when analyzers are exposed in the info, equality has to be checked on them
// see: TransportSQLActionTest.testSelectTableAliasSchemaExceptionColumnDefinition
if (columns != null ? !columns.equals(other.columns) : other.columns != null) return false;
if (primaryKey != null ? !primaryKey.equals(other.primaryKey) : other.primaryKey != null)
return false;
if (indices != null ? !indices.equals(other.indices) : other.indices != null) return false;
if (references != null ? !references.equals(other.references) : other.references != null)
return false;
if (routingCol != null ? !routingCol.equals(other.routingCol) : other.routingCol != null)
return false;
return true;
}
private ReferenceInfo newInfo(
ColumnIdent column,
DataType type,
ColumnPolicy columnPolicy,
ReferenceInfo.IndexType indexType) {
RowGranularity granularity = RowGranularity.DOC;
if (partitionedBy.contains(column)) {
granularity = RowGranularity.PARTITION;
}
return new ReferenceInfo(
new ReferenceIdent(ident, column), granularity, type, columnPolicy, indexType);
}
private void prepareCrateMeta() {
metaMap = getNested(defaultMappingMap, "_meta");
if (metaMap != null) {
indicesMap = getNested(metaMap, "indices");
if (indicesMap == null) {
indicesMap = ImmutableMap.of();
}
metaColumnsMap = getNested(metaMap, "columns");
if (metaColumnsMap == null) {
metaColumnsMap = ImmutableMap.of();
}
partitionedByList = getNested(metaMap, "partitioned_by");
if (partitionedByList == null) {
partitionedByList = ImmutableList.of();
}
} else {
metaMap = new HashMap<>();
indicesMap = new HashMap<>();
metaColumnsMap = new HashMap<>();
partitionedByList = ImmutableList.of();
}
}
private ColumnIdent childIdent(ColumnIdent ident, String name) {
if (ident == null) {
return new ColumnIdent(name);
}
if (ident.isColumn()) {
return new ColumnIdent(ident.name(), name);
} else {
ImmutableList.Builder<String> builder = ImmutableList.builder();
for (String s : ident.path()) {
builder.add(s);
}
builder.add(name);
return new ColumnIdent(ident.name(), builder.build());
}
}
public DocIndexMetaData build() {
partitionedBy = getPartitionedBy();
columnPolicy = getColumnPolicy();
createColumnDefinitions();
indices = createIndexDefinitions();
columns = ImmutableList.copyOf(columnsBuilder.build());
partitionedByColumns = partitionedByColumnsBuilder.build();
for (Tuple<ColumnIdent, ReferenceInfo> sysColumn : DocSysColumns.forTable(ident)) {
referencesBuilder.put(sysColumn.v1(), sysColumn.v2());
}
references = referencesBuilder.build();
primaryKey = getPrimaryKey();
routingCol = getRoutingCol();
return this;
}
private void add(
ColumnIdent column,
DataType type,
ColumnPolicy columnPolicy,
ReferenceInfo.IndexType indexType,
boolean partitioned) {
ReferenceInfo info = newInfo(column, type, columnPolicy, indexType);
// don't add it if there is a partitioned equivalent of this column
if (partitioned || !(partitionedBy != null && partitionedBy.contains(column))) {
if (info.ident().isColumn()) {
columnsBuilder.add(info);
}
referencesBuilder.put(info.ident().columnIdent(), info);
}
if (partitioned) {
partitionedByColumnsBuilder.add(info);
}
}
public class DocIndexMetaData {
private static final String ID = "_id";
public static final ColumnIdent ID_IDENT = new ColumnIdent(ID);
private final IndexMetaData metaData;
private final MappingMetaData defaultMappingMetaData;
private final Map<String, Object> defaultMappingMap;
private final Map<ColumnIdent, IndexReferenceInfo.Builder> indicesBuilder = new HashMap<>();
private final ImmutableSortedSet.Builder<ReferenceInfo> columnsBuilder =
ImmutableSortedSet.orderedBy(
new Comparator<ReferenceInfo>() {
@Override
public int compare(ReferenceInfo o1, ReferenceInfo o2) {
return o1.ident().columnIdent().fqn().compareTo(o2.ident().columnIdent().fqn());
}
});
// columns should be ordered
private final ImmutableMap.Builder<ColumnIdent, ReferenceInfo> referencesBuilder =
ImmutableSortedMap.naturalOrder();
private final ImmutableList.Builder<ReferenceInfo> partitionedByColumnsBuilder =
ImmutableList.builder();
private final TableIdent ident;
private final int numberOfShards;
private final BytesRef numberOfReplicas;
private final ImmutableMap<String, Object> tableParameters;
private Map<String, Object> metaMap;
private Map<String, Object> metaColumnsMap;
private Map<String, Object> indicesMap;
private List<List<String>> partitionedByList;
private ImmutableList<ReferenceInfo> columns;
private ImmutableMap<ColumnIdent, IndexReferenceInfo> indices;
private ImmutableList<ReferenceInfo> partitionedByColumns;
private ImmutableMap<ColumnIdent, ReferenceInfo> references;
private ImmutableList<ColumnIdent> primaryKey;
private ColumnIdent routingCol;
private ImmutableList<ColumnIdent> partitionedBy;
private final boolean isAlias;
private final Set<String> aliases;
private boolean hasAutoGeneratedPrimaryKey = false;
private ColumnPolicy columnPolicy = ColumnPolicy.DYNAMIC;
private static final ImmutableMap<String, DataType> dataTypeMap =
ImmutableMap.<String, DataType>builder()
.put("date", DataTypes.TIMESTAMP)
.put("string", DataTypes.STRING)
.put("boolean", DataTypes.BOOLEAN)
.put("byte", DataTypes.BYTE)
.put("short", DataTypes.SHORT)
.put("integer", DataTypes.INTEGER)
.put("long", DataTypes.LONG)
.put("float", DataTypes.FLOAT)
.put("double", DataTypes.DOUBLE)
.put("ip", DataTypes.IP)
.put("geo_point", DataTypes.GEO_POINT)
.put("object", DataTypes.OBJECT)
.put("nested", DataTypes.OBJECT)
.build();
public DocIndexMetaData(IndexMetaData metaData, TableIdent ident) throws IOException {
this.ident = ident;
this.metaData = metaData;
this.isAlias = !metaData.getIndex().equals(ident.esName());
this.numberOfShards = metaData.numberOfShards();
final Settings settings = metaData.getSettings();
this.numberOfReplicas = NumberOfReplicas.fromSettings(settings);
this.aliases = ImmutableSet.copyOf(metaData.aliases().keys().toArray(String.class));
this.defaultMappingMetaData = this.metaData.mappingOrDefault(Constants.DEFAULT_MAPPING_TYPE);
if (defaultMappingMetaData == null) {
this.defaultMappingMap = new HashMap<>();
} else {
this.defaultMappingMap = this.defaultMappingMetaData.sourceAsMap();
}
this.tableParameters = TableParameterInfo.tableParametersFromIndexMetaData(metaData);
prepareCrateMeta();
}
@SuppressWarnings("unchecked")
private static <T> T getNested(Map map, String key) {
return (T) map.get(key);
}
private void prepareCrateMeta() {
metaMap = getNested(defaultMappingMap, "_meta");
if (metaMap != null) {
indicesMap = getNested(metaMap, "indices");
if (indicesMap == null) {
indicesMap = ImmutableMap.of();
}
metaColumnsMap = getNested(metaMap, "columns");
if (metaColumnsMap == null) {
metaColumnsMap = ImmutableMap.of();
}
partitionedByList = getNested(metaMap, "partitioned_by");
if (partitionedByList == null) {
partitionedByList = ImmutableList.of();
}
} else {
metaMap = new HashMap<>();
indicesMap = new HashMap<>();
metaColumnsMap = new HashMap<>();
partitionedByList = ImmutableList.of();
}
}
private void addPartitioned(ColumnIdent column, DataType type) {
add(column, type, ColumnPolicy.DYNAMIC, ReferenceInfo.IndexType.NOT_ANALYZED, true);
}
private void add(ColumnIdent column, DataType type, ReferenceInfo.IndexType indexType) {
add(column, type, ColumnPolicy.DYNAMIC, indexType, false);
}
private void add(
ColumnIdent column,
DataType type,
ColumnPolicy columnPolicy,
ReferenceInfo.IndexType indexType,
boolean partitioned) {
ReferenceInfo info = newInfo(column, type, columnPolicy, indexType);
// don't add it if there is a partitioned equivalent of this column
if (partitioned || !(partitionedBy != null && partitionedBy.contains(column))) {
if (info.ident().isColumn()) {
columnsBuilder.add(info);
}
referencesBuilder.put(info.ident().columnIdent(), info);
}
if (partitioned) {
partitionedByColumnsBuilder.add(info);
}
}
private ReferenceInfo newInfo(
ColumnIdent column,
DataType type,
ColumnPolicy columnPolicy,
ReferenceInfo.IndexType indexType) {
RowGranularity granularity = RowGranularity.DOC;
if (partitionedBy.contains(column)) {
granularity = RowGranularity.PARTITION;
}
return new ReferenceInfo(
new ReferenceIdent(ident, column), granularity, type, columnPolicy, indexType);
}
/**
* extract dataType from given columnProperties
*
* @param columnProperties map of String to Object containing column properties
* @return dataType of the column with columnProperties
*/
public static DataType getColumnDataType(Map<String, Object> columnProperties) {
DataType type;
String typeName = (String) columnProperties.get("type");
if (typeName == null) {
if (columnProperties.containsKey("properties")) {
type = DataTypes.OBJECT;
} else {
return DataTypes.NOT_SUPPORTED;
}
} else if (typeName.equalsIgnoreCase("array")) {
Map<String, Object> innerProperties = getNested(columnProperties, "inner");
DataType innerType = getColumnDataType(innerProperties);
type = new ArrayType(innerType);
} else {
typeName = typeName.toLowerCase(Locale.ENGLISH);
type = MoreObjects.firstNonNull(dataTypeMap.get(typeName), DataTypes.NOT_SUPPORTED);
}
return type;
}
private ReferenceInfo.IndexType getColumnIndexType(Map<String, Object> columnProperties) {
String indexType = (String) columnProperties.get("index");
String analyzerName = (String) columnProperties.get("analyzer");
if (indexType != null) {
if (indexType.equals(ReferenceInfo.IndexType.NOT_ANALYZED.toString())) {
return ReferenceInfo.IndexType.NOT_ANALYZED;
} else if (indexType.equals(ReferenceInfo.IndexType.NO.toString())) {
return ReferenceInfo.IndexType.NO;
} else if (indexType.equals(ReferenceInfo.IndexType.ANALYZED.toString())
&& analyzerName != null
&& !analyzerName.equals("keyword")) {
return ReferenceInfo.IndexType.ANALYZED;
}
} // default indexType is analyzed so need to check analyzerName if indexType is null
else if (analyzerName != null && !analyzerName.equals("keyword")) {
return ReferenceInfo.IndexType.ANALYZED;
}
return ReferenceInfo.IndexType.NOT_ANALYZED;
}
private ColumnIdent childIdent(ColumnIdent ident, String name) {
if (ident == null) {
return new ColumnIdent(name);
}
if (ident.isColumn()) {
return new ColumnIdent(ident.name(), name);
} else {
ImmutableList.Builder<String> builder = ImmutableList.builder();
for (String s : ident.path()) {
builder.add(s);
}
builder.add(name);
return new ColumnIdent(ident.name(), builder.build());
}
}
/** extracts index definitions as well */
@SuppressWarnings("unchecked")
private void internalExtractColumnDefinitions(
ColumnIdent columnIdent, Map<String, Object> propertiesMap) {
if (propertiesMap == null) {
return;
}
for (Map.Entry<String, Object> columnEntry : propertiesMap.entrySet()) {
Map<String, Object> columnProperties = (Map) columnEntry.getValue();
DataType columnDataType = getColumnDataType(columnProperties);
ColumnIdent newIdent = childIdent(columnIdent, columnEntry.getKey());
columnProperties = furtherColumnProperties(columnProperties);
ReferenceInfo.IndexType columnIndexType = getColumnIndexType(columnProperties);
if (columnDataType == DataTypes.OBJECT
|| (columnDataType.id() == ArrayType.ID
&& ((ArrayType) columnDataType).innerType() == DataTypes.OBJECT)) {
ColumnPolicy columnPolicy = ColumnPolicy.of(columnProperties.get("dynamic"));
add(newIdent, columnDataType, columnPolicy, ReferenceInfo.IndexType.NO, false);
if (columnProperties.get("properties") != null) {
// walk nested
internalExtractColumnDefinitions(
newIdent, (Map<String, Object>) columnProperties.get("properties"));
}
} else if (columnDataType != DataTypes.NOT_SUPPORTED) {
List<String> copyToColumns = getNested(columnProperties, "copy_to");
// extract columns this column is copied to, needed for indices
if (copyToColumns != null) {
for (String copyToColumn : copyToColumns) {
ColumnIdent targetIdent = ColumnIdent.fromPath(copyToColumn);
IndexReferenceInfo.Builder builder = getOrCreateIndexBuilder(targetIdent);
builder.addColumn(
newInfo(newIdent, columnDataType, ColumnPolicy.DYNAMIC, columnIndexType));
}
}
// is it an index?
if (indicesMap.containsKey(newIdent.fqn())) {
IndexReferenceInfo.Builder builder = getOrCreateIndexBuilder(newIdent);
builder
.indexType(columnIndexType)
.ident(new ReferenceIdent(ident, newIdent))
.analyzer((String) columnProperties.get("analyzer"));
} else {
add(newIdent, columnDataType, columnIndexType);
}
}
}
}
/**
* get the real column properties from a possible array mapping, keeping most of this stuff inside
* "inner"
*/
private Map<String, Object> furtherColumnProperties(Map<String, Object> columnProperties) {
if (columnProperties.get("inner") != null) {
return (Map<String, Object>) columnProperties.get("inner");
} else {
return columnProperties;
}
}
private IndexReferenceInfo.Builder getOrCreateIndexBuilder(ColumnIdent ident) {
IndexReferenceInfo.Builder builder = indicesBuilder.get(ident);
if (builder == null) {
builder = new IndexReferenceInfo.Builder();
indicesBuilder.put(ident, builder);
}
return builder;
}
private ImmutableList<ColumnIdent> getPrimaryKey() {
Map<String, Object> metaMap = getNested(defaultMappingMap, "_meta");
if (metaMap != null) {
ImmutableList.Builder<ColumnIdent> builder = ImmutableList.builder();
Object pKeys = metaMap.get("primary_keys");
if (pKeys != null) {
if (pKeys instanceof String) {
builder.add(ColumnIdent.fromPath((String) pKeys));
return builder.build();
} else if (pKeys instanceof Collection) {
Collection keys = (Collection) pKeys;
if (!keys.isEmpty()) {
for (Object pkey : keys) {
builder.add(ColumnIdent.fromPath(pkey.toString()));
}
return builder.build();
}
}
}
}
if (getCustomRoutingCol() == null && partitionedByList.isEmpty()) {
hasAutoGeneratedPrimaryKey = true;
return ImmutableList.of(ID_IDENT);
}
return ImmutableList.of();
}
private ImmutableList<ColumnIdent> getPartitionedBy() {
ImmutableList.Builder<ColumnIdent> builder = ImmutableList.builder();
for (List<String> partitionedByInfo : partitionedByList) {
builder.add(ColumnIdent.fromPath(partitionedByInfo.get(0)));
}
return builder.build();
}
private ColumnPolicy getColumnPolicy() {
Object dynamic = getNested(defaultMappingMap, "dynamic");
if (ColumnPolicy.STRICT.value().equals(String.valueOf(dynamic).toLowerCase(Locale.ENGLISH))) {
return ColumnPolicy.STRICT;
} else if (Booleans.isExplicitFalse(String.valueOf(dynamic))) {
return ColumnPolicy.IGNORED;
} else {
return ColumnPolicy.DYNAMIC;
}
}
private void createColumnDefinitions() {
Map<String, Object> propertiesMap = getNested(defaultMappingMap, "properties");
internalExtractColumnDefinitions(null, propertiesMap);
extractPartitionedByColumns();
}
private ImmutableMap<ColumnIdent, IndexReferenceInfo> createIndexDefinitions() {
ImmutableMap.Builder<ColumnIdent, IndexReferenceInfo> builder = ImmutableMap.builder();
for (Map.Entry<ColumnIdent, IndexReferenceInfo.Builder> entry : indicesBuilder.entrySet()) {
builder.put(entry.getKey(), entry.getValue().build());
}
indices = builder.build();
return indices;
}
private void extractPartitionedByColumns() {
for (Tuple<ColumnIdent, DataType> partitioned :
PartitionedByMappingExtractor.extractPartitionedByColumns(partitionedByList)) {
addPartitioned(partitioned.v1(), partitioned.v2());
}
}
private ColumnIdent getCustomRoutingCol() {
if (defaultMappingMetaData != null) {
Map<String, Object> metaMap = getNested(defaultMappingMap, "_meta");
if (metaMap != null) {
String routingPath = (String) metaMap.get("routing");
if (routingPath != null && !routingPath.equals(ID)) {
return ColumnIdent.fromPath(routingPath);
}
}
}
return null;
}
private ColumnIdent getRoutingCol() {
ColumnIdent col = getCustomRoutingCol();
if (col != null) {
return col;
}
if (primaryKey.size() == 1) {
return primaryKey.get(0);
}
return ID_IDENT;
}
public DocIndexMetaData build() {
partitionedBy = getPartitionedBy();
columnPolicy = getColumnPolicy();
createColumnDefinitions();
indices = createIndexDefinitions();
columns = ImmutableList.copyOf(columnsBuilder.build());
partitionedByColumns = partitionedByColumnsBuilder.build();
for (Tuple<ColumnIdent, ReferenceInfo> sysColumn : DocSysColumns.forTable(ident)) {
referencesBuilder.put(sysColumn.v1(), sysColumn.v2());
}
references = referencesBuilder.build();
primaryKey = getPrimaryKey();
routingCol = getRoutingCol();
return this;
}
public ImmutableMap<ColumnIdent, ReferenceInfo> references() {
return references;
}
public ImmutableList<ReferenceInfo> columns() {
return columns;
}
public ImmutableMap<ColumnIdent, IndexReferenceInfo> indices() {
return indices;
}
public ImmutableList<ReferenceInfo> partitionedByColumns() {
return partitionedByColumns;
}
public ImmutableList<ColumnIdent> primaryKey() {
return primaryKey;
}
public ColumnIdent routingCol() {
return routingCol;
}
/**
* Returns true if the schema of this and <code>other</code> is the same, this includes the table
* name, as this is reflected in the ReferenceIdents of the columns.
*/
public boolean schemaEquals(DocIndexMetaData other) {
if (this == other) return true;
if (other == null) return false;
// TODO: when analyzers are exposed in the info, equality has to be checked on them
// see: TransportSQLActionTest.testSelectTableAliasSchemaExceptionColumnDefinition
if (columns != null ? !columns.equals(other.columns) : other.columns != null) return false;
if (primaryKey != null ? !primaryKey.equals(other.primaryKey) : other.primaryKey != null)
return false;
if (indices != null ? !indices.equals(other.indices) : other.indices != null) return false;
if (references != null ? !references.equals(other.references) : other.references != null)
return false;
if (routingCol != null ? !routingCol.equals(other.routingCol) : other.routingCol != null)
return false;
return true;
}
protected DocIndexMetaData merge(
DocIndexMetaData other,
TransportPutIndexTemplateAction transportPutIndexTemplateAction,
boolean thisIsCreatedFromTemplate)
throws IOException {
if (schemaEquals(other)) {
return this;
} else if (thisIsCreatedFromTemplate) {
if (this.references.size() < other.references.size()) {
// this is older, update template and return other
// settings in template are always authoritative for table information about
// number_of_shards and number_of_replicas
updateTemplate(other, transportPutIndexTemplateAction, this.metaData.settings());
// merge the new mapping with the template settings
return new DocIndexMetaData(
IndexMetaData.builder(other.metaData).settings(this.metaData.settings()).build(),
other.ident)
.build();
} else if (references().size() == other.references().size()
&& !references().keySet().equals(other.references().keySet())) {
XContentHelper.update(defaultMappingMap, other.defaultMappingMap, false);
// update the template with new information
updateTemplate(this, transportPutIndexTemplateAction, this.metaData.settings());
return this;
}
// other is older, just return this
return this;
} else {
throw new TableAliasSchemaException(other.ident.name());
}
}
private void updateTemplate(
DocIndexMetaData md,
TransportPutIndexTemplateAction transportPutIndexTemplateAction,
Settings updateSettings) {
String templateName = PartitionName.templateName(ident.schema(), ident.name());
PutIndexTemplateRequest request =
new PutIndexTemplateRequest(templateName)
.mapping(Constants.DEFAULT_MAPPING_TYPE, md.defaultMappingMap)
.create(false)
.settings(updateSettings)
.template(templateName + "*");
for (String alias : md.aliases()) {
request = request.alias(new Alias(alias));
}
transportPutIndexTemplateAction.execute(request);
}
/** @return the name of the underlying index even if this table is referenced by alias */
public String concreteIndexName() {
return metaData.index();
}
public boolean isAlias() {
return isAlias;
}
public Set<String> aliases() {
return aliases;
}
public boolean hasAutoGeneratedPrimaryKey() {
return hasAutoGeneratedPrimaryKey;
}
public int numberOfShards() {
return numberOfShards;
}
public BytesRef numberOfReplicas() {
return numberOfReplicas;
}
public ImmutableList<ColumnIdent> partitionedBy() {
return partitionedBy;
}
public ColumnPolicy columnPolicy() {
return columnPolicy;
}
public ImmutableMap<String, Object> tableParameters() {
return tableParameters;
}
private ImmutableMap<ColumnIdent, String> getAnalyzers(
ColumnIdent columnIdent, Map<String, Object> propertiesMap) {
ImmutableMap.Builder<ColumnIdent, String> builder = ImmutableMap.builder();
for (Map.Entry<String, Object> columnEntry : propertiesMap.entrySet()) {
Map<String, Object> columnProperties = (Map) columnEntry.getValue();
DataType columnDataType = getColumnDataType(columnProperties);
ColumnIdent newIdent = childIdent(columnIdent, columnEntry.getKey());
columnProperties = furtherColumnProperties(columnProperties);
if (columnDataType == DataTypes.OBJECT
|| (columnDataType.id() == ArrayType.ID
&& ((ArrayType) columnDataType).innerType() == DataTypes.OBJECT)) {
if (columnProperties.get("properties") != null) {
builder.putAll(
getAnalyzers(newIdent, (Map<String, Object>) columnProperties.get("properties")));
}
}
String analyzer = (String) columnProperties.get("analyzer");
if (analyzer != null) {
builder.put(newIdent, analyzer);
}
}
return builder.build();
}
public ImmutableMap<ColumnIdent, String> analyzers() {
Map<String, Object> propertiesMap = getNested(defaultMappingMap, "properties");
if (propertiesMap == null) {
return ImmutableMap.of();
} else {
return getAnalyzers(null, propertiesMap);
}
}
}