コード例 #1
0
ファイル: DocIndexMetaData.java プロジェクト: proofy/crate
 private ImmutableList<ColumnIdent> getPrimaryKey() {
   Map<String, Object> metaMap = getNested(defaultMappingMap, "_meta");
   if (metaMap != null) {
     ImmutableList.Builder<ColumnIdent> builder = ImmutableList.builder();
     Object pKeys = metaMap.get("primary_keys");
     if (pKeys != null) {
       if (pKeys instanceof String) {
         builder.add(ColumnIdent.fromPath((String) pKeys));
         return builder.build();
       } else if (pKeys instanceof Collection) {
         Collection keys = (Collection) pKeys;
         if (!keys.isEmpty()) {
           for (Object pkey : keys) {
             builder.add(ColumnIdent.fromPath(pkey.toString()));
           }
           return builder.build();
         }
       }
     }
   }
   if (getCustomRoutingCol() == null && partitionedByList.isEmpty()) {
     hasAutoGeneratedPrimaryKey = true;
     return ImmutableList.of(ID_IDENT);
   }
   return ImmutableList.of();
 }
コード例 #2
0
ファイル: DocIndexMetaData.java プロジェクト: proofy/crate
 private ImmutableList<ColumnIdent> getPartitionedBy() {
   ImmutableList.Builder<ColumnIdent> builder = ImmutableList.builder();
   for (List<String> partitionedByInfo : partitionedByList) {
     builder.add(ColumnIdent.fromPath(partitionedByInfo.get(0)));
   }
   return builder.build();
 }
コード例 #3
0
ファイル: DocIndexMetaData.java プロジェクト: proofy/crate
 private ColumnIdent childIdent(ColumnIdent ident, String name) {
   if (ident == null) {
     return new ColumnIdent(name);
   }
   if (ident.isColumn()) {
     return new ColumnIdent(ident.name(), name);
   } else {
     ImmutableList.Builder<String> builder = ImmutableList.builder();
     for (String s : ident.path()) {
       builder.add(s);
     }
     builder.add(name);
     return new ColumnIdent(ident.name(), builder.build());
   }
 }
コード例 #4
0
ファイル: DocIndexMetaData.java プロジェクト: proofy/crate
public class DocIndexMetaData {

  private static final String ID = "_id";
  public static final ColumnIdent ID_IDENT = new ColumnIdent(ID);
  private final IndexMetaData metaData;

  private final MappingMetaData defaultMappingMetaData;
  private final Map<String, Object> defaultMappingMap;

  private final Map<ColumnIdent, IndexReferenceInfo.Builder> indicesBuilder = new HashMap<>();

  private final ImmutableSortedSet.Builder<ReferenceInfo> columnsBuilder =
      ImmutableSortedSet.orderedBy(
          new Comparator<ReferenceInfo>() {
            @Override
            public int compare(ReferenceInfo o1, ReferenceInfo o2) {
              return o1.ident().columnIdent().fqn().compareTo(o2.ident().columnIdent().fqn());
            }
          });

  // columns should be ordered
  private final ImmutableMap.Builder<ColumnIdent, ReferenceInfo> referencesBuilder =
      ImmutableSortedMap.naturalOrder();
  private final ImmutableList.Builder<ReferenceInfo> partitionedByColumnsBuilder =
      ImmutableList.builder();

  private final TableIdent ident;
  private final int numberOfShards;
  private final BytesRef numberOfReplicas;
  private final ImmutableMap<String, Object> tableParameters;
  private Map<String, Object> metaMap;
  private Map<String, Object> metaColumnsMap;
  private Map<String, Object> indicesMap;
  private List<List<String>> partitionedByList;
  private ImmutableList<ReferenceInfo> columns;
  private ImmutableMap<ColumnIdent, IndexReferenceInfo> indices;
  private ImmutableList<ReferenceInfo> partitionedByColumns;
  private ImmutableMap<ColumnIdent, ReferenceInfo> references;
  private ImmutableList<ColumnIdent> primaryKey;
  private ColumnIdent routingCol;
  private ImmutableList<ColumnIdent> partitionedBy;
  private final boolean isAlias;
  private final Set<String> aliases;
  private boolean hasAutoGeneratedPrimaryKey = false;

  private ColumnPolicy columnPolicy = ColumnPolicy.DYNAMIC;

  private static final ImmutableMap<String, DataType> dataTypeMap =
      ImmutableMap.<String, DataType>builder()
          .put("date", DataTypes.TIMESTAMP)
          .put("string", DataTypes.STRING)
          .put("boolean", DataTypes.BOOLEAN)
          .put("byte", DataTypes.BYTE)
          .put("short", DataTypes.SHORT)
          .put("integer", DataTypes.INTEGER)
          .put("long", DataTypes.LONG)
          .put("float", DataTypes.FLOAT)
          .put("double", DataTypes.DOUBLE)
          .put("ip", DataTypes.IP)
          .put("geo_point", DataTypes.GEO_POINT)
          .put("object", DataTypes.OBJECT)
          .put("nested", DataTypes.OBJECT)
          .build();

  public DocIndexMetaData(IndexMetaData metaData, TableIdent ident) throws IOException {
    this.ident = ident;
    this.metaData = metaData;
    this.isAlias = !metaData.getIndex().equals(ident.esName());
    this.numberOfShards = metaData.numberOfShards();
    final Settings settings = metaData.getSettings();
    this.numberOfReplicas = NumberOfReplicas.fromSettings(settings);
    this.aliases = ImmutableSet.copyOf(metaData.aliases().keys().toArray(String.class));
    this.defaultMappingMetaData = this.metaData.mappingOrDefault(Constants.DEFAULT_MAPPING_TYPE);
    if (defaultMappingMetaData == null) {
      this.defaultMappingMap = new HashMap<>();
    } else {
      this.defaultMappingMap = this.defaultMappingMetaData.sourceAsMap();
    }
    this.tableParameters = TableParameterInfo.tableParametersFromIndexMetaData(metaData);

    prepareCrateMeta();
  }

  @SuppressWarnings("unchecked")
  private static <T> T getNested(Map map, String key) {
    return (T) map.get(key);
  }

  private void prepareCrateMeta() {
    metaMap = getNested(defaultMappingMap, "_meta");
    if (metaMap != null) {
      indicesMap = getNested(metaMap, "indices");
      if (indicesMap == null) {
        indicesMap = ImmutableMap.of();
      }
      metaColumnsMap = getNested(metaMap, "columns");
      if (metaColumnsMap == null) {
        metaColumnsMap = ImmutableMap.of();
      }
      partitionedByList = getNested(metaMap, "partitioned_by");
      if (partitionedByList == null) {
        partitionedByList = ImmutableList.of();
      }
    } else {
      metaMap = new HashMap<>();
      indicesMap = new HashMap<>();
      metaColumnsMap = new HashMap<>();
      partitionedByList = ImmutableList.of();
    }
  }

  private void addPartitioned(ColumnIdent column, DataType type) {
    add(column, type, ColumnPolicy.DYNAMIC, ReferenceInfo.IndexType.NOT_ANALYZED, true);
  }

  private void add(ColumnIdent column, DataType type, ReferenceInfo.IndexType indexType) {
    add(column, type, ColumnPolicy.DYNAMIC, indexType, false);
  }

  private void add(
      ColumnIdent column,
      DataType type,
      ColumnPolicy columnPolicy,
      ReferenceInfo.IndexType indexType,
      boolean partitioned) {
    ReferenceInfo info = newInfo(column, type, columnPolicy, indexType);
    // don't add it if there is a partitioned equivalent of this column
    if (partitioned || !(partitionedBy != null && partitionedBy.contains(column))) {
      if (info.ident().isColumn()) {
        columnsBuilder.add(info);
      }
      referencesBuilder.put(info.ident().columnIdent(), info);
    }
    if (partitioned) {
      partitionedByColumnsBuilder.add(info);
    }
  }

  private ReferenceInfo newInfo(
      ColumnIdent column,
      DataType type,
      ColumnPolicy columnPolicy,
      ReferenceInfo.IndexType indexType) {
    RowGranularity granularity = RowGranularity.DOC;
    if (partitionedBy.contains(column)) {
      granularity = RowGranularity.PARTITION;
    }
    return new ReferenceInfo(
        new ReferenceIdent(ident, column), granularity, type, columnPolicy, indexType);
  }

  /**
   * extract dataType from given columnProperties
   *
   * @param columnProperties map of String to Object containing column properties
   * @return dataType of the column with columnProperties
   */
  public static DataType getColumnDataType(Map<String, Object> columnProperties) {
    DataType type;
    String typeName = (String) columnProperties.get("type");

    if (typeName == null) {
      if (columnProperties.containsKey("properties")) {
        type = DataTypes.OBJECT;
      } else {
        return DataTypes.NOT_SUPPORTED;
      }
    } else if (typeName.equalsIgnoreCase("array")) {

      Map<String, Object> innerProperties = getNested(columnProperties, "inner");
      DataType innerType = getColumnDataType(innerProperties);
      type = new ArrayType(innerType);
    } else {
      typeName = typeName.toLowerCase(Locale.ENGLISH);
      type = MoreObjects.firstNonNull(dataTypeMap.get(typeName), DataTypes.NOT_SUPPORTED);
    }
    return type;
  }

  private ReferenceInfo.IndexType getColumnIndexType(Map<String, Object> columnProperties) {
    String indexType = (String) columnProperties.get("index");
    String analyzerName = (String) columnProperties.get("analyzer");
    if (indexType != null) {
      if (indexType.equals(ReferenceInfo.IndexType.NOT_ANALYZED.toString())) {
        return ReferenceInfo.IndexType.NOT_ANALYZED;
      } else if (indexType.equals(ReferenceInfo.IndexType.NO.toString())) {
        return ReferenceInfo.IndexType.NO;
      } else if (indexType.equals(ReferenceInfo.IndexType.ANALYZED.toString())
          && analyzerName != null
          && !analyzerName.equals("keyword")) {
        return ReferenceInfo.IndexType.ANALYZED;
      }
    } // default indexType is analyzed so need to check analyzerName if indexType is null
    else if (analyzerName != null && !analyzerName.equals("keyword")) {
      return ReferenceInfo.IndexType.ANALYZED;
    }
    return ReferenceInfo.IndexType.NOT_ANALYZED;
  }

  private ColumnIdent childIdent(ColumnIdent ident, String name) {
    if (ident == null) {
      return new ColumnIdent(name);
    }
    if (ident.isColumn()) {
      return new ColumnIdent(ident.name(), name);
    } else {
      ImmutableList.Builder<String> builder = ImmutableList.builder();
      for (String s : ident.path()) {
        builder.add(s);
      }
      builder.add(name);
      return new ColumnIdent(ident.name(), builder.build());
    }
  }

  /** extracts index definitions as well */
  @SuppressWarnings("unchecked")
  private void internalExtractColumnDefinitions(
      ColumnIdent columnIdent, Map<String, Object> propertiesMap) {
    if (propertiesMap == null) {
      return;
    }

    for (Map.Entry<String, Object> columnEntry : propertiesMap.entrySet()) {
      Map<String, Object> columnProperties = (Map) columnEntry.getValue();
      DataType columnDataType = getColumnDataType(columnProperties);
      ColumnIdent newIdent = childIdent(columnIdent, columnEntry.getKey());

      columnProperties = furtherColumnProperties(columnProperties);
      ReferenceInfo.IndexType columnIndexType = getColumnIndexType(columnProperties);
      if (columnDataType == DataTypes.OBJECT
          || (columnDataType.id() == ArrayType.ID
              && ((ArrayType) columnDataType).innerType() == DataTypes.OBJECT)) {
        ColumnPolicy columnPolicy = ColumnPolicy.of(columnProperties.get("dynamic"));
        add(newIdent, columnDataType, columnPolicy, ReferenceInfo.IndexType.NO, false);

        if (columnProperties.get("properties") != null) {
          // walk nested
          internalExtractColumnDefinitions(
              newIdent, (Map<String, Object>) columnProperties.get("properties"));
        }
      } else if (columnDataType != DataTypes.NOT_SUPPORTED) {
        List<String> copyToColumns = getNested(columnProperties, "copy_to");

        // extract columns this column is copied to, needed for indices
        if (copyToColumns != null) {
          for (String copyToColumn : copyToColumns) {
            ColumnIdent targetIdent = ColumnIdent.fromPath(copyToColumn);
            IndexReferenceInfo.Builder builder = getOrCreateIndexBuilder(targetIdent);
            builder.addColumn(
                newInfo(newIdent, columnDataType, ColumnPolicy.DYNAMIC, columnIndexType));
          }
        }
        // is it an index?
        if (indicesMap.containsKey(newIdent.fqn())) {
          IndexReferenceInfo.Builder builder = getOrCreateIndexBuilder(newIdent);
          builder
              .indexType(columnIndexType)
              .ident(new ReferenceIdent(ident, newIdent))
              .analyzer((String) columnProperties.get("analyzer"));
        } else {
          add(newIdent, columnDataType, columnIndexType);
        }
      }
    }
  }

  /**
   * get the real column properties from a possible array mapping, keeping most of this stuff inside
   * "inner"
   */
  private Map<String, Object> furtherColumnProperties(Map<String, Object> columnProperties) {
    if (columnProperties.get("inner") != null) {
      return (Map<String, Object>) columnProperties.get("inner");
    } else {
      return columnProperties;
    }
  }

  private IndexReferenceInfo.Builder getOrCreateIndexBuilder(ColumnIdent ident) {
    IndexReferenceInfo.Builder builder = indicesBuilder.get(ident);
    if (builder == null) {
      builder = new IndexReferenceInfo.Builder();
      indicesBuilder.put(ident, builder);
    }
    return builder;
  }

  private ImmutableList<ColumnIdent> getPrimaryKey() {
    Map<String, Object> metaMap = getNested(defaultMappingMap, "_meta");
    if (metaMap != null) {
      ImmutableList.Builder<ColumnIdent> builder = ImmutableList.builder();
      Object pKeys = metaMap.get("primary_keys");
      if (pKeys != null) {
        if (pKeys instanceof String) {
          builder.add(ColumnIdent.fromPath((String) pKeys));
          return builder.build();
        } else if (pKeys instanceof Collection) {
          Collection keys = (Collection) pKeys;
          if (!keys.isEmpty()) {
            for (Object pkey : keys) {
              builder.add(ColumnIdent.fromPath(pkey.toString()));
            }
            return builder.build();
          }
        }
      }
    }
    if (getCustomRoutingCol() == null && partitionedByList.isEmpty()) {
      hasAutoGeneratedPrimaryKey = true;
      return ImmutableList.of(ID_IDENT);
    }
    return ImmutableList.of();
  }

  private ImmutableList<ColumnIdent> getPartitionedBy() {
    ImmutableList.Builder<ColumnIdent> builder = ImmutableList.builder();
    for (List<String> partitionedByInfo : partitionedByList) {
      builder.add(ColumnIdent.fromPath(partitionedByInfo.get(0)));
    }
    return builder.build();
  }

  private ColumnPolicy getColumnPolicy() {
    Object dynamic = getNested(defaultMappingMap, "dynamic");
    if (ColumnPolicy.STRICT.value().equals(String.valueOf(dynamic).toLowerCase(Locale.ENGLISH))) {
      return ColumnPolicy.STRICT;
    } else if (Booleans.isExplicitFalse(String.valueOf(dynamic))) {
      return ColumnPolicy.IGNORED;
    } else {
      return ColumnPolicy.DYNAMIC;
    }
  }

  private void createColumnDefinitions() {
    Map<String, Object> propertiesMap = getNested(defaultMappingMap, "properties");
    internalExtractColumnDefinitions(null, propertiesMap);
    extractPartitionedByColumns();
  }

  private ImmutableMap<ColumnIdent, IndexReferenceInfo> createIndexDefinitions() {
    ImmutableMap.Builder<ColumnIdent, IndexReferenceInfo> builder = ImmutableMap.builder();
    for (Map.Entry<ColumnIdent, IndexReferenceInfo.Builder> entry : indicesBuilder.entrySet()) {
      builder.put(entry.getKey(), entry.getValue().build());
    }
    indices = builder.build();
    return indices;
  }

  private void extractPartitionedByColumns() {
    for (Tuple<ColumnIdent, DataType> partitioned :
        PartitionedByMappingExtractor.extractPartitionedByColumns(partitionedByList)) {
      addPartitioned(partitioned.v1(), partitioned.v2());
    }
  }

  private ColumnIdent getCustomRoutingCol() {
    if (defaultMappingMetaData != null) {
      Map<String, Object> metaMap = getNested(defaultMappingMap, "_meta");
      if (metaMap != null) {
        String routingPath = (String) metaMap.get("routing");
        if (routingPath != null && !routingPath.equals(ID)) {
          return ColumnIdent.fromPath(routingPath);
        }
      }
    }
    return null;
  }

  private ColumnIdent getRoutingCol() {
    ColumnIdent col = getCustomRoutingCol();
    if (col != null) {
      return col;
    }
    if (primaryKey.size() == 1) {
      return primaryKey.get(0);
    }
    return ID_IDENT;
  }

  public DocIndexMetaData build() {
    partitionedBy = getPartitionedBy();
    columnPolicy = getColumnPolicy();
    createColumnDefinitions();
    indices = createIndexDefinitions();
    columns = ImmutableList.copyOf(columnsBuilder.build());
    partitionedByColumns = partitionedByColumnsBuilder.build();

    for (Tuple<ColumnIdent, ReferenceInfo> sysColumn : DocSysColumns.forTable(ident)) {
      referencesBuilder.put(sysColumn.v1(), sysColumn.v2());
    }
    references = referencesBuilder.build();
    primaryKey = getPrimaryKey();
    routingCol = getRoutingCol();
    return this;
  }

  public ImmutableMap<ColumnIdent, ReferenceInfo> references() {
    return references;
  }

  public ImmutableList<ReferenceInfo> columns() {
    return columns;
  }

  public ImmutableMap<ColumnIdent, IndexReferenceInfo> indices() {
    return indices;
  }

  public ImmutableList<ReferenceInfo> partitionedByColumns() {
    return partitionedByColumns;
  }

  public ImmutableList<ColumnIdent> primaryKey() {
    return primaryKey;
  }

  public ColumnIdent routingCol() {
    return routingCol;
  }

  /**
   * Returns true if the schema of this and <code>other</code> is the same, this includes the table
   * name, as this is reflected in the ReferenceIdents of the columns.
   */
  public boolean schemaEquals(DocIndexMetaData other) {
    if (this == other) return true;
    if (other == null) return false;

    // TODO: when analyzers are exposed in the info, equality has to be checked on them
    // see: TransportSQLActionTest.testSelectTableAliasSchemaExceptionColumnDefinition
    if (columns != null ? !columns.equals(other.columns) : other.columns != null) return false;
    if (primaryKey != null ? !primaryKey.equals(other.primaryKey) : other.primaryKey != null)
      return false;
    if (indices != null ? !indices.equals(other.indices) : other.indices != null) return false;
    if (references != null ? !references.equals(other.references) : other.references != null)
      return false;
    if (routingCol != null ? !routingCol.equals(other.routingCol) : other.routingCol != null)
      return false;

    return true;
  }

  protected DocIndexMetaData merge(
      DocIndexMetaData other,
      TransportPutIndexTemplateAction transportPutIndexTemplateAction,
      boolean thisIsCreatedFromTemplate)
      throws IOException {
    if (schemaEquals(other)) {
      return this;
    } else if (thisIsCreatedFromTemplate) {
      if (this.references.size() < other.references.size()) {
        // this is older, update template and return other
        // settings in template are always authoritative for table information about
        // number_of_shards and number_of_replicas
        updateTemplate(other, transportPutIndexTemplateAction, this.metaData.settings());
        // merge the new mapping with the template settings
        return new DocIndexMetaData(
                IndexMetaData.builder(other.metaData).settings(this.metaData.settings()).build(),
                other.ident)
            .build();
      } else if (references().size() == other.references().size()
          && !references().keySet().equals(other.references().keySet())) {
        XContentHelper.update(defaultMappingMap, other.defaultMappingMap, false);
        // update the template with new information
        updateTemplate(this, transportPutIndexTemplateAction, this.metaData.settings());
        return this;
      }
      // other is older, just return this
      return this;
    } else {
      throw new TableAliasSchemaException(other.ident.name());
    }
  }

  private void updateTemplate(
      DocIndexMetaData md,
      TransportPutIndexTemplateAction transportPutIndexTemplateAction,
      Settings updateSettings) {
    String templateName = PartitionName.templateName(ident.schema(), ident.name());
    PutIndexTemplateRequest request =
        new PutIndexTemplateRequest(templateName)
            .mapping(Constants.DEFAULT_MAPPING_TYPE, md.defaultMappingMap)
            .create(false)
            .settings(updateSettings)
            .template(templateName + "*");
    for (String alias : md.aliases()) {
      request = request.alias(new Alias(alias));
    }
    transportPutIndexTemplateAction.execute(request);
  }

  /** @return the name of the underlying index even if this table is referenced by alias */
  public String concreteIndexName() {
    return metaData.index();
  }

  public boolean isAlias() {
    return isAlias;
  }

  public Set<String> aliases() {
    return aliases;
  }

  public boolean hasAutoGeneratedPrimaryKey() {
    return hasAutoGeneratedPrimaryKey;
  }

  public int numberOfShards() {
    return numberOfShards;
  }

  public BytesRef numberOfReplicas() {
    return numberOfReplicas;
  }

  public ImmutableList<ColumnIdent> partitionedBy() {
    return partitionedBy;
  }

  public ColumnPolicy columnPolicy() {
    return columnPolicy;
  }

  public ImmutableMap<String, Object> tableParameters() {
    return tableParameters;
  }

  private ImmutableMap<ColumnIdent, String> getAnalyzers(
      ColumnIdent columnIdent, Map<String, Object> propertiesMap) {
    ImmutableMap.Builder<ColumnIdent, String> builder = ImmutableMap.builder();
    for (Map.Entry<String, Object> columnEntry : propertiesMap.entrySet()) {
      Map<String, Object> columnProperties = (Map) columnEntry.getValue();
      DataType columnDataType = getColumnDataType(columnProperties);
      ColumnIdent newIdent = childIdent(columnIdent, columnEntry.getKey());
      columnProperties = furtherColumnProperties(columnProperties);
      if (columnDataType == DataTypes.OBJECT
          || (columnDataType.id() == ArrayType.ID
              && ((ArrayType) columnDataType).innerType() == DataTypes.OBJECT)) {
        if (columnProperties.get("properties") != null) {
          builder.putAll(
              getAnalyzers(newIdent, (Map<String, Object>) columnProperties.get("properties")));
        }
      }
      String analyzer = (String) columnProperties.get("analyzer");
      if (analyzer != null) {
        builder.put(newIdent, analyzer);
      }
    }
    return builder.build();
  }

  public ImmutableMap<ColumnIdent, String> analyzers() {
    Map<String, Object> propertiesMap = getNested(defaultMappingMap, "properties");
    if (propertiesMap == null) {
      return ImmutableMap.of();
    } else {
      return getAnalyzers(null, propertiesMap);
    }
  }
}