private List<String> getCurrentServerIds(boolean nag, boolean lagged) { try (Jedis jedis = pool.getResource()) { long time = getRedisTime(jedis.time()); int nagTime = 0; if (nag) { nagTime = nagAboutServers.decrementAndGet(); if (nagTime <= 0) { nagAboutServers.set(10); } } ImmutableList.Builder<String> servers = ImmutableList.builder(); Map<String, String> heartbeats = jedis.hgetAll("heartbeats"); for (Map.Entry<String, String> entry : heartbeats.entrySet()) { try { long stamp = Long.parseLong(entry.getValue()); if (lagged ? time >= stamp + 30 : time <= stamp + 30) servers.add(entry.getKey()); else if (nag && nagTime <= 0) { getLogger() .severe( entry.getKey() + " is " + (time - stamp) + " seconds behind! (Time not synchronized or server down?)"); } } catch (NumberFormatException ignored) { } } return servers.build(); } catch (JedisConnectionException e) { getLogger().log(Level.SEVERE, "Unable to fetch server IDs", e); return Collections.singletonList(configuration.getServerId()); } }
private ImmutableList<ColumnIdent> getPartitionedBy() { ImmutableList.Builder<ColumnIdent> builder = ImmutableList.builder(); for (List<String> partitionedByInfo : partitionedByList) { builder.add(ColumnIdent.fromPath(partitionedByInfo.get(0))); } return builder.build(); }
private ImmutableList<ColumnIdent> getPrimaryKey() { Map<String, Object> metaMap = getNested(defaultMappingMap, "_meta"); if (metaMap != null) { ImmutableList.Builder<ColumnIdent> builder = ImmutableList.builder(); Object pKeys = metaMap.get("primary_keys"); if (pKeys != null) { if (pKeys instanceof String) { builder.add(ColumnIdent.fromPath((String) pKeys)); return builder.build(); } else if (pKeys instanceof Collection) { Collection keys = (Collection) pKeys; if (!keys.isEmpty()) { for (Object pkey : keys) { builder.add(ColumnIdent.fromPath(pkey.toString())); } return builder.build(); } } } } if (getCustomRoutingCol() == null && partitionedByList.isEmpty()) { hasAutoGeneratedPrimaryKey = true; return ImmutableList.of(ID_IDENT); } return ImmutableList.of(); }
/* * Get the exponentially-decayed approximate counts of values in multiple buckets. The elements in * the provided list denote the upper bound each of the buckets and must be sorted in ascending * order. * * The approximate count in each bucket is guaranteed to be within 2 * totalCount * maxError of * the real count. */ public List<Bucket> getHistogram(List<Long> bucketUpperBounds) { checkArgument( Ordering.natural().isOrdered(bucketUpperBounds), "buckets must be sorted in increasing order"); final ImmutableList.Builder<Bucket> builder = ImmutableList.builder(); final PeekingIterator<Long> iterator = Iterators.peekingIterator(bucketUpperBounds.iterator()); final AtomicDouble sum = new AtomicDouble(); final AtomicDouble lastSum = new AtomicDouble(); // for computing weighed average of values in bucket final AtomicDouble bucketWeightedSum = new AtomicDouble(); final double normalizationFactor = weight(TimeUnit.NANOSECONDS.toSeconds(ticker.read())); postOrderTraversal( root, new Callback() { @Override public boolean process(Node node) { while (iterator.hasNext() && iterator.peek() <= node.getUpperBound()) { double bucketCount = sum.get() - lastSum.get(); Bucket bucket = new Bucket( bucketCount / normalizationFactor, bucketWeightedSum.get() / bucketCount); builder.add(bucket); lastSum.set(sum.get()); bucketWeightedSum.set(0); iterator.next(); } bucketWeightedSum.addAndGet(node.getMiddle() * node.weightedCount); sum.addAndGet(node.weightedCount); return iterator.hasNext(); } }); while (iterator.hasNext()) { double bucketCount = sum.get() - lastSum.get(); Bucket bucket = new Bucket(bucketCount / normalizationFactor, bucketWeightedSum.get() / bucketCount); builder.add(bucket); iterator.next(); } return builder.build(); }
private ColumnIdent childIdent(ColumnIdent ident, String name) { if (ident == null) { return new ColumnIdent(name); } if (ident.isColumn()) { return new ColumnIdent(ident.name(), name); } else { ImmutableList.Builder<String> builder = ImmutableList.builder(); for (String s : ident.path()) { builder.add(s); } builder.add(name); return new ColumnIdent(ident.name(), builder.build()); } }
/** * Gets the values at the specified quantiles +/- maxError. The list of quantiles must be sorted * in increasing order, and each value must be in the range [0, 1] */ public List<Long> getQuantiles(List<Double> quantiles) { checkArgument( Ordering.natural().isOrdered(quantiles), "quantiles must be sorted in increasing order"); for (double quantile : quantiles) { checkArgument(quantile >= 0 && quantile <= 1, "quantile must be between [0,1]"); } final ImmutableList.Builder<Long> builder = ImmutableList.builder(); final PeekingIterator<Double> iterator = Iterators.peekingIterator(quantiles.iterator()); postOrderTraversal( root, new Callback() { private double sum = 0; @Override public boolean process(Node node) { sum += node.weightedCount; while (iterator.hasNext() && sum > iterator.peek() * weightedCount) { iterator.next(); // we know the max value ever seen, so cap the percentile to provide better error // bounds in this case long value = Math.min(node.getUpperBound(), max); builder.add(value); } return iterator.hasNext(); } }); // we finished the traversal without consuming all quantiles. This means the remaining quantiles // correspond to the max known value while (iterator.hasNext()) { builder.add(max); iterator.next(); } return builder.build(); }
public class DocIndexMetaData { private static final String ID = "_id"; public static final ColumnIdent ID_IDENT = new ColumnIdent(ID); private final IndexMetaData metaData; private final MappingMetaData defaultMappingMetaData; private final Map<String, Object> defaultMappingMap; private final Map<ColumnIdent, IndexReferenceInfo.Builder> indicesBuilder = new HashMap<>(); private final ImmutableSortedSet.Builder<ReferenceInfo> columnsBuilder = ImmutableSortedSet.orderedBy( new Comparator<ReferenceInfo>() { @Override public int compare(ReferenceInfo o1, ReferenceInfo o2) { return o1.ident().columnIdent().fqn().compareTo(o2.ident().columnIdent().fqn()); } }); // columns should be ordered private final ImmutableMap.Builder<ColumnIdent, ReferenceInfo> referencesBuilder = ImmutableSortedMap.naturalOrder(); private final ImmutableList.Builder<ReferenceInfo> partitionedByColumnsBuilder = ImmutableList.builder(); private final TableIdent ident; private final int numberOfShards; private final BytesRef numberOfReplicas; private final ImmutableMap<String, Object> tableParameters; private Map<String, Object> metaMap; private Map<String, Object> metaColumnsMap; private Map<String, Object> indicesMap; private List<List<String>> partitionedByList; private ImmutableList<ReferenceInfo> columns; private ImmutableMap<ColumnIdent, IndexReferenceInfo> indices; private ImmutableList<ReferenceInfo> partitionedByColumns; private ImmutableMap<ColumnIdent, ReferenceInfo> references; private ImmutableList<ColumnIdent> primaryKey; private ColumnIdent routingCol; private ImmutableList<ColumnIdent> partitionedBy; private final boolean isAlias; private final Set<String> aliases; private boolean hasAutoGeneratedPrimaryKey = false; private ColumnPolicy columnPolicy = ColumnPolicy.DYNAMIC; private static final ImmutableMap<String, DataType> dataTypeMap = ImmutableMap.<String, DataType>builder() .put("date", DataTypes.TIMESTAMP) .put("string", DataTypes.STRING) .put("boolean", DataTypes.BOOLEAN) .put("byte", DataTypes.BYTE) .put("short", DataTypes.SHORT) .put("integer", DataTypes.INTEGER) .put("long", DataTypes.LONG) .put("float", DataTypes.FLOAT) .put("double", DataTypes.DOUBLE) .put("ip", DataTypes.IP) .put("geo_point", DataTypes.GEO_POINT) .put("object", DataTypes.OBJECT) .put("nested", DataTypes.OBJECT) .build(); public DocIndexMetaData(IndexMetaData metaData, TableIdent ident) throws IOException { this.ident = ident; this.metaData = metaData; this.isAlias = !metaData.getIndex().equals(ident.esName()); this.numberOfShards = metaData.numberOfShards(); final Settings settings = metaData.getSettings(); this.numberOfReplicas = NumberOfReplicas.fromSettings(settings); this.aliases = ImmutableSet.copyOf(metaData.aliases().keys().toArray(String.class)); this.defaultMappingMetaData = this.metaData.mappingOrDefault(Constants.DEFAULT_MAPPING_TYPE); if (defaultMappingMetaData == null) { this.defaultMappingMap = new HashMap<>(); } else { this.defaultMappingMap = this.defaultMappingMetaData.sourceAsMap(); } this.tableParameters = TableParameterInfo.tableParametersFromIndexMetaData(metaData); prepareCrateMeta(); } @SuppressWarnings("unchecked") private static <T> T getNested(Map map, String key) { return (T) map.get(key); } private void prepareCrateMeta() { metaMap = getNested(defaultMappingMap, "_meta"); if (metaMap != null) { indicesMap = getNested(metaMap, "indices"); if (indicesMap == null) { indicesMap = ImmutableMap.of(); } metaColumnsMap = getNested(metaMap, "columns"); if (metaColumnsMap == null) { metaColumnsMap = ImmutableMap.of(); } partitionedByList = getNested(metaMap, "partitioned_by"); if (partitionedByList == null) { partitionedByList = ImmutableList.of(); } } else { metaMap = new HashMap<>(); indicesMap = new HashMap<>(); metaColumnsMap = new HashMap<>(); partitionedByList = ImmutableList.of(); } } private void addPartitioned(ColumnIdent column, DataType type) { add(column, type, ColumnPolicy.DYNAMIC, ReferenceInfo.IndexType.NOT_ANALYZED, true); } private void add(ColumnIdent column, DataType type, ReferenceInfo.IndexType indexType) { add(column, type, ColumnPolicy.DYNAMIC, indexType, false); } private void add( ColumnIdent column, DataType type, ColumnPolicy columnPolicy, ReferenceInfo.IndexType indexType, boolean partitioned) { ReferenceInfo info = newInfo(column, type, columnPolicy, indexType); // don't add it if there is a partitioned equivalent of this column if (partitioned || !(partitionedBy != null && partitionedBy.contains(column))) { if (info.ident().isColumn()) { columnsBuilder.add(info); } referencesBuilder.put(info.ident().columnIdent(), info); } if (partitioned) { partitionedByColumnsBuilder.add(info); } } private ReferenceInfo newInfo( ColumnIdent column, DataType type, ColumnPolicy columnPolicy, ReferenceInfo.IndexType indexType) { RowGranularity granularity = RowGranularity.DOC; if (partitionedBy.contains(column)) { granularity = RowGranularity.PARTITION; } return new ReferenceInfo( new ReferenceIdent(ident, column), granularity, type, columnPolicy, indexType); } /** * extract dataType from given columnProperties * * @param columnProperties map of String to Object containing column properties * @return dataType of the column with columnProperties */ public static DataType getColumnDataType(Map<String, Object> columnProperties) { DataType type; String typeName = (String) columnProperties.get("type"); if (typeName == null) { if (columnProperties.containsKey("properties")) { type = DataTypes.OBJECT; } else { return DataTypes.NOT_SUPPORTED; } } else if (typeName.equalsIgnoreCase("array")) { Map<String, Object> innerProperties = getNested(columnProperties, "inner"); DataType innerType = getColumnDataType(innerProperties); type = new ArrayType(innerType); } else { typeName = typeName.toLowerCase(Locale.ENGLISH); type = MoreObjects.firstNonNull(dataTypeMap.get(typeName), DataTypes.NOT_SUPPORTED); } return type; } private ReferenceInfo.IndexType getColumnIndexType(Map<String, Object> columnProperties) { String indexType = (String) columnProperties.get("index"); String analyzerName = (String) columnProperties.get("analyzer"); if (indexType != null) { if (indexType.equals(ReferenceInfo.IndexType.NOT_ANALYZED.toString())) { return ReferenceInfo.IndexType.NOT_ANALYZED; } else if (indexType.equals(ReferenceInfo.IndexType.NO.toString())) { return ReferenceInfo.IndexType.NO; } else if (indexType.equals(ReferenceInfo.IndexType.ANALYZED.toString()) && analyzerName != null && !analyzerName.equals("keyword")) { return ReferenceInfo.IndexType.ANALYZED; } } // default indexType is analyzed so need to check analyzerName if indexType is null else if (analyzerName != null && !analyzerName.equals("keyword")) { return ReferenceInfo.IndexType.ANALYZED; } return ReferenceInfo.IndexType.NOT_ANALYZED; } private ColumnIdent childIdent(ColumnIdent ident, String name) { if (ident == null) { return new ColumnIdent(name); } if (ident.isColumn()) { return new ColumnIdent(ident.name(), name); } else { ImmutableList.Builder<String> builder = ImmutableList.builder(); for (String s : ident.path()) { builder.add(s); } builder.add(name); return new ColumnIdent(ident.name(), builder.build()); } } /** extracts index definitions as well */ @SuppressWarnings("unchecked") private void internalExtractColumnDefinitions( ColumnIdent columnIdent, Map<String, Object> propertiesMap) { if (propertiesMap == null) { return; } for (Map.Entry<String, Object> columnEntry : propertiesMap.entrySet()) { Map<String, Object> columnProperties = (Map) columnEntry.getValue(); DataType columnDataType = getColumnDataType(columnProperties); ColumnIdent newIdent = childIdent(columnIdent, columnEntry.getKey()); columnProperties = furtherColumnProperties(columnProperties); ReferenceInfo.IndexType columnIndexType = getColumnIndexType(columnProperties); if (columnDataType == DataTypes.OBJECT || (columnDataType.id() == ArrayType.ID && ((ArrayType) columnDataType).innerType() == DataTypes.OBJECT)) { ColumnPolicy columnPolicy = ColumnPolicy.of(columnProperties.get("dynamic")); add(newIdent, columnDataType, columnPolicy, ReferenceInfo.IndexType.NO, false); if (columnProperties.get("properties") != null) { // walk nested internalExtractColumnDefinitions( newIdent, (Map<String, Object>) columnProperties.get("properties")); } } else if (columnDataType != DataTypes.NOT_SUPPORTED) { List<String> copyToColumns = getNested(columnProperties, "copy_to"); // extract columns this column is copied to, needed for indices if (copyToColumns != null) { for (String copyToColumn : copyToColumns) { ColumnIdent targetIdent = ColumnIdent.fromPath(copyToColumn); IndexReferenceInfo.Builder builder = getOrCreateIndexBuilder(targetIdent); builder.addColumn( newInfo(newIdent, columnDataType, ColumnPolicy.DYNAMIC, columnIndexType)); } } // is it an index? if (indicesMap.containsKey(newIdent.fqn())) { IndexReferenceInfo.Builder builder = getOrCreateIndexBuilder(newIdent); builder .indexType(columnIndexType) .ident(new ReferenceIdent(ident, newIdent)) .analyzer((String) columnProperties.get("analyzer")); } else { add(newIdent, columnDataType, columnIndexType); } } } } /** * get the real column properties from a possible array mapping, keeping most of this stuff inside * "inner" */ private Map<String, Object> furtherColumnProperties(Map<String, Object> columnProperties) { if (columnProperties.get("inner") != null) { return (Map<String, Object>) columnProperties.get("inner"); } else { return columnProperties; } } private IndexReferenceInfo.Builder getOrCreateIndexBuilder(ColumnIdent ident) { IndexReferenceInfo.Builder builder = indicesBuilder.get(ident); if (builder == null) { builder = new IndexReferenceInfo.Builder(); indicesBuilder.put(ident, builder); } return builder; } private ImmutableList<ColumnIdent> getPrimaryKey() { Map<String, Object> metaMap = getNested(defaultMappingMap, "_meta"); if (metaMap != null) { ImmutableList.Builder<ColumnIdent> builder = ImmutableList.builder(); Object pKeys = metaMap.get("primary_keys"); if (pKeys != null) { if (pKeys instanceof String) { builder.add(ColumnIdent.fromPath((String) pKeys)); return builder.build(); } else if (pKeys instanceof Collection) { Collection keys = (Collection) pKeys; if (!keys.isEmpty()) { for (Object pkey : keys) { builder.add(ColumnIdent.fromPath(pkey.toString())); } return builder.build(); } } } } if (getCustomRoutingCol() == null && partitionedByList.isEmpty()) { hasAutoGeneratedPrimaryKey = true; return ImmutableList.of(ID_IDENT); } return ImmutableList.of(); } private ImmutableList<ColumnIdent> getPartitionedBy() { ImmutableList.Builder<ColumnIdent> builder = ImmutableList.builder(); for (List<String> partitionedByInfo : partitionedByList) { builder.add(ColumnIdent.fromPath(partitionedByInfo.get(0))); } return builder.build(); } private ColumnPolicy getColumnPolicy() { Object dynamic = getNested(defaultMappingMap, "dynamic"); if (ColumnPolicy.STRICT.value().equals(String.valueOf(dynamic).toLowerCase(Locale.ENGLISH))) { return ColumnPolicy.STRICT; } else if (Booleans.isExplicitFalse(String.valueOf(dynamic))) { return ColumnPolicy.IGNORED; } else { return ColumnPolicy.DYNAMIC; } } private void createColumnDefinitions() { Map<String, Object> propertiesMap = getNested(defaultMappingMap, "properties"); internalExtractColumnDefinitions(null, propertiesMap); extractPartitionedByColumns(); } private ImmutableMap<ColumnIdent, IndexReferenceInfo> createIndexDefinitions() { ImmutableMap.Builder<ColumnIdent, IndexReferenceInfo> builder = ImmutableMap.builder(); for (Map.Entry<ColumnIdent, IndexReferenceInfo.Builder> entry : indicesBuilder.entrySet()) { builder.put(entry.getKey(), entry.getValue().build()); } indices = builder.build(); return indices; } private void extractPartitionedByColumns() { for (Tuple<ColumnIdent, DataType> partitioned : PartitionedByMappingExtractor.extractPartitionedByColumns(partitionedByList)) { addPartitioned(partitioned.v1(), partitioned.v2()); } } private ColumnIdent getCustomRoutingCol() { if (defaultMappingMetaData != null) { Map<String, Object> metaMap = getNested(defaultMappingMap, "_meta"); if (metaMap != null) { String routingPath = (String) metaMap.get("routing"); if (routingPath != null && !routingPath.equals(ID)) { return ColumnIdent.fromPath(routingPath); } } } return null; } private ColumnIdent getRoutingCol() { ColumnIdent col = getCustomRoutingCol(); if (col != null) { return col; } if (primaryKey.size() == 1) { return primaryKey.get(0); } return ID_IDENT; } public DocIndexMetaData build() { partitionedBy = getPartitionedBy(); columnPolicy = getColumnPolicy(); createColumnDefinitions(); indices = createIndexDefinitions(); columns = ImmutableList.copyOf(columnsBuilder.build()); partitionedByColumns = partitionedByColumnsBuilder.build(); for (Tuple<ColumnIdent, ReferenceInfo> sysColumn : DocSysColumns.forTable(ident)) { referencesBuilder.put(sysColumn.v1(), sysColumn.v2()); } references = referencesBuilder.build(); primaryKey = getPrimaryKey(); routingCol = getRoutingCol(); return this; } public ImmutableMap<ColumnIdent, ReferenceInfo> references() { return references; } public ImmutableList<ReferenceInfo> columns() { return columns; } public ImmutableMap<ColumnIdent, IndexReferenceInfo> indices() { return indices; } public ImmutableList<ReferenceInfo> partitionedByColumns() { return partitionedByColumns; } public ImmutableList<ColumnIdent> primaryKey() { return primaryKey; } public ColumnIdent routingCol() { return routingCol; } /** * Returns true if the schema of this and <code>other</code> is the same, this includes the table * name, as this is reflected in the ReferenceIdents of the columns. */ public boolean schemaEquals(DocIndexMetaData other) { if (this == other) return true; if (other == null) return false; // TODO: when analyzers are exposed in the info, equality has to be checked on them // see: TransportSQLActionTest.testSelectTableAliasSchemaExceptionColumnDefinition if (columns != null ? !columns.equals(other.columns) : other.columns != null) return false; if (primaryKey != null ? !primaryKey.equals(other.primaryKey) : other.primaryKey != null) return false; if (indices != null ? !indices.equals(other.indices) : other.indices != null) return false; if (references != null ? !references.equals(other.references) : other.references != null) return false; if (routingCol != null ? !routingCol.equals(other.routingCol) : other.routingCol != null) return false; return true; } protected DocIndexMetaData merge( DocIndexMetaData other, TransportPutIndexTemplateAction transportPutIndexTemplateAction, boolean thisIsCreatedFromTemplate) throws IOException { if (schemaEquals(other)) { return this; } else if (thisIsCreatedFromTemplate) { if (this.references.size() < other.references.size()) { // this is older, update template and return other // settings in template are always authoritative for table information about // number_of_shards and number_of_replicas updateTemplate(other, transportPutIndexTemplateAction, this.metaData.settings()); // merge the new mapping with the template settings return new DocIndexMetaData( IndexMetaData.builder(other.metaData).settings(this.metaData.settings()).build(), other.ident) .build(); } else if (references().size() == other.references().size() && !references().keySet().equals(other.references().keySet())) { XContentHelper.update(defaultMappingMap, other.defaultMappingMap, false); // update the template with new information updateTemplate(this, transportPutIndexTemplateAction, this.metaData.settings()); return this; } // other is older, just return this return this; } else { throw new TableAliasSchemaException(other.ident.name()); } } private void updateTemplate( DocIndexMetaData md, TransportPutIndexTemplateAction transportPutIndexTemplateAction, Settings updateSettings) { String templateName = PartitionName.templateName(ident.schema(), ident.name()); PutIndexTemplateRequest request = new PutIndexTemplateRequest(templateName) .mapping(Constants.DEFAULT_MAPPING_TYPE, md.defaultMappingMap) .create(false) .settings(updateSettings) .template(templateName + "*"); for (String alias : md.aliases()) { request = request.alias(new Alias(alias)); } transportPutIndexTemplateAction.execute(request); } /** @return the name of the underlying index even if this table is referenced by alias */ public String concreteIndexName() { return metaData.index(); } public boolean isAlias() { return isAlias; } public Set<String> aliases() { return aliases; } public boolean hasAutoGeneratedPrimaryKey() { return hasAutoGeneratedPrimaryKey; } public int numberOfShards() { return numberOfShards; } public BytesRef numberOfReplicas() { return numberOfReplicas; } public ImmutableList<ColumnIdent> partitionedBy() { return partitionedBy; } public ColumnPolicy columnPolicy() { return columnPolicy; } public ImmutableMap<String, Object> tableParameters() { return tableParameters; } private ImmutableMap<ColumnIdent, String> getAnalyzers( ColumnIdent columnIdent, Map<String, Object> propertiesMap) { ImmutableMap.Builder<ColumnIdent, String> builder = ImmutableMap.builder(); for (Map.Entry<String, Object> columnEntry : propertiesMap.entrySet()) { Map<String, Object> columnProperties = (Map) columnEntry.getValue(); DataType columnDataType = getColumnDataType(columnProperties); ColumnIdent newIdent = childIdent(columnIdent, columnEntry.getKey()); columnProperties = furtherColumnProperties(columnProperties); if (columnDataType == DataTypes.OBJECT || (columnDataType.id() == ArrayType.ID && ((ArrayType) columnDataType).innerType() == DataTypes.OBJECT)) { if (columnProperties.get("properties") != null) { builder.putAll( getAnalyzers(newIdent, (Map<String, Object>) columnProperties.get("properties"))); } } String analyzer = (String) columnProperties.get("analyzer"); if (analyzer != null) { builder.put(newIdent, analyzer); } } return builder.build(); } public ImmutableMap<ColumnIdent, String> analyzers() { Map<String, Object> propertiesMap = getNested(defaultMappingMap, "properties"); if (propertiesMap == null) { return ImmutableMap.of(); } else { return getAnalyzers(null, propertiesMap); } } }