private static ColumnIdentifier getKeyId(CFMetaData cfm, int i) {
List<ByteBuffer> definedNames = cfm.getKeyAliases();
// For compatibility sake, non-composite key default alias is 'key', not 'key1'.
return definedNames == null || i >= definedNames.size() || cfm.getKeyAliases().get(i) == null
? new ColumnIdentifier(i == 0 ? DEFAULT_KEY_ALIAS : DEFAULT_KEY_ALIAS + (i + 1), false)
: new ColumnIdentifier(cfm.getKeyAliases().get(i), definitionType);
}
private int discardHead(
ColumnFamily cf,
int toDiscard,
ColumnFamily copy,
Iterator<Column> iter,
DeletionInfo.InOrderTester tester) {
ColumnCounter counter = columnCounter();
List<Column> staticColumns = new ArrayList<>(cfm.staticColumns().size());
// Discard the first 'toDiscard' live, non-static columns
while (iter.hasNext()) {
Column c = iter.next();
// if it's a static column, don't count it and save it to add to the trimmed results
ColumnDefinition columnDef = cfm.getColumnDefinitionFromColumnName(c.name());
if (columnDef != null && columnDef.type == ColumnDefinition.Type.STATIC) {
staticColumns.add(c);
continue;
}
counter.count(c, tester);
// once we've discarded the required amount, add the rest
if (counter.live() > toDiscard) {
for (Column staticColumn : staticColumns) copy.addColumn(staticColumn);
copy.addColumn(c);
while (iter.hasNext()) copy.addColumn(iter.next());
}
}
return Math.min(counter.live(), toDiscard);
}
public boolean announceMigration(boolean isLocalOnly) throws RequestValidationException {
CFMetaData cfm = Schema.instance.getCFMetaData(keyspace(), columnFamily()).copy();
IndexTarget target = rawTarget.prepare(cfm);
logger.debug("Updating column {} definition for index {}", target.column, indexName);
ColumnDefinition cd = cfm.getColumnDefinition(target.column);
if (cd.getIndexType() != null && ifNotExists) return false;
if (properties.isCustom) {
cd.setIndexType(IndexType.CUSTOM, properties.getOptions());
} else if (cfm.comparator.isCompound()) {
Map<String, String> options = Collections.emptyMap();
// For now, we only allow indexing values for collections, but we could later allow
// to also index map keys, so we record that this is the values we index to make our
// lives easier then.
if (cd.type.isCollection() && cd.type.isMultiCell())
options =
ImmutableMap.of(
target.isCollectionKeys
? SecondaryIndex.INDEX_KEYS_OPTION_NAME
: SecondaryIndex.INDEX_VALUES_OPTION_NAME,
"");
cd.setIndexType(IndexType.COMPOSITES, options);
} else {
cd.setIndexType(IndexType.KEYS, Collections.<String, String>emptyMap());
}
cd.setIndexName(indexName);
cfm.addDefaultIndexNames();
MigrationManager.announceColumnFamilyUpdate(cfm, false, isLocalOnly);
return true;
}
private static CFMetaData newCFMetaData(String ksname, String cfname) {
CFMetaData metadata =
CFMetaData.Builder.create(ksname, cfname)
.addPartitionKey("key", UTF8Type.instance)
.addClusteringColumn("col", UTF8Type.instance)
.addRegularColumn("value", UTF8Type.instance)
.withPartitioner(Murmur3Partitioner.instance)
.build();
metadata.caching(CachingParams.CACHE_NOTHING);
return metadata;
}
/**
* Returns the first non-static cell in the ColumnFamily. This is necessary to avoid recording a
* static column as the "last" cell seen in a reversed query. Because we will always query static
* columns alongside the normal data for a page, they are not a good indicator of where paging
* should resume. When we begin the next page, we need to start from the last non-static cell.
*/
protected Column firstNonStaticColumn(ColumnFamily cf) {
for (Column column : cf) {
ColumnDefinition def = cfm.getColumnDefinitionFromColumnName(column.name());
if (def == null || def.type != ColumnDefinition.Type.STATIC) return column;
}
return null;
}
public static CFMetaData initializeCfMetaData(Configuration configuration) {
final String cql = configuration.get(Aegisthus.Feature.CONF_CQL_SCHEMA);
Preconditions.checkNotNull(cql, "Cannot proceed without CQL definition.");
final CreateTableStatement statement = getCreateTableStatement(cql);
try {
final CFMetaData cfMetaData = statement.getCFMetaData();
cfMetaData.rebuild();
return cfMetaData;
} catch (RequestValidationException e) {
// Cannot proceed if an error occurs
throw new RuntimeException("Error initializing CFMetadata from CQL.", e);
}
}
// ONLY to be used for testing, to create a mock Memtable
@VisibleForTesting
public Memtable(CFMetaData metadata) {
this.initialComparator = metadata.comparator;
this.cfs = null;
this.allocator = null;
this.columnsCollector = new ColumnsCollector(metadata.partitionColumns());
}
public RowMutation mutationForKey(
ByteBuffer key,
String keyspace,
Long timestamp,
ThriftClientState clientState,
List<ByteBuffer> variables,
CFMetaData metadata)
throws InvalidRequestException {
RowMutation rm = new RowMutation(keyspace, key);
QueryProcessor.validateKeyAlias(metadata, keyName);
AbstractType<?> comparator = metadata.getComparatorFor(null);
if (columns.size() < 1) {
// No columns, delete the row
rm.delete(
new QueryPath(columnFamily), (timestamp == null) ? getTimestamp(clientState) : timestamp);
} else {
// Delete specific columns
for (Term column : columns) {
ByteBuffer columnName = column.getByteBuffer(comparator, variables);
validateColumnName(columnName);
rm.delete(
new QueryPath(columnFamily, null, columnName),
(timestamp == null) ? getTimestamp(clientState) : timestamp);
}
}
return rm;
}
public static CompositeType buildIndexComparator(
CFMetaData baseMetadata, ColumnDefinition columnDef) {
int ckCount = baseMetadata.clusteringKeyColumns().size();
List<AbstractType<?>> types = new ArrayList<AbstractType<?>>(ckCount + 1);
types.add(SecondaryIndex.keyComparator);
types.addAll(baseMetadata.comparator.getComponents());
return CompositeType.getInstance(types);
}
public void applyPropertiesTo(CFMetaData cfmd) throws RequestValidationException {
cfmd.defaultValidator(defaultValidator)
.keyValidator(keyValidator)
.columnMetadata(getColumns())
.setDense(isDense);
cfmd.addColumnMetadataFromAliases(
keyAliases, keyValidator, ColumnDefinition.Type.PARTITION_KEY);
cfmd.addColumnMetadataFromAliases(
columnAliases, comparator, ColumnDefinition.Type.CLUSTERING_KEY);
if (valueAlias != null)
cfmd.addColumnMetadataFromAliases(
Collections.<ByteBuffer>singletonList(valueAlias),
defaultValidator,
ColumnDefinition.Type.COMPACT_VALUE);
properties.applyToCFMetadata(cfmd);
}
private static Name createValue(CFMetaData cfm) {
ColumnIdentifier alias = getValueId(cfm);
// That's how we distinguish between 'no value alias because coming from thrift' and 'I
// explicitely did not
// define a value' (see CreateColumnFamilyStatement)
return alias.key.equals(ByteBufferUtil.EMPTY_BYTE_BUFFER)
? null
: new Name(cfm.ksName, cfm.cfName, alias, Name.Kind.VALUE_ALIAS, cfm.getDefaultValidator());
}
public ParsedStatement.Prepared prepare(ColumnSpecification[] boundNames)
throws InvalidRequestException {
CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace(), columnFamily());
type = metadata.getDefaultValidator().isCommutative() ? Type.COUNTER : Type.LOGGED;
cfDef = metadata.getCfDef();
UpdateStatement.processKeys(cfDef, whereClause, processedKeys, boundNames);
for (Selector column : columns) {
CFDefinition.Name name = cfDef.get(column.id());
if (name == null)
throw new InvalidRequestException(String.format("Unknown identifier %s", column));
// For compact, we only have one value except the key, so the only form of DELETE that make
// sense is without a column
// list. However, we support having the value name for coherence with the static/sparse case
if (name.kind != CFDefinition.Name.Kind.COLUMN_METADATA
&& name.kind != CFDefinition.Name.Kind.VALUE_ALIAS)
throw new InvalidRequestException(
String.format(
"Invalid identifier %s for deletion (should not be a PRIMARY KEY part)", column));
if (column.key() != null) {
if (name.type instanceof ListType) {
if (column.key().isBindMarker())
boundNames[column.key().bindIndex] = ListOperation.indexSpecOf(name);
} else if (name.type instanceof MapType) {
if (column.key().isBindMarker())
boundNames[column.key().bindIndex] = MapOperation.keySpecOf(name, (MapType) name.type);
} else {
throw new InvalidRequestException(
String.format(
"Invalid selection %s since %s is neither a list or a map", column, column.id()));
}
}
toRemove.add(Pair.create(name, column.key()));
}
return new ParsedStatement.Prepared(this, Arrays.<ColumnSpecification>asList(boundNames));
}
/**
* Create a new buffering writer.
*
* @param directory the directory where to write the sstables
* @param partitioner the partitioner
* @param keyspace the keyspace name
* @param columnFamily the column family name
* @param comparator the column family comparator
* @param subComparator the column family subComparator or null if not a Super column family.
* @param bufferSizeInMB the data size in MB before which a sstable is written and the buffer
* reseted. This correspond roughly to the written data size (i.e. the size of the create
* sstable). The actual size used in memory will be higher (by how much depends on the size of
* the columns you add). For 1GB of heap, a 128 bufferSizeInMB is probably a reasonable
* choice. If you experience OOM, this value should be lowered.
*/
public SSTableSimpleUnsortedWriter(
File directory,
IPartitioner partitioner,
String keyspace,
String columnFamily,
AbstractType<?> comparator,
AbstractType<?> subComparator,
int bufferSizeInMB,
CompressionParameters compressParameters) {
this(
directory,
CFMetaData.denseCFMetaData(keyspace, columnFamily, comparator, subComparator)
.compressionParameters(compressParameters),
partitioner,
bufferSizeInMB);
}
@Override
public void init(String keyspace) {
Iterator<InetAddress> hostiter = hosts.iterator();
while (hostiter.hasNext()) {
try {
// Query endpoint to ranges map and schemas from thrift
InetAddress host = hostiter.next();
Cassandra.Client client =
createThriftClient(
host.getHostAddress(), rpcPort, this.user, this.passwd, this.transportFactory);
setPartitioner(client.describe_partitioner());
Token.TokenFactory tkFactory = getPartitioner().getTokenFactory();
for (TokenRange tr : client.describe_ring(keyspace)) {
Range<Token> range =
new Range<>(
tkFactory.fromString(tr.start_token),
tkFactory.fromString(tr.end_token),
getPartitioner());
for (String ep : tr.endpoints) {
addRangeForEndpoint(range, InetAddress.getByName(ep));
}
}
String query =
String.format(
"SELECT * FROM %s.%s WHERE keyspace_name = '%s'",
Keyspace.SYSTEM_KS, SystemKeyspace.SCHEMA_COLUMNFAMILIES_CF, keyspace);
CqlResult result =
client.execute_cql3_query(
ByteBufferUtil.bytes(query), Compression.NONE, ConsistencyLevel.ONE);
for (CqlRow row : result.rows) {
CFMetaData metadata = CFMetaData.fromThriftCqlRow(row);
knownCfs.put(metadata.cfName, metadata);
}
break;
} catch (Exception e) {
if (!hostiter.hasNext())
throw new RuntimeException("Could not retrieve endpoint ranges: ", e);
}
}
}
@Override
public void applyModels() throws IOException {
KSMetaData ksm = DatabaseDescriptor.getTableDefinition(name);
// remove the table from the static instances.
Table table = Table.clear(ksm.name);
if (table == null) throw new IOException("Table is not active. " + ksm.name);
// remove all cfs from the table instance.
for (CFMetaData cfm : ksm.cfMetaData().values()) {
CFMetaData.purge(cfm);
table.dropCf(cfm.cfId);
SystemTable.markForRemoval(cfm);
}
// reset defs.
DatabaseDescriptor.clearTableDefinition(ksm, newVersion);
CommitLog.instance().forceNewSegment();
Migration.cleanupDeadFiles(blockOnFileDeletion);
// clear up any local hinted data for this keyspace.
HintedHandOffManager.renameHints(name, null);
}
public void applyModels() throws IOException {
ColumnFamilyStore cfs = Table.open(tableName).getColumnFamilyStore(cfName);
// reinitialize the table.
KSMetaData existing = DatabaseDescriptor.getTableDefinition(tableName);
CFMetaData cfm = existing.cfMetaData().get(cfName);
KSMetaData ksm = makeNewKeyspaceDefinition(existing);
CFMetaData.purge(cfm);
DatabaseDescriptor.setTableDefinition(ksm, newVersion);
if (!clientMode) {
cfs.snapshot(Table.getTimestampedSnapshotName(null));
CompactionManager.instance.getCompactionLock().lock();
cfs.flushLock.lock();
try {
Table.open(ksm.name).dropCf(cfm.cfId);
} finally {
cfs.flushLock.unlock();
CompactionManager.instance.getCompactionLock().unlock();
}
}
}
public static Message createMessage(
String keyspace, byte[] key, String columnFamily, List<ColumnFamily> columnFamilies) {
ColumnFamily baseColumnFamily;
DataOutputBuffer bufOut = new DataOutputBuffer();
RowMutation rm;
Message message;
Column column;
/* Get the first column family from list, this is just to get past validation */
baseColumnFamily =
new ColumnFamily(
ColumnFamilyType.Standard,
DatabaseDescriptor.getComparator(keyspace, columnFamily),
DatabaseDescriptor.getSubComparator(keyspace, columnFamily),
CFMetaData.getId(keyspace, columnFamily));
for (ColumnFamily cf : columnFamilies) {
bufOut.reset();
ColumnFamily.serializer().serializeWithIndexes(cf, bufOut);
byte[] data = new byte[bufOut.getLength()];
System.arraycopy(bufOut.getData(), 0, data, 0, bufOut.getLength());
column = new Column(FBUtilities.toByteBuffer(cf.id()), ByteBuffer.wrap(data), 0);
baseColumnFamily.addColumn(column);
}
rm = new RowMutation(keyspace, ByteBuffer.wrap(key));
rm.add(baseColumnFamily);
try {
/* Make message */
message = rm.makeRowMutationMessage(StorageService.Verb.BINARY, MessagingService.version_);
} catch (IOException e) {
throw new RuntimeException(e);
}
return message;
}
public ColumnNameBuilder getKeyNameBuilder() {
return hasCompositeKey
? new CompositeType.Builder((CompositeType) cfm.getKeyValidator())
: new NonCompositeBuilder(cfm.getKeyValidator());
}
/**
* Creates a new <code>TokenRestriction</code> that apply to the specified columns.
*
* @param columnDefs the definition of the columns to which apply the token restriction
*/
public TokenRestriction(CFMetaData metadata, List<ColumnDefinition> columnDefs) {
super(metadata.getKeyValidatorAsClusteringComparator());
this.columnDefs = columnDefs;
this.metadata = metadata;
}
private static ColumnIdentifier getValueId(CFMetaData cfm) {
return cfm.getValueAlias() == null
? new ColumnIdentifier(DEFAULT_VALUE_ALIAS, false)
: new ColumnIdentifier(cfm.getValueAlias(), definitionType);
}
private static ColumnIdentifier getColumnId(CFMetaData cfm, int i) {
List<ByteBuffer> definedNames = cfm.getColumnAliases();
return definedNames == null || i >= definedNames.size() || cfm.getColumnAliases().get(i) == null
? new ColumnIdentifier(DEFAULT_COLUMN_ALIAS + (i + 1), false)
: new ColumnIdentifier(cfm.getColumnAliases().get(i), definitionType);
}
public void announceMigration() throws RequestValidationException {
CFMetaData meta = validateColumnFamily(keyspace(), columnFamily());
CFMetaData cfm = meta.clone();
CFDefinition cfDef = meta.getCfDef();
CFDefinition.Name name = columnName == null ? null : cfDef.get(columnName);
switch (oType) {
case ADD:
if (cfDef.isCompact)
throw new InvalidRequestException("Cannot add new column to a compact CF");
if (name != null) {
switch (name.kind) {
case KEY_ALIAS:
case COLUMN_ALIAS:
throw new InvalidRequestException(
String.format(
"Invalid column name %s because it conflicts with a PRIMARY KEY part",
columnName));
case COLUMN_METADATA:
throw new InvalidRequestException(
String.format(
"Invalid column name %s because it conflicts with an existing column",
columnName));
}
}
AbstractType<?> type = validator.getType();
if (type instanceof CollectionType) {
if (!cfDef.isComposite)
throw new InvalidRequestException(
"Cannot use collection types with non-composite PRIMARY KEY");
if (cfDef.cfm.isSuper())
throw new InvalidRequestException(
"Cannot use collection types with Super column family");
Map<ByteBuffer, CollectionType> collections =
cfDef.hasCollections
? new HashMap<ByteBuffer, CollectionType>(cfDef.getCollectionType().defined)
: new HashMap<ByteBuffer, CollectionType>();
collections.put(columnName.key, (CollectionType) type);
ColumnToCollectionType newColType = ColumnToCollectionType.getInstance(collections);
List<AbstractType<?>> ctypes =
new ArrayList<AbstractType<?>>(((CompositeType) cfm.comparator).types);
if (cfDef.hasCollections) ctypes.set(ctypes.size() - 1, newColType);
else ctypes.add(newColType);
cfm.comparator = CompositeType.getInstance(ctypes);
}
Integer componentIndex =
cfDef.isComposite
? ((CompositeType) meta.comparator).types.size() - (cfDef.hasCollections ? 2 : 1)
: null;
cfm.addColumnDefinition(ColumnDefinition.regularDef(columnName.key, type, componentIndex));
break;
case ALTER:
if (name == null)
throw new InvalidRequestException(
String.format("Column %s was not found in table %s", columnName, columnFamily()));
switch (name.kind) {
case KEY_ALIAS:
AbstractType<?> newType = validator.getType();
if (newType instanceof CounterColumnType)
throw new InvalidRequestException(
String.format(
"counter type is not supported for PRIMARY KEY part %s", columnName));
if (cfDef.hasCompositeKey) {
List<AbstractType<?>> newTypes =
new ArrayList<AbstractType<?>>(((CompositeType) cfm.getKeyValidator()).types);
newTypes.set(name.position, newType);
cfm.keyValidator(CompositeType.getInstance(newTypes));
} else {
cfm.keyValidator(newType);
}
break;
case COLUMN_ALIAS:
assert cfDef.isComposite;
List<AbstractType<?>> newTypes =
new ArrayList<AbstractType<?>>(((CompositeType) cfm.comparator).types);
newTypes.set(name.position, validator.getType());
cfm.comparator = CompositeType.getInstance(newTypes);
break;
case VALUE_ALIAS:
cfm.defaultValidator(validator.getType());
break;
case COLUMN_METADATA:
ColumnDefinition column = cfm.getColumnDefinition(columnName.key);
column.setValidator(validator.getType());
break;
}
break;
case DROP:
if (cfDef.isCompact)
throw new InvalidRequestException("Cannot drop columns from a compact CF");
if (name == null)
throw new InvalidRequestException(
String.format("Column %s was not found in table %s", columnName, columnFamily()));
switch (name.kind) {
case KEY_ALIAS:
case COLUMN_ALIAS:
throw new InvalidRequestException(
String.format("Cannot drop PRIMARY KEY part %s", columnName));
case COLUMN_METADATA:
ColumnDefinition toDelete = null;
for (ColumnDefinition columnDef : cfm.regularColumns()) {
if (columnDef.name.equals(columnName.key)) toDelete = columnDef;
}
assert toDelete != null;
cfm.removeColumnDefinition(toDelete);
break;
}
break;
case OPTS:
if (cfProps == null)
throw new InvalidRequestException(
String.format("ALTER COLUMNFAMILY WITH invoked, but no parameters found"));
cfProps.validate();
cfProps.applyToCFMetadata(cfm);
break;
case RENAME:
for (Map.Entry<ColumnIdentifier, ColumnIdentifier> entry : renames.entrySet()) {
ColumnIdentifier from = entry.getKey();
ColumnIdentifier to = entry.getValue();
cfm.renameColumn(from.key, from.toString(), to.key, to.toString());
}
break;
}
MigrationManager.announceColumnFamilyUpdate(cfm, false);
}
public void validate() throws InvalidRequestException, ConfigurationException {
compactionStrategyClass = CFMetaData.DEFAULT_COMPACTION_STRATEGY_CLASS;
// we need to remove parent:key = value pairs from the main properties
Set<String> propsToRemove = new HashSet<String>();
// check if we have compaction/compression options
for (String property : properties.keySet()) {
if (!property.contains(":")) continue;
String key = property.split(":")[1];
String val = properties.get(property);
if (property.startsWith(COMPACTION_OPTIONS_PREFIX)) {
compactionStrategyOptions.put(key, val);
propsToRemove.add(property);
}
if (property.startsWith(COMPRESSION_PARAMETERS_PREFIX)) {
compressionParameters.put(key, val);
propsToRemove.add(property);
}
}
for (String property : propsToRemove) properties.remove(property);
// Catch the case where someone passed a kwarg that is not recognized.
for (String bogus : Sets.difference(properties.keySet(), allowedKeywords))
throw new InvalidRequestException(
bogus + " is not a valid keyword argument for CREATE COLUMNFAMILY");
for (String obsolete : Sets.intersection(properties.keySet(), obsoleteKeywords))
logger.warn("Ignoring obsolete property {}", obsolete);
// Validate min/max compaction thresholds
Integer minCompaction = getPropertyInt(KW_MINCOMPACTIONTHRESHOLD, null);
Integer maxCompaction = getPropertyInt(KW_MAXCOMPACTIONTHRESHOLD, null);
if ((minCompaction != null) && (maxCompaction != null)) // Both min and max are set
{
if ((minCompaction > maxCompaction) && (maxCompaction != 0))
throw new InvalidRequestException(
String.format(
"%s cannot be larger than %s",
KW_MINCOMPACTIONTHRESHOLD, KW_MAXCOMPACTIONTHRESHOLD));
} else if (minCompaction != null) // Only the min threshold is set
{
if (minCompaction > CFMetaData.DEFAULT_MAX_COMPACTION_THRESHOLD)
throw new InvalidRequestException(
String.format(
"%s cannot be larger than %s, (default %s)",
KW_MINCOMPACTIONTHRESHOLD,
KW_MAXCOMPACTIONTHRESHOLD,
CFMetaData.DEFAULT_MAX_COMPACTION_THRESHOLD));
} else if (maxCompaction != null) // Only the max threshold is set
{
if ((maxCompaction < CFMetaData.DEFAULT_MIN_COMPACTION_THRESHOLD) && (maxCompaction != 0))
throw new InvalidRequestException(
String.format(
"%s cannot be smaller than %s, (default %s)",
KW_MAXCOMPACTIONTHRESHOLD,
KW_MINCOMPACTIONTHRESHOLD,
CFMetaData.DEFAULT_MIN_COMPACTION_THRESHOLD));
}
CFMetaData.validateCompactionOptions(compactionStrategyClass, compactionStrategyOptions);
}
public static ByteBuffer buildName(CFMetaData meta, ByteBuffer... args) {
ColumnNameBuilder builder = meta.getCfDef().getColumnNameBuilder();
for (ByteBuffer arg : args) builder.add(arg);
return builder.build();
}
private static AbstractType[] getKeyTypes(CFMetaData cfm) {
AbstractType[] types = new AbstractType[cfm.getCfDef().keys.size()];
int i = 0;
for (CFDefinition.Name name : cfm.getCfDef().keys.values()) types[i++] = name.type;
return types;
}
public TokenFct(CFMetaData cfm) {
super("token", partitioner.getTokenValidator(), getKeyTypes(cfm));
this.cfDef = cfm.getCfDef();
}
public CFDefinition(CFMetaData cfm) {
this.cfm = cfm;
if (cfm.getKeyValidator() instanceof CompositeType) {
this.hasCompositeKey = true;
CompositeType keyComposite = (CompositeType) cfm.getKeyValidator();
assert keyComposite.types.size() > 1;
for (int i = 0; i < keyComposite.types.size(); i++) {
ColumnIdentifier id = getKeyId(cfm, i);
this.keys.put(
id,
new Name(
cfm.ksName, cfm.cfName, id, Name.Kind.KEY_ALIAS, i, keyComposite.types.get(i)));
}
} else {
this.hasCompositeKey = false;
ColumnIdentifier id = getKeyId(cfm, 0);
this.keys.put(
id, new Name(cfm.ksName, cfm.cfName, id, Name.Kind.KEY_ALIAS, 0, cfm.getKeyValidator()));
}
if (cfm.comparator instanceof CompositeType) {
this.isComposite = true;
CompositeType composite = (CompositeType) cfm.comparator;
/*
* We are a "sparse" composite, i.e. a non-compact one, if either:
* - the last type of the composite is a ColumnToCollectionType
* - or we have one less alias than of composite types and the last type is UTF8Type.
*
* Note that this is not perfect: if someone upgrading from thrift "renames" all but
* the last column alias, the cf will be considered "sparse" and he will be stuck with
* that even though that might not be what he wants. But the simple workaround is
* for that user to rename all the aliases at the same time in the first place.
*/
int last = composite.types.size() - 1;
AbstractType<?> lastType = composite.types.get(last);
if (lastType instanceof ColumnToCollectionType
|| (cfm.getColumnAliases().size() == last && lastType instanceof UTF8Type)) {
// "sparse" composite
this.isCompact = false;
this.value = null;
assert cfm.getValueAlias() == null;
// check for collection type
if (lastType instanceof ColumnToCollectionType) {
--last;
this.hasCollections = true;
} else {
this.hasCollections = false;
}
for (int i = 0; i < last; i++) {
ColumnIdentifier id = getColumnId(cfm, i);
this.columns.put(
id,
new Name(
cfm.ksName, cfm.cfName, id, Name.Kind.COLUMN_ALIAS, i, composite.types.get(i)));
}
for (Map.Entry<ByteBuffer, ColumnDefinition> def : cfm.getColumn_metadata().entrySet()) {
ColumnIdentifier id =
new ColumnIdentifier(def.getKey(), cfm.getColumnDefinitionComparator(def.getValue()));
this.metadata.put(
id,
new Name(
cfm.ksName,
cfm.cfName,
id,
Name.Kind.COLUMN_METADATA,
def.getValue().getValidator()));
}
} else {
// "dense" composite
this.isCompact = true;
this.hasCollections = false;
for (int i = 0; i < composite.types.size(); i++) {
ColumnIdentifier id = getColumnId(cfm, i);
this.columns.put(
id,
new Name(
cfm.ksName, cfm.cfName, id, Name.Kind.COLUMN_ALIAS, i, composite.types.get(i)));
}
this.value = createValue(cfm);
}
} else {
this.isComposite = false;
this.hasCollections = false;
if (!cfm.getColumnAliases().isEmpty() || cfm.getColumn_metadata().isEmpty()) {
// dynamic CF
this.isCompact = true;
ColumnIdentifier id = getColumnId(cfm, 0);
Name name = new Name(cfm.ksName, cfm.cfName, id, Name.Kind.COLUMN_ALIAS, 0, cfm.comparator);
this.columns.put(id, name);
this.value = createValue(cfm);
} else {
// static CF
this.isCompact = false;
this.value = null;
assert cfm.getValueAlias() == null;
assert cfm.getColumnAliases() == null || cfm.getColumnAliases().isEmpty();
for (Map.Entry<ByteBuffer, ColumnDefinition> def : cfm.getColumn_metadata().entrySet()) {
ColumnIdentifier id =
new ColumnIdentifier(def.getKey(), cfm.getColumnDefinitionComparator(def.getValue()));
this.metadata.put(
id,
new Name(
cfm.ksName,
cfm.cfName,
id,
Name.Kind.COLUMN_METADATA,
def.getValue().getValidator()));
}
}
}
assert value == null || metadata.isEmpty();
}
/** Returns true if the slice includes static columns, false otherwise. */
private boolean sliceIncludesStatics(ColumnSlice slice, CFMetaData cfm) {
return cfm.hasStaticColumns()
&& slice.includes(
reversed ? cfm.comparator.reverseComparator() : cfm.comparator,
cfm.comparator.staticPrefix().end());
}
/**
* Given arguments specifying an SSTable, and optionally an output file, export the contents of
* the SSTable to JSON.
*
* @param args command lines arguments
* @throws ConfigurationException on configuration failure (wrong params given)
*/
public static void main(String[] args) throws ConfigurationException {
CommandLineParser parser = new PosixParser();
try {
cmd = parser.parse(options, args);
} catch (ParseException e1) {
System.err.println(e1.getMessage());
printUsage();
System.exit(1);
}
if (cmd.getArgs().length != 1) {
System.err.println("You must supply exactly one sstable");
printUsage();
System.exit(1);
}
String[] keys = cmd.getOptionValues(KEY_OPTION);
HashSet<String> excludes =
new HashSet<>(
Arrays.asList(
cmd.getOptionValues(EXCLUDE_KEY_OPTION) == null
? new String[0]
: cmd.getOptionValues(EXCLUDE_KEY_OPTION)));
String ssTableFileName = new File(cmd.getArgs()[0]).getAbsolutePath();
if (Descriptor.isLegacyFile(new File(ssTableFileName))) {
System.err.println("Unsupported legacy sstable");
System.exit(1);
}
if (!new File(ssTableFileName).exists()) {
System.err.println("Cannot find file " + ssTableFileName);
System.exit(1);
}
Descriptor desc = Descriptor.fromFilename(ssTableFileName);
try {
CFMetaData metadata = metadataFromSSTable(desc);
if (cmd.hasOption(ENUMERATE_KEYS_OPTION)) {
JsonTransformer.keysToJson(
null,
iterToStream(new KeyIterator(desc, metadata)),
cmd.hasOption(RAW_TIMESTAMPS),
metadata,
System.out);
} else {
SSTableReader sstable = SSTableReader.openNoValidation(desc, metadata);
IPartitioner partitioner = sstable.getPartitioner();
final ISSTableScanner currentScanner;
if ((keys != null) && (keys.length > 0)) {
List<AbstractBounds<PartitionPosition>> bounds =
Arrays.stream(keys)
.filter(key -> !excludes.contains(key))
.map(metadata.getKeyValidator()::fromString)
.map(partitioner::decorateKey)
.sorted()
.map(DecoratedKey::getToken)
.map(token -> new Bounds<>(token.minKeyBound(), token.maxKeyBound()))
.collect(Collectors.toList());
currentScanner = sstable.getScanner(bounds.iterator());
} else {
currentScanner = sstable.getScanner();
}
Stream<UnfilteredRowIterator> partitions =
iterToStream(currentScanner)
.filter(
i ->
excludes.isEmpty()
|| !excludes.contains(
metadata.getKeyValidator().getString(i.partitionKey().getKey())));
if (cmd.hasOption(DEBUG_OUTPUT_OPTION)) {
AtomicLong position = new AtomicLong();
partitions.forEach(
partition -> {
position.set(currentScanner.getCurrentPosition());
if (!partition.partitionLevelDeletion().isLive()) {
System.out.println(
"["
+ metadata.getKeyValidator().getString(partition.partitionKey().getKey())
+ "]@"
+ position.get()
+ " "
+ partition.partitionLevelDeletion());
}
if (!partition.staticRow().isEmpty()) {
System.out.println(
"["
+ metadata.getKeyValidator().getString(partition.partitionKey().getKey())
+ "]@"
+ position.get()
+ " "
+ partition.staticRow().toString(metadata, true));
}
partition.forEachRemaining(
row -> {
System.out.println(
"["
+ metadata
.getKeyValidator()
.getString(partition.partitionKey().getKey())
+ "]@"
+ position.get()
+ " "
+ row.toString(metadata, false, true));
position.set(currentScanner.getCurrentPosition());
});
});
} else {
JsonTransformer.toJson(
currentScanner, partitions, cmd.hasOption(RAW_TIMESTAMPS), metadata, System.out);
}
}
} catch (IOException e) {
// throwing exception outside main with broken pipe causes windows cmd to hang
e.printStackTrace(System.err);
}
System.exit(0);
}
public void validate(ClientState state) throws RequestValidationException {
CFMetaData cfm = ThriftValidation.validateColumnFamily(keyspace(), columnFamily());
if (cfm.isCounter())
throw new InvalidRequestException("Secondary indexes are not supported on counter tables");
IndexTarget target = rawTarget.prepare(cfm);
ColumnDefinition cd = cfm.getColumnDefinition(target.column);
if (cd == null)
throw new InvalidRequestException("No column definition found for column " + target.column);
boolean isMap = cd.type instanceof MapType;
boolean isFrozenCollection = cd.type.isCollection() && !cd.type.isMultiCell();
if (target.isCollectionKeys) {
if (!isMap)
throw new InvalidRequestException(
"Cannot create index on keys of column " + target + " with non-map type");
if (!cd.type.isMultiCell())
throw new InvalidRequestException(
"Cannot create index on keys of frozen<map> column " + target);
} else if (target.isFullCollection) {
if (!isFrozenCollection)
throw new InvalidRequestException(
"full() indexes can only be created on frozen collections");
} else if (isFrozenCollection) {
throw new InvalidRequestException(
"Frozen collections currently only support full-collection indexes. "
+ "For example, 'CREATE INDEX ON <table>(full(<columnName>))'.");
}
if (cd.getIndexType() != null) {
boolean previousIsKeys = cd.hasIndexOption(SecondaryIndex.INDEX_KEYS_OPTION_NAME);
if (isMap && target.isCollectionKeys != previousIsKeys) {
String msg =
"Cannot create index on %s %s, an index on %s %s already exists and indexing "
+ "a map on both keys and values at the same time is not currently supported";
throw new InvalidRequestException(
String.format(
msg,
target.column,
target.isCollectionKeys ? "keys" : "values",
target.column,
previousIsKeys ? "keys" : "values"));
}
if (ifNotExists) return;
else throw new InvalidRequestException("Index already exists");
}
properties.validate();
// TODO: we could lift that limitation
if ((cfm.comparator.isDense() || !cfm.comparator.isCompound())
&& cd.kind != ColumnDefinition.Kind.REGULAR)
throw new InvalidRequestException(
"Secondary indexes are not supported on PRIMARY KEY columns in COMPACT STORAGE tables");
// It would be possible to support 2ndary index on static columns (but not without modifications
// of at least ExtendedFilter and
// CompositesIndex) and maybe we should, but that means a query like:
// SELECT * FROM foo WHERE static_column = 'bar'
// would pull the full partition every time the static column of partition is 'bar', which
// sounds like offering a
// fair potential for foot-shooting, so I prefer leaving that to a follow up ticket once we have
// identified cases where
// such indexing is actually useful.
if (cd.isStatic())
throw new InvalidRequestException("Secondary indexes are not allowed on static columns");
if (cd.kind == ColumnDefinition.Kind.PARTITION_KEY && cd.isOnAllComponents())
throw new InvalidRequestException(
String.format("Cannot create secondary index on partition key column %s", target.column));
}