@Test
public void testGetSliceFromSuperBasic() throws Throwable {
// tests slicing against data from one row spread across two sstables
final Table table = Table.open("Keyspace1");
final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Super1");
final String ROW = "row2";
RowMutation rm = new RowMutation("Keyspace1", ROW);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Super1");
SuperColumn sc = new SuperColumn("sc1".getBytes(), new LongType());
sc.addColumn(new Column(getBytes(1), "val1".getBytes(), 1L));
cf.addColumn(sc);
rm.add(cf);
rm.apply();
Runnable verify =
new WrappedRunnable() {
public void runMayThrow() throws Exception {
ColumnFamily cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Super1"),
ArrayUtils.EMPTY_BYTE_ARRAY,
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
10);
assertColumns(cf, "sc1");
assertEquals(
new String(cf.getColumn("sc1".getBytes()).getSubColumn(getBytes(1)).value()),
"val1");
}
};
reTest(table.getColumnFamilyStore("Standard1"), verify);
}
private int discardLast(ColumnFamily cf, int toDiscard, ColumnFamily newCf) {
boolean isReversed = isReversed();
DeletionInfo.InOrderTester tester = cf.deletionInfo().inOrderTester(isReversed);
return isReversed
? discardHead(cf, toDiscard, newCf, cf.reverseIterator(), tester)
: discardTail(cf, toDiscard, newCf, cf.iterator(), tester);
}
public Column get_column(String table, String key, ColumnPath column_path, int consistency_level)
throws InvalidRequestException, NotFoundException {
if (logger.isDebugEnabled()) logger.debug("get_column");
ThriftValidation.validateColumnPath(table, column_path);
QueryPath path = new QueryPath(column_path.column_family, column_path.super_column);
ColumnFamily cfamily =
readColumnFamily(
new SliceByNamesReadCommand(table, key, path, Arrays.asList(column_path.column)),
consistency_level);
// TODO can we leverage getSlice here and just check that it returns one column?
if (cfamily == null) {
throw new NotFoundException();
}
Collection<IColumn> columns = null;
if (column_path.super_column != null) {
IColumn column = cfamily.getColumn(column_path.super_column);
if (column != null) {
columns = column.getSubColumns();
}
} else {
columns = cfamily.getSortedColumns();
}
if (columns == null || columns.size() == 0) {
throw new NotFoundException();
}
assert columns.size() == 1;
IColumn column = columns.iterator().next();
if (column.isMarkedForDelete()) {
throw new NotFoundException();
}
return new Column(column.name(), column.value(), column.timestamp());
}
private int discardHead(
ColumnFamily cf,
int toDiscard,
ColumnFamily copy,
Iterator<Column> iter,
DeletionInfo.InOrderTester tester) {
ColumnCounter counter = columnCounter();
List<Column> staticColumns = new ArrayList<>(cfm.staticColumns().size());
// Discard the first 'toDiscard' live, non-static columns
while (iter.hasNext()) {
Column c = iter.next();
// if it's a static column, don't count it and save it to add to the trimmed results
ColumnDefinition columnDef = cfm.getColumnDefinitionFromColumnName(c.name());
if (columnDef != null && columnDef.type == ColumnDefinition.Type.STATIC) {
staticColumns.add(c);
continue;
}
counter.count(c, tester);
// once we've discarded the required amount, add the rest
if (counter.live() > toDiscard) {
for (Column staticColumn : staticColumns) copy.addColumn(staticColumn);
copy.addColumn(c);
while (iter.hasNext()) copy.addColumn(iter.next());
}
}
return Math.min(counter.live(), toDiscard);
}
public List<SuperColumn> get_slice_super(
String table,
String key,
String column_family,
byte[] start,
byte[] finish,
boolean is_ascending,
int count,
int consistency_level)
throws InvalidRequestException {
if (logger.isDebugEnabled()) logger.debug("get_slice_super");
if (!DatabaseDescriptor.getColumnFamilyType(table, column_family).equals("Super"))
throw new InvalidRequestException("get_slice_super requires a super CF name");
if (count <= 0) throw new InvalidRequestException("get_slice_super requires positive count");
ColumnFamily cfamily =
readColumnFamily(
new SliceFromReadCommand(
table, key, new QueryPath(column_family), start, finish, is_ascending, count),
consistency_level);
if (cfamily == null) {
return EMPTY_SUPERCOLUMNS;
}
Collection<IColumn> columns = cfamily.getSortedColumns();
return thriftifySuperColumns(columns, !is_ascending);
}
@Test
public void testGetSliceFromLarge() throws Throwable {
// tests slicing against 1000 columns in an sstable
Table table = Table.open("Keyspace1");
ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
String key = "row3";
RowMutation rm = new RowMutation("Keyspace1", key);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
for (int i = 1000; i < 2000; i++) cf.addColumn(column("col" + i, ("v" + i), 1L));
rm.add(cf);
rm.apply();
cfStore.forceBlockingFlush();
validateSliceLarge(cfStore);
// compact so we have a big row with more than the minimum index count
if (cfStore.getSSTables().size() > 1) {
CompactionManager.instance.submitMajor(cfStore).get();
}
SSTableReader sstable = cfStore.getSSTables().iterator().next();
DecoratedKey decKey = sstable.getPartitioner().decorateKey(key);
SSTable.PositionSize info = sstable.getPosition(decKey);
BufferedRandomAccessFile file = new BufferedRandomAccessFile(sstable.getFilename(), "r");
file.seek(info.position);
assert file.readUTF().equals(key);
file.readInt();
IndexHelper.skipBloomFilter(file);
ArrayList<IndexHelper.IndexInfo> indexes = IndexHelper.deserializeIndex(file);
assert indexes.size() > 2;
validateSliceLarge(cfStore);
}
@VisibleForTesting
List<Row> discardLast(List<Row> rows, int toDiscard) {
if (toDiscard == 0 || rows.isEmpty()) return rows;
int i = rows.size() - 1;
DecoratedKey lastKey = null;
ColumnFamily lastCf = null;
while (toDiscard > 0 && i >= 0) {
Row last = rows.get(i--);
lastKey = last.key;
lastCf = last.cf.cloneMeShallow(isReversed());
toDiscard -=
isReversed()
? discardFirst(last.cf, toDiscard, lastCf)
: discardLast(last.cf, toDiscard, lastCf);
}
// If there is less live data than to discard, all is discarded
if (toDiscard > 0) return Collections.<Row>emptyList();
// i is the index of the last row that we are sure to keep. On top of that,
// we also keep lastCf is it hasn't been fully emptied by the last iteration above.
int count = lastCf.getColumnCount();
int newSize = count == 0 ? i + 1 : i + 2;
List<Row> newRows = new ArrayList<Row>(newSize);
newRows.addAll(rows.subList(0, i + 1));
if (count != 0) newRows.add(new Row(lastKey, lastCf));
return newRows;
}
public SuperColumn get_super_column(
String table, String key, SuperColumnPath super_column_path, int consistency_level)
throws InvalidRequestException, NotFoundException {
if (logger.isDebugEnabled()) logger.debug("get_superColumn");
ThriftValidation.validateSuperColumnPath(table, super_column_path);
ColumnFamily cfamily =
readColumnFamily(
new SliceByNamesReadCommand(
table,
key,
new QueryPath(super_column_path.column_family),
Arrays.asList(super_column_path.super_column)),
consistency_level);
if (cfamily == null) {
throw new NotFoundException();
}
Collection<IColumn> columns = cfamily.getSortedColumns();
if (columns == null || columns.size() == 0) {
throw new NotFoundException();
}
assert columns.size() == 1;
IColumn column = columns.iterator().next();
if (column.getSubColumns().size() == 0) {
throw new NotFoundException();
}
return new SuperColumn(column.name(), thriftifyColumns(column.getSubColumns()));
}
@Test
public void testGetRowSingleColumn() throws Throwable {
final Table table = Table.open("Keyspace1");
final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
RowMutation rm = new RowMutation("Keyspace1", TEST_KEY);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
cf.addColumn(column("col1", "val1", 1L));
cf.addColumn(column("col2", "val2", 1L));
cf.addColumn(column("col3", "val3", 1L));
rm.add(cf);
rm.apply();
Runnable verify =
new WrappedRunnable() {
public void runMayThrow() throws Exception {
ColumnFamily cf;
cf =
cfStore.getColumnFamily(
new NamesQueryFilter(TEST_KEY, new QueryPath("Standard1"), "col1".getBytes()));
assertColumns(cf, "col1");
cf =
cfStore.getColumnFamily(
new NamesQueryFilter(TEST_KEY, new QueryPath("Standard1"), "col3".getBytes()));
assertColumns(cf, "col3");
}
};
reTest(table.getColumnFamilyStore("Standard1"), verify);
}
private static void delete(ColumnFamily cf, int from, int to, long timestamp) {
cf.delete(
new DeletionInfo(
b(from),
b(to),
cf.getComparator(),
timestamp,
(int) (System.currentTimeMillis() / 1000)));
}
@Test
public void simpleQueryWithRangeTombstoneTest() throws Exception {
Table table = Table.open(KSNAME);
ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME);
// Inserting data
String key = "k1";
RowMutation rm;
ColumnFamily cf;
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
for (int i = 0; i < 40; i += 2) add(rm, i, 0);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
cf = rm.addOrGet(CFNAME);
delete(cf, 10, 22, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
for (int i = 1; i < 40; i += 2) add(rm, i, 2);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
cf = rm.addOrGet(CFNAME);
delete(cf, 19, 27, 3);
rm.apply();
// We don't flush to test with both a range tomsbtone in memtable and in sstable
QueryPath path = new QueryPath(CFNAME);
// Queries by name
int[] live = new int[] {4, 9, 11, 17, 28};
int[] dead = new int[] {12, 19, 21, 24, 27};
SortedSet<ByteBuffer> columns = new TreeSet<ByteBuffer>(cfs.getComparator());
for (int i : live) columns.add(b(i));
for (int i : dead) columns.add(b(i));
cf = cfs.getColumnFamily(QueryFilter.getNamesFilter(dk(key), path, columns));
for (int i : live) assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
for (int i : dead)
assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live";
// Queries by slices
cf =
cfs.getColumnFamily(
QueryFilter.getSliceFilter(dk(key), path, b(7), b(30), false, Integer.MAX_VALUE));
for (int i : new int[] {7, 8, 9, 11, 13, 15, 17, 28, 29, 30})
assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
for (int i : new int[] {10, 12, 14, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27})
assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live";
}
public RowMutation toSchema(long timestamp) {
RowMutation rm = new RowMutation(Table.SYSTEM_TABLE, SystemTable.getSchemaKSKey(name));
ColumnFamily cf = rm.addOrGet(SystemTable.SCHEMA_KEYSPACES_CF);
cf.addColumn(Column.create(name, timestamp, "name"));
cf.addColumn(Column.create(durableWrites, timestamp, "durable_writes"));
cf.addColumn(Column.create(strategyClass.getName(), timestamp, "strategy_class"));
cf.addColumn(Column.create(json(strategyOptions), timestamp, "strategy_options"));
for (CFMetaData cfm : cfMetaData.values()) cfm.toSchema(rm, timestamp);
return rm;
}
public ColumnFamily updateForKey(
ByteBuffer key, ColumnNameBuilder builder, UpdateParameters params)
throws InvalidRequestException {
CFDefinition cfDef = cfm.getCfDef();
ColumnFamily cf = UnsortedColumns.factory.create(cfm);
// Inserting the CQL row marker (see #4361)
// We always need to insert a marker, because of the following situation:
// CREATE TABLE t ( k int PRIMARY KEY, c text );
// INSERT INTO t(k, c) VALUES (1, 1)
// DELETE c FROM t WHERE k = 1;
// SELECT * FROM t;
// The last query should return one row (but with c == null). Adding
// the marker with the insert make sure the semantic is correct (while making sure a
// 'DELETE FROM t WHERE k = 1' does remove the row entirely)
//
// We never insert markers for Super CF as this would confuse the thrift side.
if (cfDef.isComposite && !cfDef.isCompact && !cfm.isSuper()) {
ByteBuffer name = builder.copy().add(ByteBufferUtil.EMPTY_BYTE_BUFFER).build();
cf.addColumn(params.makeColumn(name, ByteBufferUtil.EMPTY_BYTE_BUFFER));
}
List<Operation> updates = getOperations();
if (cfDef.isCompact) {
if (builder.componentCount() == 0)
throw new InvalidRequestException(
String.format("Missing PRIMARY KEY part %s", cfDef.columns.values().iterator().next()));
if (cfDef.value == null) {
// compact + no compact value implies there is no column outside the PK. So no operation
// could
// have passed through validation
assert updates.isEmpty();
setToEmptyOperation.execute(key, cf, builder.copy(), params);
} else {
// compact means we don't have a row marker, so don't accept to set only the PK. See
// CASSANDRA-5648.
if (updates.isEmpty())
throw new InvalidRequestException(
String.format("Column %s is mandatory for this COMPACT STORAGE table", cfDef.value));
for (Operation update : updates) update.execute(key, cf, builder.copy(), params);
}
} else {
for (Operation update : updates) update.execute(key, cf, builder.copy(), params);
}
return cf;
}
private void testDontPurgeAccidentaly(String k, String cfname)
throws IOException, ExecutionException, InterruptedException {
// This test catches the regression of CASSANDRA-2786
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
// disable compaction while flushing
cfs.clearUnsafe();
cfs.disableAutoCompaction();
// Add test row
DecoratedKey key = Util.dk(k);
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
rm.add(
cfname,
CompositeType.build(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
0);
rm.apply();
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesBefore = cfs.getSSTables();
QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis());
assert !(cfs.getColumnFamily(filter).getColumnCount() == 0);
// Remove key
rm = new RowMutation(KEYSPACE1, key.key);
rm.delete(cfname, 2);
rm.apply();
ColumnFamily cf = cfs.getColumnFamily(filter);
assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;
// Sleep one second so that the removal is indeed purgeable even with gcgrace == 0
Thread.sleep(1000);
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesAfter = cfs.getSSTables();
Collection<SSTableReader> toCompact = new ArrayList<SSTableReader>();
for (SSTableReader sstable : sstablesAfter)
if (!sstablesBefore.contains(sstable)) toCompact.add(sstable);
Util.compact(cfs, toCompact);
cf = cfs.getColumnFamily(filter);
assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;
}
public static void assertColumns(ColumnFamily cf, String... columnNames) {
Collection<IColumn> columns = cf == null ? new TreeSet<IColumn>() : cf.getSortedColumns();
List<String> L = new ArrayList<String>();
for (IColumn column : columns) {
L.add(new String(column.name()));
}
assert Arrays.equals(L.toArray(new String[columns.size()]), columnNames)
: "Columns ["
+ ((cf == null) ? "" : cf.getComparator().getColumnsString(columns))
+ "]"
+ " is not expected ["
+ StringUtils.join(columnNames, ",")
+ "]";
}
public Iterator<RangeTombstone> getRangeTombstoneIterator(final ColumnFamily source) {
final DeletionInfo delInfo = source.deletionInfo();
if (!delInfo.hasRanges() || slices.length == 0) return Iterators.emptyIterator();
return new AbstractIterator<RangeTombstone>() {
private int sliceIdx = 0;
private Iterator<RangeTombstone> sliceIter = currentRangeIter();
protected RangeTombstone computeNext() {
while (true) {
if (sliceIter.hasNext()) return sliceIter.next();
if (!nextSlice()) return endOfData();
sliceIter = currentRangeIter();
}
}
private Iterator<RangeTombstone> currentRangeIter() {
ColumnSlice slice = slices[reversed ? (slices.length - 1 - sliceIdx) : sliceIdx];
return reversed
? delInfo.rangeIterator(slice.finish, slice.start)
: delInfo.rangeIterator(slice.start, slice.finish);
}
private boolean nextSlice() {
return ++sliceIdx < slices.length;
}
};
}
/** for resultsets of standard columns */
private List<Column> getSlice(ReadCommand command, int consistency_level)
throws InvalidRequestException {
ColumnFamily cfamily = readColumnFamily(command, consistency_level);
boolean reverseOrder = false;
if (command instanceof SliceFromReadCommand)
reverseOrder = !((SliceFromReadCommand) command).isAscending;
if (cfamily == null || cfamily.getColumnsMap().size() == 0) {
return EMPTY_COLUMNS;
}
if (cfamily.isSuper()) {
IColumn column = cfamily.getColumnsMap().values().iterator().next();
return thriftifyColumns(column.getSubColumns(), reverseOrder);
}
return thriftifyColumns(cfamily.getSortedColumns(), reverseOrder);
}
@Test
public void testGetRowSliceByRange() throws Throwable {
String key = TEST_KEY + "slicerow";
Table table = Table.open("Keyspace1");
ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
RowMutation rm = new RowMutation("Keyspace1", key);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
// First write "a", "b", "c"
cf.addColumn(column("a", "val1", 1L));
cf.addColumn(column("b", "val2", 1L));
cf.addColumn(column("c", "val3", 1L));
rm.add(cf);
rm.apply();
cf =
cfStore.getColumnFamily(
key, new QueryPath("Standard1"), "b".getBytes(), "c".getBytes(), false, 100);
assertEquals(2, cf.getColumnCount());
cf =
cfStore.getColumnFamily(
key, new QueryPath("Standard1"), "b".getBytes(), "b".getBytes(), false, 100);
assertEquals(1, cf.getColumnCount());
cf =
cfStore.getColumnFamily(
key, new QueryPath("Standard1"), "b".getBytes(), "c".getBytes(), false, 1);
assertEquals(1, cf.getColumnCount());
cf =
cfStore.getColumnFamily(
key, new QueryPath("Standard1"), "c".getBytes(), "b".getBytes(), false, 1);
assertNull(cf);
}
@Test
public void testGetSliceNoMatch() throws Throwable {
Table table = Table.open("Keyspace1");
RowMutation rm = new RowMutation("Keyspace1", "row1000");
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard2");
cf.addColumn(column("col1", "val1", 1));
rm.add(cf);
rm.apply();
validateGetSliceNoMatch(table);
table.getColumnFamilyStore("Standard2").forceBlockingFlush();
validateGetSliceNoMatch(table);
Collection<SSTableReader> ssTables = table.getColumnFamilyStore("Standard2").getSSTables();
assertEquals(1, ssTables.size());
ssTables.iterator().next().forceBloomFilterFailures();
validateGetSliceNoMatch(table);
}
public boolean isFullyCoveredBy(ColumnFamily cf, long now) {
// cf is the beginning of a partition. It covers this filter if:
// 1) either this filter requests the head of the partition and request less
// than what cf has to offer (note: we do need to use getLiveCount() for that
// as it knows if the filter count cells or CQL3 rows).
// 2) the start and finish bound of this filter are included in cf.
if (isHeadFilter() && count <= getLiveCount(cf, now)) return true;
if (start().isEmpty() || finish().isEmpty() || !cf.hasColumns()) return false;
Composite low = isReversed() ? finish() : start();
Composite high = isReversed() ? start() : finish();
CellName first = cf.iterator(ColumnSlice.ALL_COLUMNS_ARRAY).next().name();
CellName last = cf.reverseIterator(ColumnSlice.ALL_COLUMNS_ARRAY).next().name();
return cf.getComparator().compare(first, low) <= 0
&& cf.getComparator().compare(high, last) <= 0;
}
// TODO move gcBefore into a field
public void collateColumns(
final ColumnFamily returnCF,
List<? extends CloseableIterator<IColumn>> toCollate,
final int gcBefore) {
IFilter topLevelFilter = (superFilter == null ? filter : superFilter);
Comparator<IColumn> fcomp = topLevelFilter.getColumnComparator(returnCF.getComparator());
// define a 'reduced' iterator that merges columns w/ the same name, which
// greatly simplifies computing liveColumns in the presence of tombstones.
MergeIterator.Reducer<IColumn, IColumn> reducer =
new MergeIterator.Reducer<IColumn, IColumn>() {
ColumnFamily curCF = returnCF.cloneMeShallow();
public void reduce(IColumn current) {
if (curCF.isSuper() && curCF.isEmpty()) {
// If it is the first super column we add, we must clone it since other super column
// may modify
// it otherwise and it could be aliased in a memtable somewhere. We'll also don't have
// to care about what
// consumers make of the result (for instance CFS.getColumnFamily() call
// removeDeleted() on the
// result which removes column; which shouldn't be done on the original super column).
assert current instanceof SuperColumn;
curCF.addColumn(((SuperColumn) current).cloneMe());
} else {
curCF.addColumn(current);
}
}
protected IColumn getReduced() {
IColumn c = curCF.iterator().next();
if (superFilter != null) {
// filterSuperColumn only looks at immediate parent (the supercolumn) when determining
// if a subcolumn
// is still live, i.e., not shadowed by the parent's tombstone. so, bump it up
// temporarily to the tombstone
// time of the cf, if that is greater.
long deletedAt = c.getMarkedForDeleteAt();
if (returnCF.getMarkedForDeleteAt() > deletedAt)
((SuperColumn) c).delete(c.getLocalDeletionTime(), returnCF.getMarkedForDeleteAt());
c = filter.filterSuperColumn((SuperColumn) c, gcBefore);
((SuperColumn) c)
.delete(
c.getLocalDeletionTime(),
deletedAt); // reset sc tombstone time to what it should be
}
curCF.clear();
return c;
}
};
Iterator<IColumn> reduced = MergeIterator.get(toCollate, fcomp, reducer);
topLevelFilter.collectReducedColumns(returnCF, reduced, gcBefore);
}
public int get_column_count(
String table, String key, ColumnParent column_parent, int consistency_level)
throws InvalidRequestException {
if (logger.isDebugEnabled()) logger.debug("get_column_count");
// validateColumnParent assumes we require simple columns; g_c_c is the only
// one of the columnParent-taking apis that can also work at the SC level.
// so we roll a one-off validator here.
String cfType = ThriftValidation.validateColumnFamily(table, column_parent.column_family);
if (cfType.equals("Standard") && column_parent.super_column != null) {
throw new InvalidRequestException(
"columnfamily alone is required for standard CF " + column_parent.column_family);
}
ColumnFamily cfamily;
cfamily =
readColumnFamily(
new SliceFromReadCommand(
table,
key,
column_parent,
ArrayUtils.EMPTY_BYTE_ARRAY,
ArrayUtils.EMPTY_BYTE_ARRAY,
true,
Integer.MAX_VALUE),
consistency_level);
if (cfamily == null) {
return 0;
}
Collection<IColumn> columns = null;
if (column_parent.super_column != null) {
IColumn column = cfamily.getColumn(column_parent.super_column);
if (column != null) {
columns = column.getSubColumns();
}
} else {
columns = cfamily.getSortedColumns();
}
if (columns == null || columns.size() == 0) {
return 0;
}
return columns.size();
}
public List<SuperColumn> get_slice_super_by_names(
String table,
String key,
String column_family,
List<byte[]> super_column_names,
int consistency_level)
throws InvalidRequestException {
if (logger.isDebugEnabled()) logger.debug("get_slice_super_by_names");
ThriftValidation.validateColumnFamily(table, column_family);
ColumnFamily cfamily =
readColumnFamily(
new SliceByNamesReadCommand(
table, key, new QueryPath(column_family), super_column_names),
consistency_level);
if (cfamily == null) {
return EMPTY_SUPERCOLUMNS;
}
return thriftifySuperColumns(cfamily.getSortedColumns());
}
@Test
public void testGetRowNoColumns() throws Throwable {
final Table table = Table.open("Keyspace2");
final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard3");
RowMutation rm = new RowMutation("Keyspace2", TEST_KEY);
ColumnFamily cf = ColumnFamily.create("Keyspace2", "Standard3");
cf.addColumn(column("col1", "val1", 1L));
rm.add(cf);
rm.apply();
Runnable verify =
new WrappedRunnable() {
public void runMayThrow() throws Exception {
ColumnFamily cf;
cf =
cfStore.getColumnFamily(
new NamesQueryFilter(
TEST_KEY, new QueryPath("Standard3"), new TreeSet<byte[]>()));
assertColumns(cf);
cf =
cfStore.getColumnFamily(
new SliceQueryFilter(
TEST_KEY,
new QueryPath("Standard3"),
ArrayUtils.EMPTY_BYTE_ARRAY,
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
0));
assertColumns(cf);
cf =
cfStore.getColumnFamily(
new NamesQueryFilter(TEST_KEY, new QueryPath("Standard3"), "col99".getBytes()));
assertColumns(cf);
}
};
reTest(table.getColumnFamilyStore("Standard3"), verify);
}
@Test
public void overlappingRangeTest() throws Exception {
CompactionManager.instance.disableAutoCompaction();
Table table = Table.open(KSNAME);
ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME);
// Inserting data
String key = "k2";
RowMutation rm;
ColumnFamily cf;
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
for (int i = 0; i < 20; i++) add(rm, i, 0);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
cf = rm.addOrGet(CFNAME);
delete(cf, 5, 15, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
cf = rm.addOrGet(CFNAME);
delete(cf, 5, 10, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
cf = rm.addOrGet(CFNAME);
delete(cf, 5, 8, 2);
rm.apply();
cfs.forceBlockingFlush();
QueryPath path = new QueryPath(CFNAME);
cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(key), path));
for (int i = 0; i < 5; i++)
assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
for (int i = 16; i < 20; i++)
assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
for (int i = 5; i <= 15; i++)
assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live";
// Compact everything and re-test
CompactionManager.instance.performMaximal(cfs);
cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(key), path));
for (int i = 0; i < 5; i++)
assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
for (int i = 16; i < 20; i++)
assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
for (int i = 5; i <= 15; i++)
assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live";
}
@Test
public void testGetSliceFromAdvanced() throws Throwable {
// tests slicing against data from one row spread across two sstables
final Table table = Table.open("Keyspace1");
final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
final String ROW = "row2";
RowMutation rm = new RowMutation("Keyspace1", ROW);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
cf.addColumn(column("col1", "val1", 1L));
cf.addColumn(column("col2", "val2", 1L));
cf.addColumn(column("col3", "val3", 1L));
cf.addColumn(column("col4", "val4", 1L));
cf.addColumn(column("col5", "val5", 1L));
cf.addColumn(column("col6", "val6", 1L));
rm.add(cf);
rm.apply();
cfStore.forceBlockingFlush();
rm = new RowMutation("Keyspace1", ROW);
cf = ColumnFamily.create("Keyspace1", "Standard1");
cf.addColumn(column("col1", "valx", 2L));
cf.addColumn(column("col2", "valx", 2L));
cf.addColumn(column("col3", "valx", 2L));
rm.add(cf);
rm.apply();
Runnable verify =
new WrappedRunnable() {
public void runMayThrow() throws Exception {
ColumnFamily cf;
cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Standard1"),
"col2".getBytes(),
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
3);
assertColumns(cf, "col2", "col3", "col4");
assertEquals(new String(cf.getColumn("col2".getBytes()).value()), "valx");
assertEquals(new String(cf.getColumn("col3".getBytes()).value()), "valx");
assertEquals(new String(cf.getColumn("col4".getBytes()).value()), "val4");
}
};
reTest(table.getColumnFamilyStore("Standard1"), verify);
}
private String getSlicesInfo(ColumnFamily container) {
StringBuilder sb = new StringBuilder();
CellNameType type = container.metadata().comparator;
for (ColumnSlice sl : slices) {
assert sl != null;
sb.append('[');
sb.append(type.getString(sl.start));
sb.append('-');
sb.append(type.getString(sl.finish));
sb.append(']');
}
return sb.toString();
}
public void trim(ColumnFamily cf, int trimTo, long now) {
// each cell can increment the count by at most one, so if we have fewer cells than trimTo, we
// can skip trimming
if (cf.getColumnCount() < trimTo) return;
ColumnCounter counter = columnCounter(cf.getComparator(), now);
Collection<Cell> cells = reversed ? cf.getReverseSortedColumns() : cf.getSortedColumns();
DeletionInfo.InOrderTester tester = cf.deletionInfo().inOrderTester(reversed);
for (Iterator<Cell> iter = cells.iterator(); iter.hasNext(); ) {
Cell cell = iter.next();
counter.count(cell, tester);
if (counter.live() > trimTo) {
iter.remove();
while (iter.hasNext()) {
iter.next();
iter.remove();
}
}
}
}
private int discardTail(
ColumnFamily cf,
int toDiscard,
ColumnFamily copy,
Iterator<Column> iter,
DeletionInfo.InOrderTester tester) {
// Redoing the counting like that is not extremely efficient.
// This is called only for reversed slices or in the case of a race between
// paging and a deletion (pretty unlikely), so this is probably acceptable.
int liveCount = columnCounter().countAll(cf).live();
ColumnCounter counter = columnCounter();
// Discard the last 'toDiscard' live (so stop adding as sound as we're past 'liveCount -
// toDiscard')
while (iter.hasNext()) {
Column c = iter.next();
counter.count(c, tester);
if (counter.live() > liveCount - toDiscard) break;
copy.addColumn(c);
}
return Math.min(liveCount, toDiscard);
}
/**
* Retrieves a local subBlock
*
* @param blockId row key
* @param sblockId SubBlock column name
* @param offset inside the sblock
* @return a local sublock
* @throws TException
*/
private LocalBlock getLocalSubBlock(
String subBlockCFName, ByteBuffer blockId, ByteBuffer sblockId, int offset)
throws TException {
DecoratedKey<Token<?>> decoratedKey =
new DecoratedKey<Token<?>>(StorageService.getPartitioner().getToken(blockId), blockId);
Table table = Table.open(cfsKeyspace);
ColumnFamilyStore sblockStore = table.getColumnFamilyStore(subBlockCFName);
Collection<SSTableReader> sstables = sblockStore.getSSTables();
for (SSTableReader sstable : sstables) {
long position = sstable.getPosition(decoratedKey, Operator.EQ);
if (position == -1) continue;
String filename = sstable.descriptor.filenameFor(Component.DATA);
RandomAccessFile raf = null;
int mappedLength = -1;
MappedByteBuffer mappedData = null;
MappedFileDataInput file = null;
try {
raf = new RandomAccessFile(filename, "r");
assert position < raf.length();
mappedLength =
(raf.length() - position) < Integer.MAX_VALUE
? (int) (raf.length() - position)
: Integer.MAX_VALUE;
mappedData = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, position, mappedLength);
file = new MappedFileDataInput(mappedData, filename, 0);
if (file == null) continue;
// Verify key was found in data file
DecoratedKey keyInDisk =
SSTableReader.decodeKey(
sstable.partitioner, sstable.descriptor, ByteBufferUtil.readWithShortLength(file));
assert keyInDisk.equals(decoratedKey)
: String.format("%s != %s in %s", keyInDisk, decoratedKey, file.getPath());
long rowSize = SSTableReader.readRowSize(file, sstable.descriptor);
assert rowSize > 0;
assert rowSize < mappedLength;
Filter bf = IndexHelper.defreezeBloomFilter(file, sstable.descriptor.usesOldBloomFilter);
// verify this column in in this version of the row.
if (!bf.isPresent(sblockId)) continue;
List<IndexHelper.IndexInfo> indexList = IndexHelper.deserializeIndex(file);
// we can stop early if bloom filter says none of the
// columns actually exist -- but,
// we can't stop before initializing the cf above, in
// case there's a relevant tombstone
ColumnFamilySerializer serializer = ColumnFamily.serializer();
try {
ColumnFamily cf =
serializer.deserializeFromSSTableNoColumns(
ColumnFamily.create(sstable.metadata), file);
if (cf.isMarkedForDelete()) continue;
} catch (Exception e) {
e.printStackTrace();
throw new IOException(
serializer
+ " failed to deserialize "
+ sstable.getColumnFamilyName()
+ " with "
+ sstable.metadata
+ " from "
+ file,
e);
}
Integer sblockLength = null;
if (indexList == null) sblockLength = seekToSubColumn(sstable.metadata, file, sblockId);
else sblockLength = seekToSubColumn(sstable.metadata, file, sblockId, indexList);
if (sblockLength == null || sblockLength < 0) continue;
int bytesReadFromStart = mappedLength - (int) file.bytesRemaining();
if (logger.isDebugEnabled())
logger.debug("BlockLength = " + sblockLength + " Availible " + file.bytesRemaining());
assert offset <= sblockLength : String.format("%d > %d", offset, sblockLength);
long dataOffset = position + bytesReadFromStart;
if (file.bytesRemaining() == 0 || sblockLength == 0) continue;
return new LocalBlock(file.getPath(), dataOffset + offset, sblockLength - offset);
} catch (IOException e) {
throw new TException(e);
} finally {
FileUtils.closeQuietly(raf);
}
}
return null;
}