@Test
public void testGetSliceFromLarge() throws Throwable {
// tests slicing against 1000 columns in an sstable
Table table = Table.open("Keyspace1");
ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
String key = "row3";
RowMutation rm = new RowMutation("Keyspace1", key);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
for (int i = 1000; i < 2000; i++) cf.addColumn(column("col" + i, ("v" + i), 1L));
rm.add(cf);
rm.apply();
cfStore.forceBlockingFlush();
validateSliceLarge(cfStore);
// compact so we have a big row with more than the minimum index count
if (cfStore.getSSTables().size() > 1) {
CompactionManager.instance.submitMajor(cfStore).get();
}
SSTableReader sstable = cfStore.getSSTables().iterator().next();
DecoratedKey decKey = sstable.getPartitioner().decorateKey(key);
SSTable.PositionSize info = sstable.getPosition(decKey);
BufferedRandomAccessFile file = new BufferedRandomAccessFile(sstable.getFilename(), "r");
file.seek(info.position);
assert file.readUTF().equals(key);
file.readInt();
IndexHelper.skipBloomFilter(file);
ArrayList<IndexHelper.IndexInfo> indexes = IndexHelper.deserializeIndex(file);
assert indexes.size() > 2;
validateSliceLarge(cfStore);
}
@Test
public void testRemoveColumn() throws IOException, ExecutionException, InterruptedException {
Table table = Table.open("Keyspace1");
ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
RowMutation rm;
DecoratedKey dk = Util.dk("key1");
// add data
rm = new RowMutation("Keyspace1", dk.key);
rm.add(
new QueryPath("Standard1", null, "Column1".getBytes()),
"asdf".getBytes(),
new TimestampClock(0));
rm.apply();
store.forceBlockingFlush();
// remove
rm = new RowMutation("Keyspace1", dk.key);
rm.delete(new QueryPath("Standard1", null, "Column1".getBytes()), new TimestampClock(1));
rm.apply();
ColumnFamily retrieved =
store.getColumnFamily(
QueryFilter.getNamesFilter(dk, new QueryPath("Standard1"), "Column1".getBytes()));
assert retrieved.getColumn("Column1".getBytes()).isMarkedForDelete();
assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE));
assertNull(
Util.cloneAndRemoveDeleted(
store.getColumnFamily(QueryFilter.getIdentityFilter(dk, new QueryPath("Standard1"))),
Integer.MAX_VALUE));
}
// Test compaction of hints column family. It shouldn't remove all columns on compaction.
@Test
public void testCompactionOfHintsCF() throws Exception {
// prepare hints column family
Table systemTable = Table.open("system");
ColumnFamilyStore hintStore = systemTable.getColumnFamilyStore(SystemTable.HINTS_CF);
hintStore.clearUnsafe();
hintStore.metadata.gcGraceSeconds(36000); // 10 hours
hintStore.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getCanonicalName());
hintStore.disableAutoCompaction();
// insert 1 hint
RowMutation rm = new RowMutation(TABLE4, ByteBufferUtil.bytes(1));
rm.add(
new QueryPath(STANDARD1_CF, null, ByteBufferUtil.bytes(String.valueOf(COLUMN1))),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
System.currentTimeMillis());
RowMutation.hintFor(rm, UUID.randomUUID()).apply();
// flush data to disk
hintStore.forceBlockingFlush();
assertEquals(1, hintStore.getSSTables().size());
// submit compaction
FBUtilities.waitOnFuture(HintedHandOffManager.instance.compact());
while (CompactionManager.instance.getPendingTasks() > 0
|| CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);
// single row should not be removed because of gc_grace_seconds
// is 10 hours and there are no any tombstones in sstable
assertEquals(1, hintStore.getSSTables().size());
}
private void testRowMutationWrite() throws IOException {
RowMutation standardRowRm = new RowMutation(statics.KS, statics.StandardRow);
RowMutation superRowRm = new RowMutation(statics.KS, statics.SuperRow);
RowMutation standardRm = new RowMutation(statics.KS, statics.Key, statics.StandardCf);
RowMutation superRm = new RowMutation(statics.KS, statics.Key, statics.SuperCf);
Map<UUID, ColumnFamily> mods = new HashMap<UUID, ColumnFamily>();
mods.put(statics.StandardCf.metadata().cfId, statics.StandardCf);
mods.put(statics.SuperCf.metadata().cfId, statics.SuperCf);
RowMutation mixedRm = new RowMutation(statics.KS, statics.Key, mods);
DataOutputStream out = getOutput("db.RowMutation.bin");
RowMutation.serializer.serialize(standardRowRm, out, getVersion());
RowMutation.serializer.serialize(superRowRm, out, getVersion());
RowMutation.serializer.serialize(standardRm, out, getVersion());
RowMutation.serializer.serialize(superRm, out, getVersion());
RowMutation.serializer.serialize(mixedRm, out, getVersion());
standardRowRm.createMessage().serialize(out, getVersion());
superRowRm.createMessage().serialize(out, getVersion());
standardRm.createMessage().serialize(out, getVersion());
superRm.createMessage().serialize(out, getVersion());
mixedRm.createMessage().serialize(out, getVersion());
out.close();
// test serializedSize
testSerializedSize(standardRowRm, RowMutation.serializer);
testSerializedSize(superRowRm, RowMutation.serializer);
testSerializedSize(standardRm, RowMutation.serializer);
testSerializedSize(superRm, RowMutation.serializer);
testSerializedSize(mixedRm, RowMutation.serializer);
}
@Test
public void testGetSliceFromSuperBasic() throws Throwable {
// tests slicing against data from one row spread across two sstables
final Table table = Table.open("Keyspace1");
final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Super1");
final String ROW = "row2";
RowMutation rm = new RowMutation("Keyspace1", ROW);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Super1");
SuperColumn sc = new SuperColumn("sc1".getBytes(), new LongType());
sc.addColumn(new Column(getBytes(1), "val1".getBytes(), 1L));
cf.addColumn(sc);
rm.add(cf);
rm.apply();
Runnable verify =
new WrappedRunnable() {
public void runMayThrow() throws Exception {
ColumnFamily cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Super1"),
ArrayUtils.EMPTY_BYTE_ARRAY,
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
10);
assertColumns(cf, "sc1");
assertEquals(
new String(cf.getColumn("sc1".getBytes()).getSubColumn(getBytes(1)).value()),
"val1");
}
};
reTest(table.getColumnFamilyStore("Standard1"), verify);
}
@Test
public void testCompactions() throws IOException, ExecutionException, InterruptedException {
CompactionManager.instance.disableAutoCompaction();
// this test does enough rows to force multiple block indexes to be used
Table table = Table.open(TABLE1);
ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
final int ROWS_PER_SSTABLE = 10;
Set<DecoratedKey> inserted = new HashSet<DecoratedKey>();
for (int j = 0; j < (DatabaseDescriptor.getIndexInterval() * 3) / ROWS_PER_SSTABLE; j++) {
for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
DecoratedKey key = Util.dk(String.valueOf(i % 2));
RowMutation rm = new RowMutation(TABLE1, key.key);
rm.add(
new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i / 2))),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
j * ROWS_PER_SSTABLE + i);
rm.apply();
inserted.add(key);
}
store.forceBlockingFlush();
assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(store).size());
}
while (true) {
Future<Integer> ft = CompactionManager.instance.submitMinorIfNeeded(store);
if (ft.get() == 0) break;
}
if (store.getSSTables().size() > 1) {
CompactionManager.instance.performMajor(store);
}
assertEquals(inserted.size(), Util.getRangeSlice(store).size());
}
@Test
public void testOne() throws IOException, ExecutionException, InterruptedException {
Table table1 = Table.open("Keyspace1");
Table table2 = Table.open("Keyspace2");
RowMutation rm;
DecoratedKey dk = Util.dk("keymulti");
ColumnFamily cf;
rm = new RowMutation("Keyspace1", dk.key);
cf = ColumnFamily.create("Keyspace1", "Standard1");
cf.addColumn(column("col1", "val1", 1L));
rm.add(cf);
rm.apply();
rm = new RowMutation("Keyspace2", dk.key);
cf = ColumnFamily.create("Keyspace2", "Standard3");
cf.addColumn(column("col2", "val2", 1L));
rm.add(cf);
rm.apply();
table1.getColumnFamilyStore("Standard1").clearUnsafe();
table2.getColumnFamilyStore("Standard3").clearUnsafe();
CommitLog.instance.resetUnsafe(); // disassociate segments from live CL
CommitLog.instance.recover();
assertColumns(Util.getColumnFamily(table1, dk, "Standard1"), "col1");
assertColumns(Util.getColumnFamily(table2, dk, "Standard3"), "col2");
}
@Test
public void testGetRowSliceByRange() throws Throwable {
String key = TEST_KEY + "slicerow";
Table table = Table.open("Keyspace1");
ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
RowMutation rm = new RowMutation("Keyspace1", key);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
// First write "a", "b", "c"
cf.addColumn(column("a", "val1", 1L));
cf.addColumn(column("b", "val2", 1L));
cf.addColumn(column("c", "val3", 1L));
rm.add(cf);
rm.apply();
cf =
cfStore.getColumnFamily(
key, new QueryPath("Standard1"), "b".getBytes(), "c".getBytes(), false, 100);
assertEquals(2, cf.getColumnCount());
cf =
cfStore.getColumnFamily(
key, new QueryPath("Standard1"), "b".getBytes(), "b".getBytes(), false, 100);
assertEquals(1, cf.getColumnCount());
cf =
cfStore.getColumnFamily(
key, new QueryPath("Standard1"), "b".getBytes(), "c".getBytes(), false, 1);
assertEquals(1, cf.getColumnCount());
cf =
cfStore.getColumnFamily(
key, new QueryPath("Standard1"), "c".getBytes(), "b".getBytes(), false, 1);
assertNull(cf);
}
@Test
public void testRecoverCounter() throws IOException, ExecutionException, InterruptedException {
Table table1 = Table.open("Keyspace1");
RowMutation rm;
DecoratedKey dk = Util.dk("key");
ColumnFamily cf;
for (int i = 0; i < 10; ++i) {
rm = new RowMutation("Keyspace1", dk.key);
cf = ColumnFamily.create("Keyspace1", "Counter1");
cf.addColumn(new CounterColumn(ByteBufferUtil.bytes("col"), 1L, 1L));
rm.add(cf);
rm.apply();
}
table1.getColumnFamilyStore("Counter1").clearUnsafe();
CommitLog.instance.resetUnsafe(); // disassociate segments from live CL
CommitLog.instance.recover();
cf = Util.getColumnFamily(table1, dk, "Counter1");
assert cf.getColumnCount() == 1;
Column c = cf.getColumn(ByteBufferUtil.bytes("col"));
assert c != null;
assert ((CounterColumn) c).total() == 10L;
}
@Test
public void testRemoveColumnFamily()
throws IOException, ExecutionException, InterruptedException {
Table table = Table.open("Keyspace1");
ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
RowMutation rm;
// add data
rm = new RowMutation("Keyspace1", "key1");
rm.add(new QueryPath("Standard1", null, "Column1".getBytes()), "asdf".getBytes(), 0);
rm.apply();
// remove
rm = new RowMutation("Keyspace1", "key1");
rm.delete(new QueryPath("Standard1"), 1);
rm.apply();
ColumnFamily retrieved =
store.getColumnFamily(
new IdentityQueryFilter(
"key1", new QueryPath("Standard1", null, "Column1".getBytes())));
assert retrieved.isMarkedForDelete();
assertNull(retrieved.getColumn("Column1".getBytes()));
assertNull(ColumnFamilyStore.removeDeleted(retrieved, Integer.MAX_VALUE));
}
@Test(timeout = 5000)
public void testTruncateHints() throws Exception {
Keyspace systemKeyspace = Keyspace.open("system");
ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS_CF);
hintStore.clearUnsafe();
// insert 1 hint
RowMutation rm = new RowMutation(KEYSPACE4, ByteBufferUtil.bytes(1));
rm.add(
STANDARD1_CF,
ByteBufferUtil.bytes(String.valueOf(COLUMN1)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
System.currentTimeMillis());
HintedHandOffManager.instance
.hintFor(
rm,
System.currentTimeMillis(),
HintedHandOffManager.calculateHintTTL(rm),
UUID.randomUUID())
.apply();
assert getNoOfHints() == 1;
HintedHandOffManager.instance.truncateAllHints();
while (getNoOfHints() > 0) {
Thread.sleep(100);
}
assert getNoOfHints() == 0;
}
@Test
public void testGetRowSingleColumn() throws Throwable {
final Table table = Table.open("Keyspace1");
final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
RowMutation rm = new RowMutation("Keyspace1", TEST_KEY);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
cf.addColumn(column("col1", "val1", 1L));
cf.addColumn(column("col2", "val2", 1L));
cf.addColumn(column("col3", "val3", 1L));
rm.add(cf);
rm.apply();
Runnable verify =
new WrappedRunnable() {
public void runMayThrow() throws Exception {
ColumnFamily cf;
cf =
cfStore.getColumnFamily(
new NamesQueryFilter(TEST_KEY, new QueryPath("Standard1"), "col1".getBytes()));
assertColumns(cf, "col1");
cf =
cfStore.getColumnFamily(
new NamesQueryFilter(TEST_KEY, new QueryPath("Standard1"), "col3".getBytes()));
assertColumns(cf, "col3");
}
};
reTest(table.getColumnFamilyStore("Standard1"), verify);
}
public RowMutation dropFromSchema(long timestamp) {
RowMutation rm = new RowMutation(Table.SYSTEM_TABLE, SystemTable.getSchemaKSKey(name));
rm.delete(new QueryPath(SystemTable.SCHEMA_KEYSPACES_CF), timestamp);
rm.delete(new QueryPath(SystemTable.SCHEMA_COLUMNFAMILIES_CF), timestamp);
rm.delete(new QueryPath(SystemTable.SCHEMA_COLUMNS_CF), timestamp);
return rm;
}
public void batch_insert(String table, BatchMutation batch_mutation, int consistency_level)
throws InvalidRequestException, UnavailableException {
if (logger.isDebugEnabled()) logger.debug("batch_insert");
RowMutation rm = RowMutation.getRowMutation(table, batch_mutation);
Set<String> cfNames = rm.columnFamilyNames();
ThriftValidation.validateKeyCommand(
rm.key(), rm.table(), cfNames.toArray(new String[cfNames.size()]));
doInsert(consistency_level, rm);
}
private static void insertRowWithKey(int key) {
long timestamp = System.currentTimeMillis();
DecoratedKey decoratedKey = Util.dk(String.format("%03d", key));
RowMutation rm = new RowMutation(KEYSPACE1, decoratedKey.key);
rm.add(
"Standard1",
ByteBufferUtil.bytes("col"),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
timestamp,
1000);
rm.apply();
}
public RowMutation toSchema(long timestamp) {
RowMutation rm = new RowMutation(Table.SYSTEM_TABLE, SystemTable.getSchemaKSKey(name));
ColumnFamily cf = rm.addOrGet(SystemTable.SCHEMA_KEYSPACES_CF);
cf.addColumn(Column.create(name, timestamp, "name"));
cf.addColumn(Column.create(durableWrites, timestamp, "durable_writes"));
cf.addColumn(Column.create(strategyClass.getName(), timestamp, "strategy_class"));
cf.addColumn(Column.create(json(strategyOptions), timestamp, "strategy_options"));
for (CFMetaData cfm : cfMetaData.values()) cfm.toSchema(rm, timestamp);
return rm;
}
private void testDontPurgeAccidentaly(String k, String cfname)
throws IOException, ExecutionException, InterruptedException {
// This test catches the regression of CASSANDRA-2786
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
// disable compaction while flushing
cfs.clearUnsafe();
cfs.disableAutoCompaction();
// Add test row
DecoratedKey key = Util.dk(k);
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
rm.add(
cfname,
CompositeType.build(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
0);
rm.apply();
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesBefore = cfs.getSSTables();
QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis());
assert !(cfs.getColumnFamily(filter).getColumnCount() == 0);
// Remove key
rm = new RowMutation(KEYSPACE1, key.key);
rm.delete(cfname, 2);
rm.apply();
ColumnFamily cf = cfs.getColumnFamily(filter);
assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;
// Sleep one second so that the removal is indeed purgeable even with gcgrace == 0
Thread.sleep(1000);
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesAfter = cfs.getSSTables();
Collection<SSTableReader> toCompact = new ArrayList<SSTableReader>();
for (SSTableReader sstable : sstablesAfter)
if (!sstablesBefore.contains(sstable)) toCompact.add(sstable);
Util.compact(cfs, toCompact);
cf = cfs.getColumnFamily(filter);
assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;
}
public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
store.clearUnsafe();
store.metadata.gcGraceSeconds(1);
store.setCompactionStrategyClass(strategyClassName);
// disable compaction while flushing
store.disableAutoCompaction();
long timestamp = System.currentTimeMillis();
for (int i = 0; i < 10; i++) {
DecoratedKey key = Util.dk(Integer.toString(i));
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
for (int j = 0; j < 10; j++)
rm.add(
"Standard1",
ByteBufferUtil.bytes(Integer.toString(j)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
timestamp,
j > 0
? 3
: 0); // let first column never expire, since deleting all columns does not produce
// sstable
rm.apply();
}
store.forceBlockingFlush();
assertEquals(1, store.getSSTables().size());
long originalSize = store.getSSTables().iterator().next().uncompressedLength();
// wait enough to force single compaction
TimeUnit.SECONDS.sleep(5);
// enable compaction, submit background and wait for it to complete
store.enableAutoCompaction();
FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
while (CompactionManager.instance.getPendingTasks() > 0
|| CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);
// and sstable with ttl should be compacted
assertEquals(1, store.getSSTables().size());
long size = store.getSSTables().iterator().next().uncompressedLength();
assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize);
// make sure max timestamp of compacted sstables is recorded properly after compaction.
assertMaxTimestamp(store, timestamp);
return store;
}
@Test
public void testGetSliceFromAdvanced() throws Throwable {
// tests slicing against data from one row spread across two sstables
final Table table = Table.open("Keyspace1");
final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
final String ROW = "row2";
RowMutation rm = new RowMutation("Keyspace1", ROW);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
cf.addColumn(column("col1", "val1", 1L));
cf.addColumn(column("col2", "val2", 1L));
cf.addColumn(column("col3", "val3", 1L));
cf.addColumn(column("col4", "val4", 1L));
cf.addColumn(column("col5", "val5", 1L));
cf.addColumn(column("col6", "val6", 1L));
rm.add(cf);
rm.apply();
cfStore.forceBlockingFlush();
rm = new RowMutation("Keyspace1", ROW);
cf = ColumnFamily.create("Keyspace1", "Standard1");
cf.addColumn(column("col1", "valx", 2L));
cf.addColumn(column("col2", "valx", 2L));
cf.addColumn(column("col3", "valx", 2L));
rm.add(cf);
rm.apply();
Runnable verify =
new WrappedRunnable() {
public void runMayThrow() throws Exception {
ColumnFamily cf;
cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Standard1"),
"col2".getBytes(),
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
3);
assertColumns(cf, "col2", "col3", "col4");
assertEquals(new String(cf.getColumn("col2".getBytes()).value()), "valx");
assertEquals(new String(cf.getColumn("col3".getBytes()).value()), "valx");
assertEquals(new String(cf.getColumn("col4".getBytes()).value()), "val4");
}
};
reTest(table.getColumnFamilyStore("Standard1"), verify);
}
public void remove(
String table,
String key,
ColumnPathOrParent column_path_or_parent,
long timestamp,
int consistency_level)
throws InvalidRequestException, UnavailableException {
if (logger.isDebugEnabled()) logger.debug("remove");
ThriftValidation.validateColumnPathOrParent(table, column_path_or_parent);
RowMutation rm = new RowMutation(table, key.trim());
rm.delete(new QueryPath(column_path_or_parent), timestamp);
doInsert(consistency_level, rm);
}
protected void fillCF(ColumnFamilyStore cfs, int rowsPerSSTable)
throws ExecutionException, InterruptedException, IOException {
CompactionManager.instance.disableAutoCompaction();
for (int i = 0; i < rowsPerSSTable; i++) {
String key = String.valueOf(i);
// create a row and update the birthdate value, test that the index query fetches the new
// version
RowMutation rm;
rm = new RowMutation(KEYSPACE1, ByteBufferUtil.bytes(key));
rm.add(cfs.name, COLUMN, VALUE, System.currentTimeMillis());
rm.applyUnsafe();
}
cfs.forceBlockingFlush();
}
@Test
public void testGetColumn() throws IOException, ColumnFamilyNotDefinedException {
Table table = Table.open("Keyspace1");
RowMutation rm;
// add data
rm = new RowMutation("Keyspace1", "key1");
rm.add(new QueryPath("Standard1", null, "Column1".getBytes()), "abcd".getBytes(), 0);
rm.apply();
ReadCommand command =
new SliceByNamesReadCommand(
"Keyspace1", "key1", new QueryPath("Standard1"), Arrays.asList("Column1".getBytes()));
Row row = command.getRow(table);
IColumn col = row.cf.getColumn("Column1".getBytes());
assert Arrays.equals(col.value(), "abcd".getBytes());
}
@Test
public void overlappingRangeTest() throws Exception {
CompactionManager.instance.disableAutoCompaction();
Table table = Table.open(KSNAME);
ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME);
// Inserting data
String key = "k2";
RowMutation rm;
ColumnFamily cf;
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
for (int i = 0; i < 20; i++) add(rm, i, 0);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
cf = rm.addOrGet(CFNAME);
delete(cf, 5, 15, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
cf = rm.addOrGet(CFNAME);
delete(cf, 5, 10, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
cf = rm.addOrGet(CFNAME);
delete(cf, 5, 8, 2);
rm.apply();
cfs.forceBlockingFlush();
QueryPath path = new QueryPath(CFNAME);
cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(key), path));
for (int i = 0; i < 5; i++)
assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
for (int i = 16; i < 20; i++)
assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
for (int i = 5; i <= 15; i++)
assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live";
// Compact everything and re-test
CompactionManager.instance.performMaximal(cfs);
cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(key), path));
for (int i = 0; i < 5; i++)
assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
for (int i = 16; i < 20; i++)
assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
for (int i = 5; i <= 15; i++)
assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live";
}
@Test
public void testGetSliceNoMatch() throws Throwable {
Table table = Table.open("Keyspace1");
RowMutation rm = new RowMutation("Keyspace1", "row1000");
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard2");
cf.addColumn(column("col1", "val1", 1));
rm.add(cf);
rm.apply();
validateGetSliceNoMatch(table);
table.getColumnFamilyStore("Standard2").forceBlockingFlush();
validateGetSliceNoMatch(table);
Collection<SSTableReader> ssTables = table.getColumnFamilyStore("Standard2").getSSTables();
assertEquals(1, ssTables.size());
ssTables.iterator().next().forceBloomFilterFailures();
validateGetSliceNoMatch(table);
}
@Test
public void testSuperColumnTombstones()
throws IOException, ExecutionException, InterruptedException {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1");
cfs.disableAutoCompaction();
DecoratedKey key = Util.dk("tskey");
ByteBuffer scName = ByteBufferUtil.bytes("TestSuperColumn");
// a subcolumn
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
rm.add(
"Super1",
CompositeType.build(scName, ByteBufferUtil.bytes(0)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
FBUtilities.timestampMicros());
rm.apply();
cfs.forceBlockingFlush();
// shadow the subcolumn with a supercolumn tombstone
rm = new RowMutation(KEYSPACE1, key.key);
rm.deleteRange(
"Super1",
SuperColumns.startOf(scName),
SuperColumns.endOf(scName),
FBUtilities.timestampMicros());
rm.apply();
cfs.forceBlockingFlush();
CompactionManager.instance.performMaximal(cfs);
assertEquals(1, cfs.getSSTables().size());
// check that the shadowed column is gone
SSTableReader sstable = cfs.getSSTables().iterator().next();
Range keyRange =
new Range<RowPosition>(key, sstable.partitioner.getMinimumToken().maxKeyBound());
SSTableScanner scanner = sstable.getScanner(DataRange.forKeyRange(keyRange));
OnDiskAtomIterator iter = scanner.next();
assertEquals(key, iter.getKey());
assert iter.next() instanceof RangeTombstone;
assert !iter.hasNext();
}
/*
* This excercise in particular the code of #4142
*/
@Test
public void testValidationMultipleSSTablePerLevel() throws Exception {
String ksname = "Keyspace1";
String cfname = "StandardLeveled";
Table table = Table.open(ksname);
ColumnFamilyStore store = table.getColumnFamilyStore(cfname);
ByteBuffer value =
ByteBuffer.wrap(new byte[100 * 1024]); // 100 KB value, make it easy to have multiple files
// Enough data to have a level 1 and 2
int rows = 20;
int columns = 10;
// Adds enough data to trigger multiple sstable per level
for (int r = 0; r < rows; r++) {
DecoratedKey key = Util.dk(String.valueOf(r));
RowMutation rm = new RowMutation(ksname, key.key);
for (int c = 0; c < columns; c++) {
rm.add(new QueryPath(cfname, null, ByteBufferUtil.bytes("column" + c)), value, 0);
}
rm.apply();
store.forceFlush();
}
LeveledCompactionStrategy strat = (LeveledCompactionStrategy) store.getCompactionStrategy();
while (strat.getLevelSize(0) > 0) {
store.forceMajorCompaction();
Thread.sleep(200);
}
// Checking we're not completely bad at math
assert strat.getLevelSize(1) > 0;
assert strat.getLevelSize(2) > 0;
AntiEntropyService.CFPair p = new AntiEntropyService.CFPair(ksname, cfname);
Range<Token> range = new Range<Token>(Util.token(""), Util.token(""));
AntiEntropyService.TreeRequest req =
new AntiEntropyService.TreeRequest("1", FBUtilities.getLocalAddress(), range, p);
AntiEntropyService.Validator validator = new AntiEntropyService.Validator(req);
CompactionManager.instance.submitValidation(store, validator).get();
}
public void insert(
String table,
String key,
ColumnPath column_path,
byte[] value,
long timestamp,
int consistency_level)
throws InvalidRequestException, UnavailableException {
if (logger.isDebugEnabled()) logger.debug("insert");
ThriftValidation.validateKey(key);
ThriftValidation.validateColumnPath(table, column_path);
RowMutation rm = new RowMutation(table, key.trim());
try {
rm.add(new QueryPath(column_path), value, timestamp);
} catch (MarshalException e) {
throw new InvalidRequestException(e.getMessage());
}
doInsert(consistency_level, rm);
}
@Test
public void testGetRowNoColumns() throws Throwable {
final Table table = Table.open("Keyspace2");
final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard3");
RowMutation rm = new RowMutation("Keyspace2", TEST_KEY);
ColumnFamily cf = ColumnFamily.create("Keyspace2", "Standard3");
cf.addColumn(column("col1", "val1", 1L));
rm.add(cf);
rm.apply();
Runnable verify =
new WrappedRunnable() {
public void runMayThrow() throws Exception {
ColumnFamily cf;
cf =
cfStore.getColumnFamily(
new NamesQueryFilter(
TEST_KEY, new QueryPath("Standard3"), new TreeSet<byte[]>()));
assertColumns(cf);
cf =
cfStore.getColumnFamily(
new SliceQueryFilter(
TEST_KEY,
new QueryPath("Standard3"),
ArrayUtils.EMPTY_BYTE_ARRAY,
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
0));
assertColumns(cf);
cf =
cfStore.getColumnFamily(
new NamesQueryFilter(TEST_KEY, new QueryPath("Standard3"), "col99".getBytes()));
assertColumns(cf);
}
};
reTest(table.getColumnFamilyStore("Standard3"), verify);
}
@Test
public void simpleQueryWithRangeTombstoneTest() throws Exception {
Table table = Table.open(KSNAME);
ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME);
// Inserting data
String key = "k1";
RowMutation rm;
ColumnFamily cf;
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
for (int i = 0; i < 40; i += 2) add(rm, i, 0);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
cf = rm.addOrGet(CFNAME);
delete(cf, 10, 22, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
for (int i = 1; i < 40; i += 2) add(rm, i, 2);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
cf = rm.addOrGet(CFNAME);
delete(cf, 19, 27, 3);
rm.apply();
// We don't flush to test with both a range tomsbtone in memtable and in sstable
QueryPath path = new QueryPath(CFNAME);
// Queries by name
int[] live = new int[] {4, 9, 11, 17, 28};
int[] dead = new int[] {12, 19, 21, 24, 27};
SortedSet<ByteBuffer> columns = new TreeSet<ByteBuffer>(cfs.getComparator());
for (int i : live) columns.add(b(i));
for (int i : dead) columns.add(b(i));
cf = cfs.getColumnFamily(QueryFilter.getNamesFilter(dk(key), path, columns));
for (int i : live) assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
for (int i : dead)
assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live";
// Queries by slices
cf =
cfs.getColumnFamily(
QueryFilter.getSliceFilter(dk(key), path, b(7), b(30), false, Integer.MAX_VALUE));
for (int i : new int[] {7, 8, 9, 11, 13, 15, 17, 28, 29, 30})
assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
for (int i : new int[] {10, 12, 14, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27})
assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live";
}
@Test
public void testUserDefinedCompaction() throws Exception {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
final String cfname = "Standard3"; // use clean(no sstable) CF
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
// disable compaction while flushing
cfs.disableAutoCompaction();
final int ROWS_PER_SSTABLE = 10;
for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
DecoratedKey key = Util.dk(String.valueOf(i));
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
rm.add(
cfname,
ByteBufferUtil.bytes("col"),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
System.currentTimeMillis());
rm.apply();
}
cfs.forceBlockingFlush();
Collection<SSTableReader> sstables = cfs.getSSTables();
assert sstables.size() == 1;
SSTableReader sstable = sstables.iterator().next();
int prevGeneration = sstable.descriptor.generation;
String file = new File(sstable.descriptor.filenameFor(Component.DATA)).getName();
// submit user defined compaction on flushed sstable
CompactionManager.instance.forceUserDefinedCompaction(file);
// wait until user defined compaction finishes
do {
Thread.sleep(100);
} while (CompactionManager.instance.getPendingTasks() > 0
|| CompactionManager.instance.getActiveCompactions() > 0);
// CF should have only one sstable with generation number advanced
sstables = cfs.getSSTables();
assert sstables.size() == 1;
assert sstables.iterator().next().descriptor.generation == prevGeneration + 1;
}
