public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1);
store.clearUnsafe();
store.metadata.gcGraceSeconds(1);
store.setCompactionStrategyClass(strategyClassName);
// disable compaction while flushing
store.disableAutoCompaction();
long timestamp = populate(KEYSPACE1, CF_STANDARD1, 0, 9, 3); // ttl=3s
store.forceBlockingFlush();
assertEquals(1, store.getSSTables().size());
long originalSize = store.getSSTables().iterator().next().uncompressedLength();
// wait enough to force single compaction
TimeUnit.SECONDS.sleep(5);
// enable compaction, submit background and wait for it to complete
store.enableAutoCompaction();
FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
while (CompactionManager.instance.getPendingTasks() > 0
|| CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);
// and sstable with ttl should be compacted
assertEquals(1, store.getSSTables().size());
long size = store.getSSTables().iterator().next().uncompressedLength();
assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize);
// make sure max timestamp of compacted sstables is recorded properly after compaction.
assertMaxTimestamp(store, timestamp);
return store;
}
public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
store.clearUnsafe();
store.metadata.gcGraceSeconds(1);
store.setCompactionStrategyClass(strategyClassName);
// disable compaction while flushing
store.disableAutoCompaction();
long timestamp = System.currentTimeMillis();
for (int i = 0; i < 10; i++) {
DecoratedKey key = Util.dk(Integer.toString(i));
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
for (int j = 0; j < 10; j++)
rm.add(
"Standard1",
ByteBufferUtil.bytes(Integer.toString(j)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
timestamp,
j > 0
? 3
: 0); // let first column never expire, since deleting all columns does not produce
// sstable
rm.apply();
}
store.forceBlockingFlush();
assertEquals(1, store.getSSTables().size());
long originalSize = store.getSSTables().iterator().next().uncompressedLength();
// wait enough to force single compaction
TimeUnit.SECONDS.sleep(5);
// enable compaction, submit background and wait for it to complete
store.enableAutoCompaction();
FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
while (CompactionManager.instance.getPendingTasks() > 0
|| CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);
// and sstable with ttl should be compacted
assertEquals(1, store.getSSTables().size());
long size = store.getSSTables().iterator().next().uncompressedLength();
assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize);
// make sure max timestamp of compacted sstables is recorded properly after compaction.
assertMaxTimestamp(store, timestamp);
return store;
}
@Test
public void testNoExpire() throws ExecutionException, InterruptedException {
ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
cfs.disableAutoCompaction();
cfs.metadata.gcGraceSeconds(0);
long timestamp = System.currentTimeMillis();
RowMutation rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.add(
"Standard1", ByteBufferUtil.bytes("col7"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col3"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
DecoratedKey noTTLKey = Util.dk("nottl");
rm = new RowMutation("Keyspace1", noTTLKey.key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col311"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp);
rm.apply();
cfs.forceBlockingFlush();
Thread.sleep(2000); // wait for ttl to expire
assertEquals(4, cfs.getSSTables().size());
cfs.enableAutoCompaction(true);
assertEquals(1, cfs.getSSTables().size());
SSTableReader sstable = cfs.getSSTables().iterator().next();
SSTableScanner scanner = sstable.getScanner(DataRange.allData(sstable.partitioner));
assertTrue(scanner.hasNext());
while (scanner.hasNext()) {
OnDiskAtomIterator iter = scanner.next();
assertEquals(noTTLKey, iter.getKey());
}
}
@Test
public void testSimpleExpire() throws ExecutionException, InterruptedException {
ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
cfs.disableAutoCompaction();
cfs.metadata.gcGraceSeconds(0);
long timestamp = System.currentTimeMillis();
RowMutation rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.add(
"Standard1", ByteBufferUtil.bytes("col7"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col3"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1",
ByteBufferUtil.bytes("col311"),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
timestamp,
1);
rm.apply();
cfs.forceBlockingFlush();
Thread.sleep(2000); // wait for ttl to expire
assertEquals(4, cfs.getSSTables().size());
cfs.enableAutoCompaction(true);
assertEquals(0, cfs.getSSTables().size());
}
@Test
public void testUncheckedTombstoneSizeTieredCompaction() throws Exception {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1);
store.clearUnsafe();
store.metadata.gcGraceSeconds(1);
store.metadata.compactionStrategyOptions.put("tombstone_compaction_interval", "1");
store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "false");
store.reload();
store.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getName());
// disable compaction while flushing
store.disableAutoCompaction();
// Populate sstable1 with with keys [0..9]
populate(KEYSPACE1, CF_STANDARD1, 0, 9, 3); // ttl=3s
store.forceBlockingFlush();
// Populate sstable2 with with keys [10..19] (keys do not overlap with SSTable1)
long timestamp2 = populate(KEYSPACE1, CF_STANDARD1, 10, 19, 3); // ttl=3s
store.forceBlockingFlush();
assertEquals(2, store.getSSTables().size());
Iterator<SSTableReader> it = store.getSSTables().iterator();
long originalSize1 = it.next().uncompressedLength();
long originalSize2 = it.next().uncompressedLength();
// wait enough to force single compaction
TimeUnit.SECONDS.sleep(5);
// enable compaction, submit background and wait for it to complete
store.enableAutoCompaction();
FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
while (CompactionManager.instance.getPendingTasks() > 0
|| CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);
// even though both sstables were candidate for tombstone compaction
// it was not executed because they have an overlapping token range
assertEquals(2, store.getSSTables().size());
it = store.getSSTables().iterator();
long newSize1 = it.next().uncompressedLength();
long newSize2 = it.next().uncompressedLength();
assertEquals(
"candidate sstable should not be tombstone-compacted because its key range overlap with other sstable",
originalSize1,
newSize1);
assertEquals(
"candidate sstable should not be tombstone-compacted because its key range overlap with other sstable",
originalSize2,
newSize2);
// now let's enable the magic property
store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "true");
store.reload();
// submit background task again and wait for it to complete
FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
while (CompactionManager.instance.getPendingTasks() > 0
|| CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);
// we still have 2 sstables, since they were not compacted against each other
assertEquals(2, store.getSSTables().size());
it = store.getSSTables().iterator();
newSize1 = it.next().uncompressedLength();
newSize2 = it.next().uncompressedLength();
assertTrue(
"should be less than " + originalSize1 + ", but was " + newSize1, newSize1 < originalSize1);
assertTrue(
"should be less than " + originalSize2 + ", but was " + newSize2, newSize2 < originalSize2);
// make sure max timestamp of compacted sstables is recorded properly after compaction.
assertMaxTimestamp(store, timestamp2);
}