public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1); store.clearUnsafe(); store.metadata.gcGraceSeconds(1); store.setCompactionStrategyClass(strategyClassName); // disable compaction while flushing store.disableAutoCompaction(); long timestamp = populate(KEYSPACE1, CF_STANDARD1, 0, 9, 3); // ttl=3s store.forceBlockingFlush(); assertEquals(1, store.getSSTables().size()); long originalSize = store.getSSTables().iterator().next().uncompressedLength(); // wait enough to force single compaction TimeUnit.SECONDS.sleep(5); // enable compaction, submit background and wait for it to complete store.enableAutoCompaction(); FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store)); while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1); // and sstable with ttl should be compacted assertEquals(1, store.getSSTables().size()); long size = store.getSSTables().iterator().next().uncompressedLength(); assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize); // make sure max timestamp of compacted sstables is recorded properly after compaction. assertMaxTimestamp(store, timestamp); return store; }
public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1"); store.clearUnsafe(); store.metadata.gcGraceSeconds(1); store.setCompactionStrategyClass(strategyClassName); // disable compaction while flushing store.disableAutoCompaction(); long timestamp = System.currentTimeMillis(); for (int i = 0; i < 10; i++) { DecoratedKey key = Util.dk(Integer.toString(i)); RowMutation rm = new RowMutation(KEYSPACE1, key.key); for (int j = 0; j < 10; j++) rm.add( "Standard1", ByteBufferUtil.bytes(Integer.toString(j)), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, j > 0 ? 3 : 0); // let first column never expire, since deleting all columns does not produce // sstable rm.apply(); } store.forceBlockingFlush(); assertEquals(1, store.getSSTables().size()); long originalSize = store.getSSTables().iterator().next().uncompressedLength(); // wait enough to force single compaction TimeUnit.SECONDS.sleep(5); // enable compaction, submit background and wait for it to complete store.enableAutoCompaction(); FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store)); while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1); // and sstable with ttl should be compacted assertEquals(1, store.getSSTables().size()); long size = store.getSSTables().iterator().next().uncompressedLength(); assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize); // make sure max timestamp of compacted sstables is recorded properly after compaction. assertMaxTimestamp(store, timestamp); return store; }
@Test public void testNoExpire() throws ExecutionException, InterruptedException { ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1"); cfs.disableAutoCompaction(); cfs.metadata.gcGraceSeconds(0); long timestamp = System.currentTimeMillis(); RowMutation rm = new RowMutation("Keyspace1", Util.dk("ttl").key); rm.add( "Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1); rm.add( "Standard1", ByteBufferUtil.bytes("col7"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation("Keyspace1", Util.dk("ttl").key); rm.add( "Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation("Keyspace1", Util.dk("ttl").key); rm.add( "Standard1", ByteBufferUtil.bytes("col3"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1); rm.apply(); cfs.forceBlockingFlush(); DecoratedKey noTTLKey = Util.dk("nottl"); rm = new RowMutation("Keyspace1", noTTLKey.key); rm.add( "Standard1", ByteBufferUtil.bytes("col311"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp); rm.apply(); cfs.forceBlockingFlush(); Thread.sleep(2000); // wait for ttl to expire assertEquals(4, cfs.getSSTables().size()); cfs.enableAutoCompaction(true); assertEquals(1, cfs.getSSTables().size()); SSTableReader sstable = cfs.getSSTables().iterator().next(); SSTableScanner scanner = sstable.getScanner(DataRange.allData(sstable.partitioner)); assertTrue(scanner.hasNext()); while (scanner.hasNext()) { OnDiskAtomIterator iter = scanner.next(); assertEquals(noTTLKey, iter.getKey()); } }
@Test public void testSimpleExpire() throws ExecutionException, InterruptedException { ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1"); cfs.disableAutoCompaction(); cfs.metadata.gcGraceSeconds(0); long timestamp = System.currentTimeMillis(); RowMutation rm = new RowMutation("Keyspace1", Util.dk("ttl").key); rm.add( "Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1); rm.add( "Standard1", ByteBufferUtil.bytes("col7"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation("Keyspace1", Util.dk("ttl").key); rm.add( "Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation("Keyspace1", Util.dk("ttl").key); rm.add( "Standard1", ByteBufferUtil.bytes("col3"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation("Keyspace1", Util.dk("ttl").key); rm.add( "Standard1", ByteBufferUtil.bytes("col311"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1); rm.apply(); cfs.forceBlockingFlush(); Thread.sleep(2000); // wait for ttl to expire assertEquals(4, cfs.getSSTables().size()); cfs.enableAutoCompaction(true); assertEquals(0, cfs.getSSTables().size()); }
@Test public void testUncheckedTombstoneSizeTieredCompaction() throws Exception { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1); store.clearUnsafe(); store.metadata.gcGraceSeconds(1); store.metadata.compactionStrategyOptions.put("tombstone_compaction_interval", "1"); store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "false"); store.reload(); store.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getName()); // disable compaction while flushing store.disableAutoCompaction(); // Populate sstable1 with with keys [0..9] populate(KEYSPACE1, CF_STANDARD1, 0, 9, 3); // ttl=3s store.forceBlockingFlush(); // Populate sstable2 with with keys [10..19] (keys do not overlap with SSTable1) long timestamp2 = populate(KEYSPACE1, CF_STANDARD1, 10, 19, 3); // ttl=3s store.forceBlockingFlush(); assertEquals(2, store.getSSTables().size()); Iterator<SSTableReader> it = store.getSSTables().iterator(); long originalSize1 = it.next().uncompressedLength(); long originalSize2 = it.next().uncompressedLength(); // wait enough to force single compaction TimeUnit.SECONDS.sleep(5); // enable compaction, submit background and wait for it to complete store.enableAutoCompaction(); FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store)); while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1); // even though both sstables were candidate for tombstone compaction // it was not executed because they have an overlapping token range assertEquals(2, store.getSSTables().size()); it = store.getSSTables().iterator(); long newSize1 = it.next().uncompressedLength(); long newSize2 = it.next().uncompressedLength(); assertEquals( "candidate sstable should not be tombstone-compacted because its key range overlap with other sstable", originalSize1, newSize1); assertEquals( "candidate sstable should not be tombstone-compacted because its key range overlap with other sstable", originalSize2, newSize2); // now let's enable the magic property store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "true"); store.reload(); // submit background task again and wait for it to complete FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store)); while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1); // we still have 2 sstables, since they were not compacted against each other assertEquals(2, store.getSSTables().size()); it = store.getSSTables().iterator(); newSize1 = it.next().uncompressedLength(); newSize2 = it.next().uncompressedLength(); assertTrue( "should be less than " + originalSize1 + ", but was " + newSize1, newSize1 < originalSize1); assertTrue( "should be less than " + originalSize2 + ", but was " + newSize2, newSize2 < originalSize2); // make sure max timestamp of compacted sstables is recorded properly after compaction. assertMaxTimestamp(store, timestamp2); }