public static void spinAssertEquals(Object expected, Supplier<Object> s, int timeoutInSeconds) {
long now = System.currentTimeMillis();
while (System.currentTimeMillis() - now < now + (1000 * timeoutInSeconds)) {
if (s.get().equals(expected)) break;
Thread.yield();
}
assertEquals(expected, s.get());
}
private static void insertRowWithKey(int key) {
long timestamp = System.currentTimeMillis();
DecoratedKey decoratedKey = Util.dk(String.format("%03d", key));
Mutation rm = new Mutation(KEYSPACE1, decoratedKey.getKey());
rm.add("CF_STANDARD1", Util.cellname("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1000);
rm.applyUnsafe();
}
public static byte[] concatByteArrays(byte[] first, byte[]... remaining) {
int length = first.length;
for (byte[] array : remaining) {
length += array.length;
}
byte[] result = new byte[length];
System.arraycopy(first, 0, result, 0, first.length);
int offset = first.length;
for (byte[] array : remaining) {
System.arraycopy(array, 0, result, offset, array.length);
offset += array.length;
}
return result;
}
private static void insertRowWithKey(int key) {
long timestamp = System.currentTimeMillis();
DecoratedKey decoratedKey = Util.dk(String.format("%03d", key));
RowMutation rm = new RowMutation(KEYSPACE1, decoratedKey.key);
rm.add(
"Standard1",
ByteBufferUtil.bytes("col"),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
timestamp,
1000);
rm.apply();
}
private void testDontPurgeAccidentaly(String k, String cfname)
throws IOException, ExecutionException, InterruptedException {
// This test catches the regression of CASSANDRA-2786
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
// disable compaction while flushing
cfs.clearUnsafe();
cfs.disableAutoCompaction();
// Add test row
DecoratedKey key = Util.dk(k);
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
rm.add(
cfname,
CompositeType.build(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
0);
rm.apply();
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesBefore = cfs.getSSTables();
QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis());
assert !(cfs.getColumnFamily(filter).getColumnCount() == 0);
// Remove key
rm = new RowMutation(KEYSPACE1, key.key);
rm.delete(cfname, 2);
rm.apply();
ColumnFamily cf = cfs.getColumnFamily(filter);
assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;
// Sleep one second so that the removal is indeed purgeable even with gcgrace == 0
Thread.sleep(1000);
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesAfter = cfs.getSSTables();
Collection<SSTableReader> toCompact = new ArrayList<SSTableReader>();
for (SSTableReader sstable : sstablesAfter)
if (!sstablesBefore.contains(sstable)) toCompact.add(sstable);
Util.compact(cfs, toCompact);
cf = cfs.getColumnFamily(filter);
assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;
}
public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
store.clearUnsafe();
store.metadata.gcGraceSeconds(1);
store.setCompactionStrategyClass(strategyClassName);
// disable compaction while flushing
store.disableAutoCompaction();
long timestamp = System.currentTimeMillis();
for (int i = 0; i < 10; i++) {
DecoratedKey key = Util.dk(Integer.toString(i));
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
for (int j = 0; j < 10; j++)
rm.add(
"Standard1",
ByteBufferUtil.bytes(Integer.toString(j)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
timestamp,
j > 0
? 3
: 0); // let first column never expire, since deleting all columns does not produce
// sstable
rm.apply();
}
store.forceBlockingFlush();
assertEquals(1, store.getSSTables().size());
long originalSize = store.getSSTables().iterator().next().uncompressedLength();
// wait enough to force single compaction
TimeUnit.SECONDS.sleep(5);
// enable compaction, submit background and wait for it to complete
store.enableAutoCompaction();
FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
while (CompactionManager.instance.getPendingTasks() > 0
|| CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);
// and sstable with ttl should be compacted
assertEquals(1, store.getSSTables().size());
long size = store.getSSTables().iterator().next().uncompressedLength();
assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize);
// make sure max timestamp of compacted sstables is recorded properly after compaction.
assertMaxTimestamp(store, timestamp);
return store;
}
private long populate(String ks, String cf, int startRowKey, int endRowKey, int ttl) {
long timestamp = System.currentTimeMillis();
for (int i = startRowKey; i <= endRowKey; i++) {
DecoratedKey key = Util.dk(Integer.toString(i));
Mutation rm = new Mutation(ks, key.getKey());
for (int j = 0; j < 10; j++)
rm.add(
cf,
Util.cellname(Integer.toString(j)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
timestamp,
j > 0
? ttl
: 0); // let first column never expire, since deleting all columns does not produce
// sstable
rm.applyUnsafe();
}
return timestamp;
}
@Test
public void testUserDefinedCompaction() throws Exception {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
final String cfname = "Standard3"; // use clean(no sstable) CF
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
// disable compaction while flushing
cfs.disableAutoCompaction();
final int ROWS_PER_SSTABLE = 10;
for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
DecoratedKey key = Util.dk(String.valueOf(i));
Mutation rm = new Mutation(KEYSPACE1, key.getKey());
rm.add(
cfname,
Util.cellname("col"),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
System.currentTimeMillis());
rm.applyUnsafe();
}
cfs.forceBlockingFlush();
Collection<SSTableReader> sstables = cfs.getSSTables();
assertEquals(1, sstables.size());
SSTableReader sstable = sstables.iterator().next();
int prevGeneration = sstable.descriptor.generation;
String file = new File(sstable.descriptor.filenameFor(Component.DATA)).getAbsolutePath();
// submit user defined compaction on flushed sstable
CompactionManager.instance.forceUserDefinedCompaction(file);
// wait until user defined compaction finishes
do {
Thread.sleep(100);
} while (CompactionManager.instance.getPendingTasks() > 0
|| CompactionManager.instance.getActiveCompactions() > 0);
// CF should have only one sstable with generation number advanced
sstables = cfs.getSSTables();
assertEquals(1, sstables.size());
assertEquals(prevGeneration + 1, sstables.iterator().next().descriptor.generation);
}
public static QueryFilter namesQueryFilter(
ColumnFamilyStore cfs, DecoratedKey key, CellName... names) {
SortedSet<CellName> s = new TreeSet<CellName>(cfs.getComparator());
for (CellName n : names) s.add(n);
return QueryFilter.getNamesFilter(key, cfs.name, s, System.currentTimeMillis());
}
public static void compact(ColumnFamilyStore cfs, Collection<SSTableReader> sstables) {
int gcBefore = cfs.gcBefore(System.currentTimeMillis());
AbstractCompactionTask task =
cfs.getCompactionStrategy().getUserDefinedTask(sstables, gcBefore);
task.execute(null);
}
public static ColumnFamily getColumnFamily(Keyspace keyspace, DecoratedKey key, String cfName) {
ColumnFamilyStore cfStore = keyspace.getColumnFamilyStore(cfName);
assert cfStore != null : "Table " + cfName + " has not been defined";
return cfStore.getColumnFamily(
QueryFilter.getIdentityFilter(key, cfName, System.currentTimeMillis()));
}
@Test
public void testRangeTombstones() throws IOException, ExecutionException, InterruptedException {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2");
cfs.clearUnsafe();
// disable compaction while flushing
cfs.disableAutoCompaction();
final CFMetaData cfmeta = cfs.metadata;
Directories dir = cfs.directories;
ArrayList<DecoratedKey> keys = new ArrayList<DecoratedKey>();
for (int i = 0; i < 4; i++) {
keys.add(Util.dk("" + i));
}
ArrayBackedSortedColumns cf = ArrayBackedSortedColumns.factory.create(cfmeta);
cf.addColumn(Util.column("01", "a", 1)); // this must not resurrect
cf.addColumn(Util.column("a", "a", 3));
cf.deletionInfo()
.add(
new RangeTombstone(
Util.cellname("0"),
Util.cellname("b"),
2,
(int) (System.currentTimeMillis() / 1000)),
cfmeta.comparator);
SSTableWriter writer =
new SSTableWriter(
cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables()),
0,
0,
cfs.metadata,
StorageService.getPartitioner(),
new MetadataCollector(cfs.metadata.comparator));
writer.append(Util.dk("0"), cf);
writer.append(Util.dk("1"), cf);
writer.append(Util.dk("3"), cf);
cfs.addSSTable(writer.closeAndOpenReader());
writer =
new SSTableWriter(
cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables()),
0,
0,
cfs.metadata,
StorageService.getPartitioner(),
new MetadataCollector(cfs.metadata.comparator));
writer.append(Util.dk("0"), cf);
writer.append(Util.dk("1"), cf);
writer.append(Util.dk("2"), cf);
writer.append(Util.dk("3"), cf);
cfs.addSSTable(writer.closeAndOpenReader());
Collection<SSTableReader> toCompact = cfs.getSSTables();
assert toCompact.size() == 2;
// Force compaction on first sstables. Since each row is in only one sstable, we will be using
// EchoedRow.
Util.compact(cfs, toCompact);
assertEquals(1, cfs.getSSTables().size());
// Now assert we do have the 4 keys
assertEquals(4, Util.getRangeSlice(cfs).size());
ArrayList<DecoratedKey> k = new ArrayList<DecoratedKey>();
for (Row r : Util.getRangeSlice(cfs)) {
k.add(r.key);
assertEquals(ByteBufferUtil.bytes("a"), r.cf.getColumn(Util.cellname("a")).value());
assertNull(r.cf.getColumn(Util.cellname("01")));
assertEquals(3, r.cf.getColumn(Util.cellname("a")).timestamp());
}
for (SSTableReader sstable : cfs.getSSTables()) {
StatsMetadata stats = sstable.getSSTableMetadata();
assertEquals(ByteBufferUtil.bytes("0"), stats.minColumnNames.get(0));
assertEquals(ByteBufferUtil.bytes("b"), stats.maxColumnNames.get(0));
}
assertEquals(keys, k);
}