@Test
public void testCleanupWithNewToken()
throws ExecutionException, InterruptedException, UnknownHostException {
StorageService.instance.getTokenMetadata().clearUnsafe();
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF2);
List<Row> rows;
// insert data and verify we get it back w/ range query
fillCF(cfs, LOOPS);
rows = Util.getRangeSlice(cfs);
assertEquals(LOOPS, rows.size());
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
byte[] tk1 = new byte[1], tk2 = new byte[1];
tk1[0] = 2;
tk2[0] = 1;
tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1"));
tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2"));
CompactionManager.instance.performCleanup(cfs);
rows = Util.getRangeSlice(cfs);
assertEquals(0, rows.size());
}
@Test
public void testRemoveColumn() throws IOException, ExecutionException, InterruptedException {
Table table = Table.open("Keyspace1");
ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
RowMutation rm;
DecoratedKey dk = Util.dk("key1");
// add data
rm = new RowMutation("Keyspace1", dk.key);
rm.add(
new QueryPath("Standard1", null, "Column1".getBytes()),
"asdf".getBytes(),
new TimestampClock(0));
rm.apply();
store.forceBlockingFlush();
// remove
rm = new RowMutation("Keyspace1", dk.key);
rm.delete(new QueryPath("Standard1", null, "Column1".getBytes()), new TimestampClock(1));
rm.apply();
ColumnFamily retrieved =
store.getColumnFamily(
QueryFilter.getNamesFilter(dk, new QueryPath("Standard1"), "Column1".getBytes()));
assert retrieved.getColumn("Column1".getBytes()).isMarkedForDelete();
assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE));
assertNull(
Util.cloneAndRemoveDeleted(
store.getColumnFamily(QueryFilter.getIdentityFilter(dk, new QueryPath("Standard1"))),
Integer.MAX_VALUE));
}
private static void insertRowWithKey(int key) {
long timestamp = System.currentTimeMillis();
DecoratedKey decoratedKey = Util.dk(String.format("%03d", key));
Mutation rm = new Mutation(KEYSPACE1, decoratedKey.getKey());
rm.add("CF_STANDARD1", Util.cellname("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1000);
rm.applyUnsafe();
}
@Test
public void testRecoverCounter() throws IOException, ExecutionException, InterruptedException {
Table table1 = Table.open("Keyspace1");
RowMutation rm;
DecoratedKey dk = Util.dk("key");
ColumnFamily cf;
for (int i = 0; i < 10; ++i) {
rm = new RowMutation("Keyspace1", dk.key);
cf = ColumnFamily.create("Keyspace1", "Counter1");
cf.addColumn(new CounterColumn(ByteBufferUtil.bytes("col"), 1L, 1L));
rm.add(cf);
rm.apply();
}
table1.getColumnFamilyStore("Counter1").clearUnsafe();
CommitLog.instance.resetUnsafe(); // disassociate segments from live CL
CommitLog.instance.recover();
cf = Util.getColumnFamily(table1, dk, "Counter1");
assert cf.getColumnCount() == 1;
Column c = cf.getColumn(ByteBufferUtil.bytes("col"));
assert c != null;
assert ((CounterColumn) c).total() == 10L;
}
@Test
public void testOne() throws IOException, ExecutionException, InterruptedException {
Table table1 = Table.open("Keyspace1");
Table table2 = Table.open("Keyspace2");
RowMutation rm;
DecoratedKey dk = Util.dk("keymulti");
ColumnFamily cf;
rm = new RowMutation("Keyspace1", dk.key);
cf = ColumnFamily.create("Keyspace1", "Standard1");
cf.addColumn(column("col1", "val1", 1L));
rm.add(cf);
rm.apply();
rm = new RowMutation("Keyspace2", dk.key);
cf = ColumnFamily.create("Keyspace2", "Standard3");
cf.addColumn(column("col2", "val2", 1L));
rm.add(cf);
rm.apply();
table1.getColumnFamilyStore("Standard1").clearUnsafe();
table2.getColumnFamilyStore("Standard3").clearUnsafe();
CommitLog.instance.resetUnsafe(); // disassociate segments from live CL
CommitLog.instance.recover();
assertColumns(Util.getColumnFamily(table1, dk, "Standard1"), "col1");
assertColumns(Util.getColumnFamily(table2, dk, "Standard3"), "col2");
}
@Test
public void testRemoveColumnFamilyWithFlush1() {
Keyspace keyspace = Keyspace.open("Keyspace1");
ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
Mutation rm;
DecoratedKey dk = Util.dk("key1");
// add data
rm = new Mutation("Keyspace1", dk.key);
rm.add("Standard1", Util.cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0);
rm.add("Standard1", Util.cellname("Column2"), ByteBufferUtil.bytes("asdf"), 0);
rm.apply();
store.forceBlockingFlush();
// remove
rm = new Mutation("Keyspace1", dk.key);
rm.delete("Standard1", 1);
rm.apply();
ColumnFamily retrieved =
store.getColumnFamily(
QueryFilter.getIdentityFilter(dk, "Standard1", System.currentTimeMillis()));
assert retrieved.isMarkedForDelete();
assertNull(retrieved.getColumn(Util.cellname("Column1")));
assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE));
}
@Test
public void testCleanup()
throws IOException, ExecutionException, InterruptedException, ConfigurationException {
StorageService.instance.initServer(0);
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF2);
List<Row> rows;
// insert data and verify we get it back w/ range query
fillCF(cfs, LOOPS);
// record max timestamps of the sstables pre-cleanup
List<Long> expectedMaxTimestamps = getMaxTimestampList(cfs);
rows = Util.getRangeSlice(cfs);
assertEquals(LOOPS, rows.size());
// with one token in the ring, owned by the local node, cleanup should be a no-op
CompactionManager.instance.performCleanup(cfs, new CounterId.OneShotRenewer());
// ensure max timestamp of the sstables are retained post-cleanup
assert expectedMaxTimestamps.equals(getMaxTimestampList(cfs));
// check data is still there
rows = Util.getRangeSlice(cfs);
assertEquals(LOOPS, rows.size());
}
@Test
public void testCompactions() throws IOException, ExecutionException, InterruptedException {
CompactionManager.instance.disableAutoCompaction();
// this test does enough rows to force multiple block indexes to be used
Table table = Table.open(TABLE1);
ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
final int ROWS_PER_SSTABLE = 10;
Set<DecoratedKey> inserted = new HashSet<DecoratedKey>();
for (int j = 0; j < (DatabaseDescriptor.getIndexInterval() * 3) / ROWS_PER_SSTABLE; j++) {
for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
DecoratedKey key = Util.dk(String.valueOf(i % 2));
RowMutation rm = new RowMutation(TABLE1, key.key);
rm.add(
new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i / 2))),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
j * ROWS_PER_SSTABLE + i);
rm.apply();
inserted.add(key);
}
store.forceBlockingFlush();
assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(store).size());
}
while (true) {
Future<Integer> ft = CompactionManager.instance.submitMinorIfNeeded(store);
if (ft.get() == 0) break;
}
if (store.getSSTables().size() > 1) {
CompactionManager.instance.performMajor(store);
}
assertEquals(inserted.size(), Util.getRangeSlice(store).size());
}
@Test
public void testCheckForExpiredSSTableBlockers() throws InterruptedException {
String KEYSPACE1 = "Keyspace1";
ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
cfs.truncateBlocking();
cfs.disableAutoCompaction();
cfs.metadata.gcGraceSeconds(0);
RowMutation rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
rm.add(
"Standard1",
ByteBufferUtil.bytes("col1"),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
System.currentTimeMillis());
rm.applyUnsafe();
cfs.forceBlockingFlush();
SSTableReader blockingSSTable = cfs.getSSTables().iterator().next();
for (int i = 0; i < 10; i++) {
rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
rm.delete("Standard1", System.currentTimeMillis());
rm.applyUnsafe();
cfs.forceBlockingFlush();
}
Multimap<SSTableReader, SSTableReader> blockers =
SSTableExpiredBlockers.checkForExpiredSSTableBlockers(
cfs.getSSTables(), (int) (System.currentTimeMillis() / 1000) + 100);
assertEquals(1, blockers.keySet().size());
assertTrue(blockers.keySet().contains(blockingSSTable));
assertEquals(10, blockers.get(blockingSSTable).size());
}
@Test
public void testEchoedRow() throws IOException, ExecutionException, InterruptedException {
// This test check that EchoedRow doesn't skipp rows: see CASSANDRA-2653
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2");
// disable compaction while flushing
cfs.disableAutoCompaction();
// Insert 4 keys in two sstables. We need the sstables to have 2 rows
// at least to trigger what was causing CASSANDRA-2653
for (int i = 1; i < 5; i++) {
DecoratedKey key = Util.dk(String.valueOf(i));
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
rm.add(
"Standard2",
ByteBufferUtil.bytes(String.valueOf(i)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
i);
rm.apply();
if (i % 2 == 0) cfs.forceBlockingFlush();
}
Collection<SSTableReader> toCompact = cfs.getSSTables();
assert toCompact.size() == 2;
// Reinserting the same keys. We will compact only the previous sstable, but we need those new
// ones
// to make sure we use EchoedRow, otherwise it won't be used because purge can be done.
for (int i = 1; i < 5; i++) {
DecoratedKey key = Util.dk(String.valueOf(i));
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
rm.add(
"Standard2",
ByteBufferUtil.bytes(String.valueOf(i)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
i);
rm.apply();
}
cfs.forceBlockingFlush();
SSTableReader tmpSSTable = null;
for (SSTableReader sstable : cfs.getSSTables())
if (!toCompact.contains(sstable)) tmpSSTable = sstable;
assert tmpSSTable != null;
// Force compaction on first sstables. Since each row is in only one sstable, we will be using
// EchoedRow.
Util.compact(cfs, toCompact);
assertEquals(2, cfs.getSSTables().size());
// Now, we remove the sstable that was just created to force the use of EchoedRow (so that it
// doesn't hide the problem)
cfs.markObsolete(Collections.singleton(tmpSSTable), OperationType.UNKNOWN);
assertEquals(1, cfs.getSSTables().size());
// Now assert we do have the 4 keys
assertEquals(4, Util.getRangeSlice(cfs).size());
}
private static class Statics {
private final String KS = "Keyspace1";
private final ByteBuffer Key = ByteBufferUtil.bytes("Key01");
private final SortedSet<ByteBuffer> NamedCols =
new TreeSet<ByteBuffer>(BytesType.instance) {
{
add(ByteBufferUtil.bytes("AAA"));
add(ByteBufferUtil.bytes("BBB"));
add(ByteBufferUtil.bytes("CCC"));
}
};
private final ByteBuffer SC = ByteBufferUtil.bytes("SCName");
private final SortedSet<ByteBuffer> NamedSCCols =
new TreeSet<ByteBuffer>(BytesType.instance) {
{
add(CompositeType.build(SC, ByteBufferUtil.bytes("AAA")));
add(CompositeType.build(SC, ByteBufferUtil.bytes("BBB")));
add(CompositeType.build(SC, ByteBufferUtil.bytes("CCC")));
}
};
private final String StandardCF = "Standard1";
private final String SuperCF = "Super1";
private final long readTs = 1369935512292L;
private final ColumnFamily StandardCf =
TreeMapBackedSortedColumns.factory.create(KS, StandardCF);
private final ColumnFamily SuperCf = TreeMapBackedSortedColumns.factory.create(KS, SuperCF);
private final Row StandardRow = new Row(Util.dk("key0"), StandardCf);
private final Row SuperRow = new Row(Util.dk("key1"), SuperCf);
private final Row NullRow = new Row(Util.dk("key2"), null);
private Statics() {
StandardCf.addColumn(new Column(bb("aaaa")));
StandardCf.addColumn(new Column(bb("bbbb"), bb("bbbbb-value")));
StandardCf.addColumn(new Column(bb("cccc"), bb("ccccc-value"), 1000L));
StandardCf.addColumn(new DeletedColumn(bb("dddd"), 500, 1000));
StandardCf.addColumn(new DeletedColumn(bb("eeee"), bb("eeee-value"), 1001));
StandardCf.addColumn(new ExpiringColumn(bb("ffff"), bb("ffff-value"), 2000, 1000));
StandardCf.addColumn(new ExpiringColumn(bb("gggg"), bb("gggg-value"), 2001, 1000, 2002));
SuperCf.addColumn(new Column(CompositeType.build(SC, bb("aaaa"))));
SuperCf.addColumn(new Column(CompositeType.build(SC, bb("bbbb")), bb("bbbbb-value")));
SuperCf.addColumn(new Column(CompositeType.build(SC, bb("cccc")), bb("ccccc-value"), 1000L));
SuperCf.addColumn(new DeletedColumn(CompositeType.build(SC, bb("dddd")), 500, 1000));
SuperCf.addColumn(
new DeletedColumn(CompositeType.build(SC, bb("eeee")), bb("eeee-value"), 1001));
SuperCf.addColumn(
new ExpiringColumn(CompositeType.build(SC, bb("ffff")), bb("ffff-value"), 2000, 1000));
SuperCf.addColumn(
new ExpiringColumn(
CompositeType.build(SC, bb("gggg")), bb("gggg-value"), 2001, 1000, 2002));
}
}
@Test
public void testCleanupWithIndexes()
throws IOException, ExecutionException, InterruptedException {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF1);
List<Row> rows;
// insert data and verify we get it back w/ range query
fillCF(cfs, LOOPS);
rows = Util.getRangeSlice(cfs);
assertEquals(LOOPS, rows.size());
SecondaryIndex index = cfs.indexManager.getIndexForColumn(COLUMN);
long start = System.nanoTime();
while (!index.isIndexBuilt(COLUMN) && System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10))
Thread.sleep(10);
// verify we get it back w/ index query too
IndexExpression expr = new IndexExpression(COLUMN, IndexExpression.Operator.EQ, VALUE);
List<IndexExpression> clause = Arrays.asList(expr);
IDiskAtomFilter filter = new IdentityQueryFilter();
IPartitioner p = StorageService.getPartitioner();
Range<RowPosition> range = Util.range("", "");
rows = keyspace.getColumnFamilyStore(CF1).search(range, clause, filter, Integer.MAX_VALUE);
assertEquals(LOOPS, rows.size());
// we don't allow cleanup when the local host has no range to avoid wipping up all data when a
// node has not join the ring.
// So to make sure cleanup erase everything here, we give the localhost the tiniest possible
// range.
TokenMetadata tmd = StorageService.instance.getTokenMetadata();
byte[] tk1 = new byte[1], tk2 = new byte[1];
tk1[0] = 2;
tk2[0] = 1;
tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1"));
tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2"));
CompactionManager.instance.performCleanup(cfs, new CounterId.OneShotRenewer());
// row data should be gone
rows = Util.getRangeSlice(cfs);
assertEquals(0, rows.size());
// not only should it be gone but there should be no data on disk, not even tombstones
assert cfs.getSSTables().isEmpty();
// 2ary indexes should result in no results, too (although tombstones won't be gone until
// compacted)
rows = cfs.search(range, clause, filter, Integer.MAX_VALUE);
assertEquals(0, rows.size());
}
private void testDontPurgeAccidentaly(String k, String cfname)
throws IOException, ExecutionException, InterruptedException {
// This test catches the regression of CASSANDRA-2786
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
// disable compaction while flushing
cfs.clearUnsafe();
cfs.disableAutoCompaction();
// Add test row
DecoratedKey key = Util.dk(k);
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
rm.add(
cfname,
CompositeType.build(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
0);
rm.apply();
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesBefore = cfs.getSSTables();
QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis());
assert !(cfs.getColumnFamily(filter).getColumnCount() == 0);
// Remove key
rm = new RowMutation(KEYSPACE1, key.key);
rm.delete(cfname, 2);
rm.apply();
ColumnFamily cf = cfs.getColumnFamily(filter);
assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;
// Sleep one second so that the removal is indeed purgeable even with gcgrace == 0
Thread.sleep(1000);
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesAfter = cfs.getSSTables();
Collection<SSTableReader> toCompact = new ArrayList<SSTableReader>();
for (SSTableReader sstable : sstablesAfter)
if (!sstablesBefore.contains(sstable)) toCompact.add(sstable);
Util.compact(cfs, toCompact);
cf = cfs.getColumnFamily(filter);
assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;
}
@Test(timeout = 5000)
public void testTruncateHints() throws Exception {
Keyspace systemKeyspace = Keyspace.open("system");
ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS);
hintStore.clearUnsafe();
// insert 1 hint
Mutation rm = new Mutation(KEYSPACE4, ByteBufferUtil.bytes(1));
rm.add(
STANDARD1_CF,
Util.cellname(COLUMN1),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
System.currentTimeMillis());
HintedHandOffManager.instance
.hintFor(
rm,
System.currentTimeMillis(),
HintedHandOffManager.calculateHintTTL(rm),
UUID.randomUUID())
.applyUnsafe();
assert getNoOfHints() == 1;
HintedHandOffManager.instance.truncateAllHints();
while (getNoOfHints() > 0) {
Thread.sleep(100);
}
assert getNoOfHints() == 0;
}
/**
* Tests that the system.peers table is not updated after a node has been removed. (See
* CASSANDRA-6053)
*/
@Test
public void testStateChangeOnRemovedNode() throws UnknownHostException {
StorageService ss = StorageService.instance;
VersionedValue.VersionedValueFactory valueFactory =
new VersionedValue.VersionedValueFactory(partitioner);
// create a ring of 2 nodes
ArrayList<Token> endpointTokens = new ArrayList<>();
List<InetAddress> hosts = new ArrayList<>();
Util.createInitialRing(
ss, partitioner, endpointTokens, new ArrayList<Token>(), hosts, new ArrayList<UUID>(), 2);
InetAddress toRemove = hosts.get(1);
SystemKeyspace.updatePeerInfo(toRemove, "data_center", "dc42");
SystemKeyspace.updatePeerInfo(toRemove, "rack", "rack42");
assertEquals("rack42", SystemKeyspace.loadDcRackInfo().get(toRemove).get("rack"));
// mark the node as removed
Gossiper.instance.injectApplicationState(
toRemove,
ApplicationState.STATUS,
valueFactory.left(
Collections.singleton(endpointTokens.get(1)), Gossiper.computeExpireTime()));
assertTrue(
Gossiper.instance.isDeadState(Gossiper.instance.getEndpointStateForEndpoint(hosts.get(1))));
// state changes made after the endpoint has left should be ignored
ss.onChange(hosts.get(1), ApplicationState.RACK, valueFactory.rack("rack9999"));
assertEquals("rack42", SystemKeyspace.loadDcRackInfo().get(toRemove).get("rack"));
}
private long populate(String ks, String cf, int startRowKey, int endRowKey, int ttl) {
long timestamp = System.currentTimeMillis();
for (int i = startRowKey; i <= endRowKey; i++) {
DecoratedKey key = Util.dk(Integer.toString(i));
Mutation rm = new Mutation(ks, key.getKey());
for (int j = 0; j < 10; j++)
rm.add(
cf,
Util.cellname(Integer.toString(j)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
timestamp,
j > 0
? ttl
: 0); // let first column never expire, since deleting all columns does not produce
// sstable
rm.applyUnsafe();
}
return timestamp;
}
@Test
public void testNoExpire() throws ExecutionException, InterruptedException {
ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
cfs.disableAutoCompaction();
cfs.metadata.gcGraceSeconds(0);
long timestamp = System.currentTimeMillis();
RowMutation rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.add(
"Standard1", ByteBufferUtil.bytes("col7"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col3"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
DecoratedKey noTTLKey = Util.dk("nottl");
rm = new RowMutation("Keyspace1", noTTLKey.key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col311"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp);
rm.apply();
cfs.forceBlockingFlush();
Thread.sleep(2000); // wait for ttl to expire
assertEquals(4, cfs.getSSTables().size());
cfs.enableAutoCompaction(true);
assertEquals(1, cfs.getSSTables().size());
SSTableReader sstable = cfs.getSSTables().iterator().next();
SSTableScanner scanner = sstable.getScanner(DataRange.allData(sstable.partitioner));
assertTrue(scanner.hasNext());
while (scanner.hasNext()) {
OnDiskAtomIterator iter = scanner.next();
assertEquals(noTTLKey, iter.getKey());
}
}
/*
* This excercise in particular the code of #4142
*/
@Test
public void testValidationMultipleSSTablePerLevel() throws Exception {
String ksname = "Keyspace1";
String cfname = "StandardLeveled";
Table table = Table.open(ksname);
ColumnFamilyStore store = table.getColumnFamilyStore(cfname);
ByteBuffer value =
ByteBuffer.wrap(new byte[100 * 1024]); // 100 KB value, make it easy to have multiple files
// Enough data to have a level 1 and 2
int rows = 20;
int columns = 10;
// Adds enough data to trigger multiple sstable per level
for (int r = 0; r < rows; r++) {
DecoratedKey key = Util.dk(String.valueOf(r));
RowMutation rm = new RowMutation(ksname, key.key);
for (int c = 0; c < columns; c++) {
rm.add(new QueryPath(cfname, null, ByteBufferUtil.bytes("column" + c)), value, 0);
}
rm.apply();
store.forceFlush();
}
LeveledCompactionStrategy strat = (LeveledCompactionStrategy) store.getCompactionStrategy();
while (strat.getLevelSize(0) > 0) {
store.forceMajorCompaction();
Thread.sleep(200);
}
// Checking we're not completely bad at math
assert strat.getLevelSize(1) > 0;
assert strat.getLevelSize(2) > 0;
AntiEntropyService.CFPair p = new AntiEntropyService.CFPair(ksname, cfname);
Range<Token> range = new Range<Token>(Util.token(""), Util.token(""));
AntiEntropyService.TreeRequest req =
new AntiEntropyService.TreeRequest("1", FBUtilities.getLocalAddress(), range, p);
AntiEntropyService.Validator validator = new AntiEntropyService.Validator(req);
CompactionManager.instance.submitValidation(store, validator).get();
}
@Test
public void testSuperColumnTombstones() throws ExecutionException, InterruptedException {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1");
cfs.disableAutoCompaction();
DecoratedKey key = Util.dk("tskey");
ByteBuffer scName = ByteBufferUtil.bytes("TestSuperColumn");
// a subcolumn
Mutation rm = new Mutation(KEYSPACE1, key.getKey());
rm.add(
"Super1",
Util.cellname(scName, ByteBufferUtil.bytes(0)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
FBUtilities.timestampMicros());
rm.applyUnsafe();
cfs.forceBlockingFlush();
// shadow the subcolumn with a supercolumn tombstone
rm = new Mutation(KEYSPACE1, key.getKey());
rm.deleteRange(
"Super1",
SuperColumns.startOf(scName),
SuperColumns.endOf(scName),
FBUtilities.timestampMicros());
rm.applyUnsafe();
cfs.forceBlockingFlush();
CompactionManager.instance.performMaximal(cfs);
assertEquals(1, cfs.getSSTables().size());
// check that the shadowed column is gone
SSTableReader sstable = cfs.getSSTables().iterator().next();
Range keyRange =
new Range<RowPosition>(key, sstable.partitioner.getMinimumToken().maxKeyBound());
SSTableScanner scanner = sstable.getScanner(DataRange.forKeyRange(keyRange));
OnDiskAtomIterator iter = scanner.next();
assertEquals(key, iter.getKey());
assertTrue(iter.next() instanceof RangeTombstone);
assertFalse(iter.hasNext());
}
@Test
public void testStateJumpToLeft() throws UnknownHostException {
StorageService ss = StorageService.instance;
TokenMetadata tmd = ss.getTokenMetadata();
tmd.clearUnsafe();
IPartitioner partitioner = RandomPartitioner.instance;
VersionedValue.VersionedValueFactory valueFactory =
new VersionedValue.VersionedValueFactory(partitioner);
ArrayList<Token> endpointTokens = new ArrayList<Token>();
ArrayList<Token> keyTokens = new ArrayList<Token>();
List<InetAddress> hosts = new ArrayList<InetAddress>();
List<UUID> hostIds = new ArrayList<UUID>();
// create a ring of 6 nodes
Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 7);
// node hosts.get(2) goes jumps to left
ss.onChange(
hosts.get(2),
ApplicationState.STATUS,
valueFactory.left(
Collections.singleton(endpointTokens.get(2)), Gossiper.computeExpireTime()));
assertFalse(tmd.isMember(hosts.get(2)));
// node hosts.get(4) goes to bootstrap
Gossiper.instance.injectApplicationState(
hosts.get(3),
ApplicationState.TOKENS,
valueFactory.tokens(Collections.singleton(keyTokens.get(1))));
ss.onChange(
hosts.get(3),
ApplicationState.STATUS,
valueFactory.bootstrapping(Collections.<Token>singleton(keyTokens.get(1))));
assertFalse(tmd.isMember(hosts.get(3)));
assertEquals(1, tmd.getBootstrapTokens().size());
assertEquals(hosts.get(3), tmd.getBootstrapTokens().get(keyTokens.get(1)));
// and then directly to 'left'
Gossiper.instance.injectApplicationState(
hosts.get(2),
ApplicationState.TOKENS,
valueFactory.tokens(Collections.singleton(keyTokens.get(1))));
ss.onChange(
hosts.get(2),
ApplicationState.STATUS,
valueFactory.left(Collections.singleton(keyTokens.get(1)), Gossiper.computeExpireTime()));
assertTrue(tmd.getBootstrapTokens().size() == 0);
assertFalse(tmd.isMember(hosts.get(2)));
assertFalse(tmd.isLeaving(hosts.get(2)));
}
private static void insertRowWithKey(int key) {
long timestamp = System.currentTimeMillis();
DecoratedKey decoratedKey = Util.dk(String.format("%03d", key));
RowMutation rm = new RowMutation(KEYSPACE1, decoratedKey.key);
rm.add(
"Standard1",
ByteBufferUtil.bytes("col"),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
timestamp,
1000);
rm.apply();
}
@Test
public void testSimpleExpire() throws ExecutionException, InterruptedException {
ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
cfs.disableAutoCompaction();
cfs.metadata.gcGraceSeconds(0);
long timestamp = System.currentTimeMillis();
RowMutation rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.add(
"Standard1", ByteBufferUtil.bytes("col7"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col3"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1",
ByteBufferUtil.bytes("col311"),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
timestamp,
1);
rm.apply();
cfs.forceBlockingFlush();
Thread.sleep(2000); // wait for ttl to expire
assertEquals(4, cfs.getSSTables().size());
cfs.enableAutoCompaction(true);
assertEquals(0, cfs.getSSTables().size());
}
@Test
public void testUserDefinedCompaction() throws Exception {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
final String cfname = "Standard3"; // use clean(no sstable) CF
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
// disable compaction while flushing
cfs.disableAutoCompaction();
final int ROWS_PER_SSTABLE = 10;
for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
DecoratedKey key = Util.dk(String.valueOf(i));
Mutation rm = new Mutation(KEYSPACE1, key.getKey());
rm.add(
cfname,
Util.cellname("col"),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
System.currentTimeMillis());
rm.applyUnsafe();
}
cfs.forceBlockingFlush();
Collection<SSTableReader> sstables = cfs.getSSTables();
assertEquals(1, sstables.size());
SSTableReader sstable = sstables.iterator().next();
int prevGeneration = sstable.descriptor.generation;
String file = new File(sstable.descriptor.filenameFor(Component.DATA)).getAbsolutePath();
// submit user defined compaction on flushed sstable
CompactionManager.instance.forceUserDefinedCompaction(file);
// wait until user defined compaction finishes
do {
Thread.sleep(100);
} while (CompactionManager.instance.getPendingTasks() > 0
|| CompactionManager.instance.getActiveCompactions() > 0);
// CF should have only one sstable with generation number advanced
sstables = cfs.getSSTables();
assertEquals(1, sstables.size());
assertEquals(prevGeneration + 1, sstables.iterator().next().descriptor.generation);
}
@Before
public void setup() throws IOException, ConfigurationException {
tmd.clearUnsafe();
// create a ring of 5 nodes
Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 6);
MessagingService.instance().listen(FBUtilities.getBroadcastAddress());
Gossiper.instance.start(1);
removalhost = hosts.get(5);
hosts.remove(removalhost);
removalId = hostIds.get(5);
hostIds.remove(removalId);
}
/** test get_count() to work correctly with 'count' settings around page size. (CASSANDRA-4833) */
@Test
public void test_get_count() throws Exception {
Schema.instance.clear(); // Schema are now written on disk and will be reloaded
new EmbeddedCassandraService().start();
ThriftSessionManager.instance.setCurrentSocket(new InetSocketAddress(9160));
DecoratedKey key = Util.dk("testkey");
for (int i = 0; i < 3050; i++) {
RowMutation rm = new RowMutation("Keyspace1", key.key);
rm.add(
new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i))),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
System.currentTimeMillis());
rm.apply();
}
CassandraServer server = new CassandraServer();
server.set_keyspace("Keyspace1");
// same as page size
int count =
server.get_count(
key.key, new ColumnParent("Standard1"), predicateWithCount(1024), ConsistencyLevel.ONE);
assert count == 1024 : "expected 1024 but was " + count;
// 1 above page size
count =
server.get_count(
key.key, new ColumnParent("Standard1"), predicateWithCount(1025), ConsistencyLevel.ONE);
assert count == 1025 : "expected 1025 but was " + count;
// above number of columns
count =
server.get_count(
key.key, new ColumnParent("Standard1"), predicateWithCount(4000), ConsistencyLevel.ONE);
assert count == 3050 : "expected 3050 but was " + count;
// same as number of columns
count =
server.get_count(
key.key, new ColumnParent("Standard1"), predicateWithCount(3050), ConsistencyLevel.ONE);
assert count == 3050 : "expected 3050 but was " + count;
// 1 above number of columns
count =
server.get_count(
key.key, new ColumnParent("Standard1"), predicateWithCount(3051), ConsistencyLevel.ONE);
assert count == 3050 : "expected 3050 but was " + count;
}
public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
store.clearUnsafe();
store.metadata.gcGraceSeconds(1);
store.setCompactionStrategyClass(strategyClassName);
// disable compaction while flushing
store.disableAutoCompaction();
long timestamp = System.currentTimeMillis();
for (int i = 0; i < 10; i++) {
DecoratedKey key = Util.dk(Integer.toString(i));
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
for (int j = 0; j < 10; j++)
rm.add(
"Standard1",
ByteBufferUtil.bytes(Integer.toString(j)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
timestamp,
j > 0
? 3
: 0); // let first column never expire, since deleting all columns does not produce
// sstable
rm.apply();
}
store.forceBlockingFlush();
assertEquals(1, store.getSSTables().size());
long originalSize = store.getSSTables().iterator().next().uncompressedLength();
// wait enough to force single compaction
TimeUnit.SECONDS.sleep(5);
// enable compaction, submit background and wait for it to complete
store.enableAutoCompaction();
FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
while (CompactionManager.instance.getPendingTasks() > 0
|| CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);
// and sstable with ttl should be compacted
assertEquals(1, store.getSSTables().size());
long size = store.getSSTables().iterator().next().uncompressedLength();
assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize);
// make sure max timestamp of compacted sstables is recorded properly after compaction.
assertMaxTimestamp(store, timestamp);
return store;
}
protected void fillCF(ColumnFamilyStore cfs, int rowsPerSSTable) {
CompactionManager.instance.disableAutoCompaction();
for (int i = 0; i < rowsPerSSTable; i++) {
String key = String.valueOf(i);
// create a row and update the birthdate value, test that the index query fetches the new
// version
Mutation rm;
rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes(key));
rm.add(cfs.name, Util.cellname(COLUMN), VALUE, System.currentTimeMillis());
rm.applyUnsafe();
}
cfs.forceBlockingFlush();
}
@Test
public void testAggressiveFullyExpired() {
String KEYSPACE1 = "Keyspace1";
ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
cfs.disableAutoCompaction();
cfs.metadata.gcGraceSeconds(0);
DecoratedKey ttlKey = Util.dk("ttl");
RowMutation rm = new RowMutation("Keyspace1", ttlKey.key);
rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 1, 1);
rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 3, 1);
rm.applyUnsafe();
cfs.forceBlockingFlush();
rm = new RowMutation(KEYSPACE1, ttlKey.key);
rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 2, 1);
rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 5, 1);
rm.applyUnsafe();
cfs.forceBlockingFlush();
rm = new RowMutation(KEYSPACE1, ttlKey.key);
rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 4, 1);
rm.add("Standard1", ByteBufferUtil.bytes("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 7, 1);
rm.applyUnsafe();
cfs.forceBlockingFlush();
rm = new RowMutation(KEYSPACE1, ttlKey.key);
rm.add("Standard1", ByteBufferUtil.bytes("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 6, 3);
rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 8, 1);
rm.applyUnsafe();
cfs.forceBlockingFlush();
Set<SSTableReader> sstables = Sets.newHashSet(cfs.getSSTables());
int now = (int) (System.currentTimeMillis() / 1000);
int gcBefore = now + 2;
Set<SSTableReader> expired =
CompactionController.getFullyExpiredSSTables(
cfs, sstables, Collections.EMPTY_SET, gcBefore);
assertEquals(2, expired.size());
cfs.clearUnsafe();
}
protected void testCompaction(int sstableCount, int rowsPerSSTable, int colsPerRow)
throws Exception {
CompactionManager.instance.disableAutoCompaction();
Table table = Table.open(TABLE1);
ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
ArrayList<SSTableReader> sstables = new ArrayList<SSTableReader>();
for (int k = 0; k < sstableCount; k++) {
SortedMap<String, ColumnFamily> rows = new TreeMap<String, ColumnFamily>();
for (int j = 0; j < rowsPerSSTable; j++) {
String key = String.valueOf(j);
IColumn[] cols = new IColumn[colsPerRow];
for (int i = 0; i < colsPerRow; i++) {
// last sstable has highest timestamps
cols[i] = Util.column(String.valueOf(i), String.valueOf(i), k);
}
rows.put(key, SSTableUtils.createCF(Long.MIN_VALUE, Integer.MIN_VALUE, cols));
}
SSTableReader sstable = SSTableUtils.writeSSTable(rows);
sstables.add(sstable);
store.addSSTable(sstable);
}
// give garbage collection a bit of time to catch up
Thread.sleep(1000);
long start = System.currentTimeMillis();
CompactionManager.instance.doCompaction(
store,
sstables,
(int) (System.currentTimeMillis() / 1000)
- DatabaseDescriptor.getCFMetaData(TABLE1, "Standard1").gcGraceSeconds);
System.out.println(
String.format(
"%s: sstables=%d rowsper=%d colsper=%d: %d ms",
this.getClass().getName(),
sstableCount,
rowsPerSSTable,
colsPerRow,
System.currentTimeMillis() - start));
}
@Test
public void testRemovingStatusForNonMember() throws UnknownHostException {
// create a ring of 1 node
StorageService ss = StorageService.instance;
VersionedValue.VersionedValueFactory valueFactory =
new VersionedValue.VersionedValueFactory(partitioner);
Util.createInitialRing(
ss,
partitioner,
new ArrayList<Token>(),
new ArrayList<Token>(),
new ArrayList<InetAddress>(),
new ArrayList<UUID>(),
1);
// make a REMOVING state change on a non-member endpoint; without the CASSANDRA-6564 fix, this
// would result in an ArrayIndexOutOfBoundsException
ss.onChange(
InetAddress.getByName("192.168.1.42"),
ApplicationState.STATUS,
valueFactory.removingNonlocal(UUID.randomUUID()));
}