@Test public void testEchoedRow() { // This test check that EchoedRow doesn't skipp rows: see CASSANDRA-2653 Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2"); // disable compaction while flushing cfs.disableAutoCompaction(); // Insert 4 keys in two sstables. We need the sstables to have 2 rows // at least to trigger what was causing CASSANDRA-2653 for (int i = 1; i < 5; i++) { DecoratedKey key = Util.dk(String.valueOf(i)); Mutation rm = new Mutation(KEYSPACE1, key.getKey()); rm.add("Standard2", Util.cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i); rm.applyUnsafe(); if (i % 2 == 0) cfs.forceBlockingFlush(); } Collection<SSTableReader> toCompact = cfs.getSSTables(); assertEquals(2, toCompact.size()); // Reinserting the same keys. We will compact only the previous sstable, but we need those new // ones // to make sure we use EchoedRow, otherwise it won't be used because purge can be done. for (int i = 1; i < 5; i++) { DecoratedKey key = Util.dk(String.valueOf(i)); Mutation rm = new Mutation(KEYSPACE1, key.getKey()); rm.add("Standard2", Util.cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i); rm.applyUnsafe(); } cfs.forceBlockingFlush(); SSTableReader tmpSSTable = null; for (SSTableReader sstable : cfs.getSSTables()) if (!toCompact.contains(sstable)) tmpSSTable = sstable; assertNotNull(tmpSSTable); // Force compaction on first sstables. Since each row is in only one sstable, we will be using // EchoedRow. Util.compact(cfs, toCompact); assertEquals(2, cfs.getSSTables().size()); // Now, we remove the sstable that was just created to force the use of EchoedRow (so that it // doesn't hide the problem) cfs.markObsolete(Collections.singleton(tmpSSTable), OperationType.UNKNOWN); assertEquals(1, cfs.getSSTables().size()); // Now assert we do have the 4 keys assertEquals(4, Util.getRangeSlice(cfs).size()); }
@Test(timeout = 5000) public void testTruncateHints() throws Exception { Keyspace systemKeyspace = Keyspace.open("system"); ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS); hintStore.clearUnsafe(); // insert 1 hint Mutation rm = new Mutation(KEYSPACE4, ByteBufferUtil.bytes(1)); rm.add( STANDARD1_CF, Util.cellname(COLUMN1), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); HintedHandOffManager.instance .hintFor( rm, System.currentTimeMillis(), HintedHandOffManager.calculateHintTTL(rm), UUID.randomUUID()) .applyUnsafe(); assert getNoOfHints() == 1; HintedHandOffManager.instance.truncateAllHints(); while (getNoOfHints() > 0) { Thread.sleep(100); } assert getNoOfHints() == 0; }
private static void insertRowWithKey(int key) { long timestamp = System.currentTimeMillis(); DecoratedKey decoratedKey = Util.dk(String.format("%03d", key)); Mutation rm = new Mutation(KEYSPACE1, decoratedKey.getKey()); rm.add("CF_STANDARD1", Util.cellname("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1000); rm.applyUnsafe(); }
@Test public void testRemoveColumnFamilyWithFlush1() { Keyspace keyspace = Keyspace.open("Keyspace1"); ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1"); Mutation rm; DecoratedKey dk = Util.dk("key1"); // add data rm = new Mutation("Keyspace1", dk.key); rm.add("Standard1", Util.cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0); rm.add("Standard1", Util.cellname("Column2"), ByteBufferUtil.bytes("asdf"), 0); rm.apply(); store.forceBlockingFlush(); // remove rm = new Mutation("Keyspace1", dk.key); rm.delete("Standard1", 1); rm.apply(); ColumnFamily retrieved = store.getColumnFamily( QueryFilter.getIdentityFilter(dk, "Standard1", System.currentTimeMillis())); assert retrieved.isMarkedForDelete(); assertNull(retrieved.getColumn(Util.cellname("Column1"))); assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE)); }
public static void addMutation( Mutation rm, String columnFamilyName, String superColumnName, long columnName, String value, long timestamp) { CellName cname = superColumnName == null ? CellNames.simpleDense(getBytes(columnName)) : CellNames.compositeDense(ByteBufferUtil.bytes(superColumnName), getBytes(columnName)); rm.add(columnFamilyName, cname, ByteBufferUtil.bytes(value), timestamp); }
private void testDontPurgeAccidentaly(String k, String cfname) throws InterruptedException { // This test catches the regression of CASSANDRA-2786 Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname); // disable compaction while flushing cfs.clearUnsafe(); cfs.disableAutoCompaction(); // Add test row DecoratedKey key = Util.dk(k); Mutation rm = new Mutation(KEYSPACE1, key.getKey()); rm.add( cfname, Util.cellname(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); rm.applyUnsafe(); cfs.forceBlockingFlush(); Collection<SSTableReader> sstablesBefore = cfs.getSSTables(); QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis()); assertTrue(cfs.getColumnFamily(filter).hasColumns()); // Remove key rm = new Mutation(KEYSPACE1, key.getKey()); rm.delete(cfname, 2); rm.applyUnsafe(); ColumnFamily cf = cfs.getColumnFamily(filter); assertTrue("should be empty: " + cf, cf == null || !cf.hasColumns()); // Sleep one second so that the removal is indeed purgeable even with gcgrace == 0 Thread.sleep(1000); cfs.forceBlockingFlush(); Collection<SSTableReader> sstablesAfter = cfs.getSSTables(); Collection<SSTableReader> toCompact = new ArrayList<SSTableReader>(); for (SSTableReader sstable : sstablesAfter) if (!sstablesBefore.contains(sstable)) toCompact.add(sstable); Util.compact(cfs, toCompact); cf = cfs.getColumnFamily(filter); assertTrue("should be empty: " + cf, cf == null || !cf.hasColumns()); }
protected void fillCF(ColumnFamilyStore cfs, int rowsPerSSTable) { CompactionManager.instance.disableAutoCompaction(); for (int i = 0; i < rowsPerSSTable; i++) { String key = String.valueOf(i); // create a row and update the birthdate value, test that the index query fetches the new // version Mutation rm; rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes(key)); rm.add(cfs.name, Util.cellname(COLUMN), VALUE, System.currentTimeMillis()); rm.applyUnsafe(); } cfs.forceBlockingFlush(); }
private long populate(String ks, String cf, int startRowKey, int endRowKey, int ttl) { long timestamp = System.currentTimeMillis(); for (int i = startRowKey; i <= endRowKey; i++) { DecoratedKey key = Util.dk(Integer.toString(i)); Mutation rm = new Mutation(ks, key.getKey()); for (int j = 0; j < 10; j++) rm.add( cf, Util.cellname(Integer.toString(j)), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, j > 0 ? ttl : 0); // let first column never expire, since deleting all columns does not produce // sstable rm.applyUnsafe(); } return timestamp; }
@Test public void testSuperColumnTombstones() throws ExecutionException, InterruptedException { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1"); cfs.disableAutoCompaction(); DecoratedKey key = Util.dk("tskey"); ByteBuffer scName = ByteBufferUtil.bytes("TestSuperColumn"); // a subcolumn Mutation rm = new Mutation(KEYSPACE1, key.getKey()); rm.add( "Super1", Util.cellname(scName, ByteBufferUtil.bytes(0)), ByteBufferUtil.EMPTY_BYTE_BUFFER, FBUtilities.timestampMicros()); rm.applyUnsafe(); cfs.forceBlockingFlush(); // shadow the subcolumn with a supercolumn tombstone rm = new Mutation(KEYSPACE1, key.getKey()); rm.deleteRange( "Super1", SuperColumns.startOf(scName), SuperColumns.endOf(scName), FBUtilities.timestampMicros()); rm.applyUnsafe(); cfs.forceBlockingFlush(); CompactionManager.instance.performMaximal(cfs); assertEquals(1, cfs.getSSTables().size()); // check that the shadowed column is gone SSTableReader sstable = cfs.getSSTables().iterator().next(); Range keyRange = new Range<RowPosition>(key, sstable.partitioner.getMinimumToken().maxKeyBound()); SSTableScanner scanner = sstable.getScanner(DataRange.forKeyRange(keyRange)); OnDiskAtomIterator iter = scanner.next(); assertEquals(key, iter.getKey()); assertTrue(iter.next() instanceof RangeTombstone); assertFalse(iter.hasNext()); }
@Test public void testUserDefinedCompaction() throws Exception { Keyspace keyspace = Keyspace.open(KEYSPACE1); final String cfname = "Standard3"; // use clean(no sstable) CF ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname); // disable compaction while flushing cfs.disableAutoCompaction(); final int ROWS_PER_SSTABLE = 10; for (int i = 0; i < ROWS_PER_SSTABLE; i++) { DecoratedKey key = Util.dk(String.valueOf(i)); Mutation rm = new Mutation(KEYSPACE1, key.getKey()); rm.add( cfname, Util.cellname("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); rm.applyUnsafe(); } cfs.forceBlockingFlush(); Collection<SSTableReader> sstables = cfs.getSSTables(); assertEquals(1, sstables.size()); SSTableReader sstable = sstables.iterator().next(); int prevGeneration = sstable.descriptor.generation; String file = new File(sstable.descriptor.filenameFor(Component.DATA)).getAbsolutePath(); // submit user defined compaction on flushed sstable CompactionManager.instance.forceUserDefinedCompaction(file); // wait until user defined compaction finishes do { Thread.sleep(100); } while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0); // CF should have only one sstable with generation number advanced sstables = cfs.getSSTables(); assertEquals(1, sstables.size()); assertEquals(prevGeneration + 1, sstables.iterator().next().descriptor.generation); }
// Test compaction of hints column family. It shouldn't remove all columns on compaction. @Test public void testCompactionOfHintsCF() throws Exception { // prepare hints column family Keyspace systemKeyspace = Keyspace.open("system"); ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS); hintStore.clearUnsafe(); hintStore.metadata.gcGraceSeconds(36000); // 10 hours hintStore.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getCanonicalName()); hintStore.disableAutoCompaction(); // insert 1 hint Mutation rm = new Mutation(KEYSPACE4, ByteBufferUtil.bytes(1)); rm.add( STANDARD1_CF, Util.cellname(COLUMN1), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); HintedHandOffManager.instance .hintFor( rm, System.currentTimeMillis(), HintedHandOffManager.calculateHintTTL(rm), UUID.randomUUID()) .applyUnsafe(); // flush data to disk hintStore.forceBlockingFlush(); assertEquals(1, hintStore.getSSTables().size()); // submit compaction HintedHandOffManager.instance.compact(); // single row should not be removed because of gc_grace_seconds // is 10 hours and there are no any tombstones in sstable assertEquals(1, hintStore.getSSTables().size()); }
private void insert(String key) { Mutation rm; rm = new Mutation(KEYSPACE, ByteBufferUtil.bytes(key)); rm.add(CF, Util.cellname("column"), ByteBufferUtil.bytes("asdf"), 0); rm.apply(); }
@Test public void testRowCacheRange() { CompactionManager.instance.disableAutoCompaction(); Keyspace keyspace = Keyspace.open(KEYSPACE_CACHED); String cf = "CachedIntCF"; ColumnFamilyStore cachedStore = keyspace.getColumnFamilyStore(cf); long startRowCacheHits = cachedStore.metric.rowCacheHit.getCount(); long startRowCacheOutOfRange = cachedStore.metric.rowCacheHitOutOfRange.getCount(); // empty the row cache CacheService.instance.invalidateRowCache(); // set global row cache size to 1 MB CacheService.instance.setRowCacheCapacityInMB(1); ByteBuffer key = ByteBufferUtil.bytes("rowcachekey"); DecoratedKey dk = cachedStore.partitioner.decorateKey(key); RowCacheKey rck = new RowCacheKey(cachedStore.metadata.ksAndCFName, dk); Mutation mutation = new Mutation(KEYSPACE_CACHED, key); for (int i = 0; i < 200; i++) mutation.add( cf, Util.cellname(i), ByteBufferUtil.bytes("val" + i), System.currentTimeMillis()); mutation.applyUnsafe(); // populate row cache, we should not get a row cache hit; cachedStore.getColumnFamily( QueryFilter.getSliceFilter( dk, cf, Composites.EMPTY, Composites.EMPTY, false, 10, System.currentTimeMillis())); assertEquals(startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); // do another query, limit is 20, which is < 100 that we cache, we should get a hit and it // should be in range cachedStore.getColumnFamily( QueryFilter.getSliceFilter( dk, cf, Composites.EMPTY, Composites.EMPTY, false, 20, System.currentTimeMillis())); assertEquals(++startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); assertEquals(startRowCacheOutOfRange, cachedStore.metric.rowCacheHitOutOfRange.getCount()); // get a slice from 95 to 105, 95->99 are in cache, we should not get a hit and then row cache // is out of range cachedStore.getColumnFamily( QueryFilter.getSliceFilter( dk, cf, CellNames.simpleDense(ByteBufferUtil.bytes(95)), CellNames.simpleDense(ByteBufferUtil.bytes(105)), false, 10, System.currentTimeMillis())); assertEquals(startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); assertEquals(++startRowCacheOutOfRange, cachedStore.metric.rowCacheHitOutOfRange.getCount()); // get a slice with limit > 100, we should get a hit out of range. cachedStore.getColumnFamily( QueryFilter.getSliceFilter( dk, cf, Composites.EMPTY, Composites.EMPTY, false, 101, System.currentTimeMillis())); assertEquals(startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); assertEquals(++startRowCacheOutOfRange, cachedStore.metric.rowCacheHitOutOfRange.getCount()); CacheService.instance.invalidateRowCache(); // try to populate row cache with a limit > rows to cache, we should still populate row cache; cachedStore.getColumnFamily( QueryFilter.getSliceFilter( dk, cf, Composites.EMPTY, Composites.EMPTY, false, 105, System.currentTimeMillis())); assertEquals(startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); // validate the stuff in cache; ColumnFamily cachedCf = (ColumnFamily) CacheService.instance.rowCache.get(rck); assertEquals(cachedCf.getColumnCount(), 100); int i = 0; for (Cell c : cachedCf) { assertEquals(c.name(), Util.cellname(i++)); } }