@Test public void testCleanupWithNewToken() throws ExecutionException, InterruptedException, UnknownHostException { StorageService.instance.getTokenMetadata().clearUnsafe(); Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF2); List<Row> rows; // insert data and verify we get it back w/ range query fillCF(cfs, LOOPS); rows = Util.getRangeSlice(cfs); assertEquals(LOOPS, rows.size()); TokenMetadata tmd = StorageService.instance.getTokenMetadata(); byte[] tk1 = new byte[1], tk2 = new byte[1]; tk1[0] = 2; tk2[0] = 1; tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1")); tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2")); CompactionManager.instance.performCleanup(cfs); rows = Util.getRangeSlice(cfs); assertEquals(0, rows.size()); }
@Test public void testRemoveColumnFamilyWithFlush1() { Keyspace keyspace = Keyspace.open("Keyspace1"); ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1"); Mutation rm; DecoratedKey dk = Util.dk("key1"); // add data rm = new Mutation("Keyspace1", dk.key); rm.add("Standard1", Util.cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0); rm.add("Standard1", Util.cellname("Column2"), ByteBufferUtil.bytes("asdf"), 0); rm.apply(); store.forceBlockingFlush(); // remove rm = new Mutation("Keyspace1", dk.key); rm.delete("Standard1", 1); rm.apply(); ColumnFamily retrieved = store.getColumnFamily( QueryFilter.getIdentityFilter(dk, "Standard1", System.currentTimeMillis())); assert retrieved.isMarkedForDelete(); assertNull(retrieved.getColumn(Util.cellname("Column1"))); assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE)); }
public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1); store.clearUnsafe(); store.metadata.gcGraceSeconds(1); store.setCompactionStrategyClass(strategyClassName); // disable compaction while flushing store.disableAutoCompaction(); long timestamp = populate(KEYSPACE1, CF_STANDARD1, 0, 9, 3); // ttl=3s store.forceBlockingFlush(); assertEquals(1, store.getSSTables().size()); long originalSize = store.getSSTables().iterator().next().uncompressedLength(); // wait enough to force single compaction TimeUnit.SECONDS.sleep(5); // enable compaction, submit background and wait for it to complete store.enableAutoCompaction(); FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store)); while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1); // and sstable with ttl should be compacted assertEquals(1, store.getSSTables().size()); long size = store.getSSTables().iterator().next().uncompressedLength(); assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize); // make sure max timestamp of compacted sstables is recorded properly after compaction. assertMaxTimestamp(store, timestamp); return store; }
@Test public void testCleanup() throws IOException, ExecutionException, InterruptedException, ConfigurationException { StorageService.instance.initServer(0); Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF2); List<Row> rows; // insert data and verify we get it back w/ range query fillCF(cfs, LOOPS); // record max timestamps of the sstables pre-cleanup List<Long> expectedMaxTimestamps = getMaxTimestampList(cfs); rows = Util.getRangeSlice(cfs); assertEquals(LOOPS, rows.size()); // with one token in the ring, owned by the local node, cleanup should be a no-op CompactionManager.instance.performCleanup(cfs, new CounterId.OneShotRenewer()); // ensure max timestamp of the sstables are retained post-cleanup assert expectedMaxTimestamps.equals(getMaxTimestampList(cfs)); // check data is still there rows = Util.getRangeSlice(cfs); assertEquals(LOOPS, rows.size()); }
@Test public void testCompactionLog() throws Exception { SystemKeyspace.discardCompactionsInProgress(); String cf = "Standard4"; ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(cf); SchemaLoader.insertData(KEYSPACE1, cf, 0, 1); cfs.forceBlockingFlush(); Collection<SSTableReader> sstables = cfs.getSSTables(); assertFalse(sstables.isEmpty()); Set<Integer> generations = Sets.newHashSet( Iterables.transform( sstables, new Function<SSTableReader, Integer>() { public Integer apply(SSTableReader sstable) { return sstable.descriptor.generation; } })); UUID taskId = SystemKeyspace.startCompaction(cfs, sstables); Map<Pair<String, String>, Map<Integer, UUID>> compactionLogs = SystemKeyspace.getUnfinishedCompactions(); Set<Integer> unfinishedCompactions = compactionLogs.get(Pair.create(KEYSPACE1, cf)).keySet(); assertTrue(unfinishedCompactions.containsAll(generations)); SystemKeyspace.finishCompaction(taskId); compactionLogs = SystemKeyspace.getUnfinishedCompactions(); assertFalse(compactionLogs.containsKey(Pair.create(KEYSPACE1, cf))); }
@Test(timeout = 5000) public void testTruncateHints() throws Exception { Keyspace systemKeyspace = Keyspace.open("system"); ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS_CF); hintStore.clearUnsafe(); // insert 1 hint RowMutation rm = new RowMutation(KEYSPACE4, ByteBufferUtil.bytes(1)); rm.add( STANDARD1_CF, ByteBufferUtil.bytes(String.valueOf(COLUMN1)), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); HintedHandOffManager.instance .hintFor( rm, System.currentTimeMillis(), HintedHandOffManager.calculateHintTTL(rm), UUID.randomUUID()) .apply(); assert getNoOfHints() == 1; HintedHandOffManager.instance.truncateAllHints(); while (getNoOfHints() > 0) { Thread.sleep(100); } assert getNoOfHints() == 0; }
@Test public void testEchoedRow() throws IOException, ExecutionException, InterruptedException { // This test check that EchoedRow doesn't skipp rows: see CASSANDRA-2653 Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2"); // disable compaction while flushing cfs.disableAutoCompaction(); // Insert 4 keys in two sstables. We need the sstables to have 2 rows // at least to trigger what was causing CASSANDRA-2653 for (int i = 1; i < 5; i++) { DecoratedKey key = Util.dk(String.valueOf(i)); RowMutation rm = new RowMutation(KEYSPACE1, key.key); rm.add( "Standard2", ByteBufferUtil.bytes(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i); rm.apply(); if (i % 2 == 0) cfs.forceBlockingFlush(); } Collection<SSTableReader> toCompact = cfs.getSSTables(); assert toCompact.size() == 2; // Reinserting the same keys. We will compact only the previous sstable, but we need those new // ones // to make sure we use EchoedRow, otherwise it won't be used because purge can be done. for (int i = 1; i < 5; i++) { DecoratedKey key = Util.dk(String.valueOf(i)); RowMutation rm = new RowMutation(KEYSPACE1, key.key); rm.add( "Standard2", ByteBufferUtil.bytes(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i); rm.apply(); } cfs.forceBlockingFlush(); SSTableReader tmpSSTable = null; for (SSTableReader sstable : cfs.getSSTables()) if (!toCompact.contains(sstable)) tmpSSTable = sstable; assert tmpSSTable != null; // Force compaction on first sstables. Since each row is in only one sstable, we will be using // EchoedRow. Util.compact(cfs, toCompact); assertEquals(2, cfs.getSSTables().size()); // Now, we remove the sstable that was just created to force the use of EchoedRow (so that it // doesn't hide the problem) cfs.markObsolete(Collections.singleton(tmpSSTable), OperationType.UNKNOWN); assertEquals(1, cfs.getSSTables().size()); // Now assert we do have the 4 keys assertEquals(4, Util.getRangeSlice(cfs).size()); }
public List<Row> executeLocally() { ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(columnFamily); ExtendedFilter exFilter = cfs.makeExtendedFilter( keyRange, (SliceQueryFilter) predicate, start, stop, rowFilter, limit, timestamp); if (cfs.indexManager.hasIndexFor(rowFilter)) return cfs.search(exFilter); else return cfs.getRangeSlice(exFilter); }
/** * Writes out a bunch of mutations for a single column family. * * @param mutations A group of Mutations for the same keyspace and column family. * @return The ColumnFamilyStore that was used. */ public static ColumnFamilyStore writeColumnFamily(List<Mutation> mutations) { IMutation first = mutations.get(0); String keyspaceName = first.getKeyspaceName(); UUID cfid = first.getColumnFamilyIds().iterator().next(); for (Mutation rm : mutations) rm.applyUnsafe(); ColumnFamilyStore store = Keyspace.open(keyspaceName).getColumnFamilyStore(cfid); store.forceBlockingFlush(); return store; }
@Test public void testCleanupWithIndexes() throws IOException, ExecutionException, InterruptedException { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF1); List<Row> rows; // insert data and verify we get it back w/ range query fillCF(cfs, LOOPS); rows = Util.getRangeSlice(cfs); assertEquals(LOOPS, rows.size()); SecondaryIndex index = cfs.indexManager.getIndexForColumn(COLUMN); long start = System.nanoTime(); while (!index.isIndexBuilt(COLUMN) && System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10)) Thread.sleep(10); // verify we get it back w/ index query too IndexExpression expr = new IndexExpression(COLUMN, IndexExpression.Operator.EQ, VALUE); List<IndexExpression> clause = Arrays.asList(expr); IDiskAtomFilter filter = new IdentityQueryFilter(); IPartitioner p = StorageService.getPartitioner(); Range<RowPosition> range = Util.range("", ""); rows = keyspace.getColumnFamilyStore(CF1).search(range, clause, filter, Integer.MAX_VALUE); assertEquals(LOOPS, rows.size()); // we don't allow cleanup when the local host has no range to avoid wipping up all data when a // node has not join the ring. // So to make sure cleanup erase everything here, we give the localhost the tiniest possible // range. TokenMetadata tmd = StorageService.instance.getTokenMetadata(); byte[] tk1 = new byte[1], tk2 = new byte[1]; tk1[0] = 2; tk2[0] = 1; tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1")); tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2")); CompactionManager.instance.performCleanup(cfs, new CounterId.OneShotRenewer()); // row data should be gone rows = Util.getRangeSlice(cfs); assertEquals(0, rows.size()); // not only should it be gone but there should be no data on disk, not even tombstones assert cfs.getSSTables().isEmpty(); // 2ary indexes should result in no results, too (although tombstones won't be gone until // compacted) rows = cfs.search(range, clause, filter, Integer.MAX_VALUE); assertEquals(0, rows.size()); }
public List<Row> fetchPage(int pageSize) throws RequestValidationException, RequestExecutionException { assert command.filter.countCQL3Rows() || command.filter.columns.size() <= pageSize; if (isExhausted()) { return Collections.<Row>emptyList(); } queried = true; return localQuery ? Collections.singletonList(command.getRow(Keyspace.open(command.ksName))) : StorageProxy.read(Collections.<ReadCommand>singletonList(command), consistencyLevel); }
private void testDontPurgeAccidentaly(String k, String cfname) throws IOException, ExecutionException, InterruptedException { // This test catches the regression of CASSANDRA-2786 Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname); // disable compaction while flushing cfs.clearUnsafe(); cfs.disableAutoCompaction(); // Add test row DecoratedKey key = Util.dk(k); RowMutation rm = new RowMutation(KEYSPACE1, key.key); rm.add( cfname, CompositeType.build(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); rm.apply(); cfs.forceBlockingFlush(); Collection<SSTableReader> sstablesBefore = cfs.getSSTables(); QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis()); assert !(cfs.getColumnFamily(filter).getColumnCount() == 0); // Remove key rm = new RowMutation(KEYSPACE1, key.key); rm.delete(cfname, 2); rm.apply(); ColumnFamily cf = cfs.getColumnFamily(filter); assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf; // Sleep one second so that the removal is indeed purgeable even with gcgrace == 0 Thread.sleep(1000); cfs.forceBlockingFlush(); Collection<SSTableReader> sstablesAfter = cfs.getSSTables(); Collection<SSTableReader> toCompact = new ArrayList<SSTableReader>(); for (SSTableReader sstable : sstablesAfter) if (!sstablesBefore.contains(sstable)) toCompact.add(sstable); Util.compact(cfs, toCompact); cf = cfs.getColumnFamily(filter); assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf; }
public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1"); store.clearUnsafe(); store.metadata.gcGraceSeconds(1); store.setCompactionStrategyClass(strategyClassName); // disable compaction while flushing store.disableAutoCompaction(); long timestamp = System.currentTimeMillis(); for (int i = 0; i < 10; i++) { DecoratedKey key = Util.dk(Integer.toString(i)); RowMutation rm = new RowMutation(KEYSPACE1, key.key); for (int j = 0; j < 10; j++) rm.add( "Standard1", ByteBufferUtil.bytes(Integer.toString(j)), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, j > 0 ? 3 : 0); // let first column never expire, since deleting all columns does not produce // sstable rm.apply(); } store.forceBlockingFlush(); assertEquals(1, store.getSSTables().size()); long originalSize = store.getSSTables().iterator().next().uncompressedLength(); // wait enough to force single compaction TimeUnit.SECONDS.sleep(5); // enable compaction, submit background and wait for it to complete store.enableAutoCompaction(); FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store)); while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1); // and sstable with ttl should be compacted assertEquals(1, store.getSSTables().size()); long size = store.getSSTables().iterator().next().uncompressedLength(); assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize); // make sure max timestamp of compacted sstables is recorded properly after compaction. assertMaxTimestamp(store, timestamp); return store; }
@Test public void testGetSliceWithCollision() throws Exception { Keyspace keyspace = Keyspace.open(KEYSPACE); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF); cfs.clearUnsafe(); insert("k1", "k2", "k3"); // token = 2 insert("key1", "key2", "key3"); // token = 4 insert("longKey1", "longKey2"); // token = 8 List<Row> rows = cfs.getRangeSlice( new Bounds<RowPosition>(dk("k2"), dk("key2")), null, new IdentityQueryFilter(), 10000); assert rows.size() == 4 : "Expecting 4 keys, got " + rows.size(); assert rows.get(0).key.key.equals(ByteBufferUtil.bytes("k2")); assert rows.get(1).key.key.equals(ByteBufferUtil.bytes("k3")); assert rows.get(2).key.key.equals(ByteBufferUtil.bytes("key1")); assert rows.get(3).key.key.equals(ByteBufferUtil.bytes("key2")); }
@Test public void testSuperColumnTombstones() throws IOException, ExecutionException, InterruptedException { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1"); cfs.disableAutoCompaction(); DecoratedKey key = Util.dk("tskey"); ByteBuffer scName = ByteBufferUtil.bytes("TestSuperColumn"); // a subcolumn RowMutation rm = new RowMutation(KEYSPACE1, key.key); rm.add( "Super1", CompositeType.build(scName, ByteBufferUtil.bytes(0)), ByteBufferUtil.EMPTY_BYTE_BUFFER, FBUtilities.timestampMicros()); rm.apply(); cfs.forceBlockingFlush(); // shadow the subcolumn with a supercolumn tombstone rm = new RowMutation(KEYSPACE1, key.key); rm.deleteRange( "Super1", SuperColumns.startOf(scName), SuperColumns.endOf(scName), FBUtilities.timestampMicros()); rm.apply(); cfs.forceBlockingFlush(); CompactionManager.instance.performMaximal(cfs); assertEquals(1, cfs.getSSTables().size()); // check that the shadowed column is gone SSTableReader sstable = cfs.getSSTables().iterator().next(); Range keyRange = new Range<RowPosition>(key, sstable.partitioner.getMinimumToken().maxKeyBound()); SSTableScanner scanner = sstable.getScanner(DataRange.forKeyRange(keyRange)); OnDiskAtomIterator iter = scanner.next(); assertEquals(key, iter.getKey()); assert iter.next() instanceof RangeTombstone; assert !iter.hasNext(); }
@Test public void testRowCacheCleanup() throws Exception { StorageService.instance.initServer(0); CacheService.instance.setRowCacheCapacityInMB(1); rowCacheLoad(100, Integer.MAX_VALUE, 1000); ColumnFamilyStore store = Keyspace.open(KEYSPACE_CACHED).getColumnFamilyStore(CF_CACHED); assertEquals(CacheService.instance.rowCache.size(), 100); store.cleanupCache(); assertEquals(CacheService.instance.rowCache.size(), 100); TokenMetadata tmd = StorageService.instance.getTokenMetadata(); byte[] tk1, tk2; tk1 = "key1000".getBytes(); tk2 = "key1050".getBytes(); tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1")); tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2")); store.cleanupCache(); assertEquals(50, CacheService.instance.rowCache.size()); CacheService.instance.setRowCacheCapacityInMB(0); }
public void bootstrap() { if (logger.isDebugEnabled()) logger.debug("Beginning bootstrap process"); RangeStreamer streamer = new RangeStreamer(tokenMetadata, tokens, address, "Bootstrap"); streamer.addSourceFilter(new RangeStreamer.FailureDetectorSourceFilter(FailureDetector.instance)); for (String keyspaceName : Schema.instance.getNonSystemKeyspaces()) { AbstractReplicationStrategy strategy = Keyspace.open(keyspaceName).getReplicationStrategy(); streamer.addRanges(keyspaceName, strategy.getPendingAddressRanges(tokenMetadata, tokens, address)); } try { streamer.fetchAsync().get(); StorageService.instance.finishBootstrapping(); } }
@Test public void testUserDefinedCompaction() throws Exception { Keyspace keyspace = Keyspace.open(KEYSPACE1); final String cfname = "Standard3"; // use clean(no sstable) CF ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname); // disable compaction while flushing cfs.disableAutoCompaction(); final int ROWS_PER_SSTABLE = 10; for (int i = 0; i < ROWS_PER_SSTABLE; i++) { DecoratedKey key = Util.dk(String.valueOf(i)); Mutation rm = new Mutation(KEYSPACE1, key.getKey()); rm.add( cfname, Util.cellname("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); rm.applyUnsafe(); } cfs.forceBlockingFlush(); Collection<SSTableReader> sstables = cfs.getSSTables(); assertEquals(1, sstables.size()); SSTableReader sstable = sstables.iterator().next(); int prevGeneration = sstable.descriptor.generation; String file = new File(sstable.descriptor.filenameFor(Component.DATA)).getAbsolutePath(); // submit user defined compaction on flushed sstable CompactionManager.instance.forceUserDefinedCompaction(file); // wait until user defined compaction finishes do { Thread.sleep(100); } while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0); // CF should have only one sstable with generation number advanced sstables = cfs.getSSTables(); assertEquals(1, sstables.size()); assertEquals(prevGeneration + 1, sstables.iterator().next().descriptor.generation); }
// Test compaction of hints column family. It shouldn't remove all columns on compaction. @Test public void testCompactionOfHintsCF() throws Exception { // prepare hints column family Keyspace systemKeyspace = Keyspace.open("system"); ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS_CF); hintStore.clearUnsafe(); hintStore.metadata.gcGraceSeconds(36000); // 10 hours hintStore.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getCanonicalName()); hintStore.disableAutoCompaction(); // insert 1 hint RowMutation rm = new RowMutation(KEYSPACE4, ByteBufferUtil.bytes(1)); rm.add( STANDARD1_CF, ByteBufferUtil.bytes(String.valueOf(COLUMN1)), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); HintedHandOffManager.instance .hintFor( rm, System.currentTimeMillis(), HintedHandOffManager.calculateHintTTL(rm), UUID.randomUUID()) .apply(); // flush data to disk hintStore.forceBlockingFlush(); assertEquals(1, hintStore.getSSTables().size()); // submit compaction FBUtilities.waitOnFuture(HintedHandOffManager.instance.compact()); while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1); // single row should not be removed because of gc_grace_seconds // is 10 hours and there are no any tombstones in sstable assertEquals(1, hintStore.getSSTables().size()); }
public void rowCacheLoad(int totalKeys, int keysToSave, int offset) throws Exception { CompactionManager.instance.disableAutoCompaction(); ColumnFamilyStore store = Keyspace.open(KEYSPACE_CACHED).getColumnFamilyStore(CF_CACHED); // empty the cache CacheService.instance.invalidateRowCache(); assertEquals(0, CacheService.instance.rowCache.size()); // insert data and fill the cache SchemaLoader.insertData(KEYSPACE_CACHED, CF_CACHED, offset, totalKeys); SchemaLoader.readData(KEYSPACE_CACHED, CF_CACHED, offset, totalKeys); assertEquals(totalKeys, CacheService.instance.rowCache.size()); // force the cache to disk CacheService.instance.rowCache.submitWrite(keysToSave).get(); // empty the cache again to make sure values came from disk CacheService.instance.invalidateRowCache(); assertEquals(0, CacheService.instance.rowCache.size()); assertEquals( keysToSave == Integer.MAX_VALUE ? totalKeys : keysToSave, CacheService.instance.rowCache.loadSaved()); }
@Test public void testUncheckedTombstoneSizeTieredCompaction() throws Exception { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1); store.clearUnsafe(); store.metadata.gcGraceSeconds(1); store.metadata.compactionStrategyOptions.put("tombstone_compaction_interval", "1"); store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "false"); store.reload(); store.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getName()); // disable compaction while flushing store.disableAutoCompaction(); // Populate sstable1 with with keys [0..9] populate(KEYSPACE1, CF_STANDARD1, 0, 9, 3); // ttl=3s store.forceBlockingFlush(); // Populate sstable2 with with keys [10..19] (keys do not overlap with SSTable1) long timestamp2 = populate(KEYSPACE1, CF_STANDARD1, 10, 19, 3); // ttl=3s store.forceBlockingFlush(); assertEquals(2, store.getSSTables().size()); Iterator<SSTableReader> it = store.getSSTables().iterator(); long originalSize1 = it.next().uncompressedLength(); long originalSize2 = it.next().uncompressedLength(); // wait enough to force single compaction TimeUnit.SECONDS.sleep(5); // enable compaction, submit background and wait for it to complete store.enableAutoCompaction(); FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store)); while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1); // even though both sstables were candidate for tombstone compaction // it was not executed because they have an overlapping token range assertEquals(2, store.getSSTables().size()); it = store.getSSTables().iterator(); long newSize1 = it.next().uncompressedLength(); long newSize2 = it.next().uncompressedLength(); assertEquals( "candidate sstable should not be tombstone-compacted because its key range overlap with other sstable", originalSize1, newSize1); assertEquals( "candidate sstable should not be tombstone-compacted because its key range overlap with other sstable", originalSize2, newSize2); // now let's enable the magic property store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "true"); store.reload(); // submit background task again and wait for it to complete FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store)); while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1); // we still have 2 sstables, since they were not compacted against each other assertEquals(2, store.getSSTables().size()); it = store.getSSTables().iterator(); newSize1 = it.next().uncompressedLength(); newSize2 = it.next().uncompressedLength(); assertTrue( "should be less than " + originalSize1 + ", but was " + newSize1, newSize1 < originalSize1); assertTrue( "should be less than " + originalSize2 + ", but was " + newSize2, newSize2 < originalSize2); // make sure max timestamp of compacted sstables is recorded properly after compaction. assertMaxTimestamp(store, timestamp2); }
@Test public void testRowCache() throws Exception { CompactionManager.instance.disableAutoCompaction(); Keyspace keyspace = Keyspace.open(KEYSPACE_CACHED); ColumnFamilyStore cachedStore = keyspace.getColumnFamilyStore(CF_CACHED); // empty the row cache CacheService.instance.invalidateRowCache(); // set global row cache size to 1 MB CacheService.instance.setRowCacheCapacityInMB(1); // inserting 100 rows into both column families SchemaLoader.insertData(KEYSPACE_CACHED, CF_CACHED, 0, 100); // now reading rows one by one and checking if row change grows for (int i = 0; i < 100; i++) { DecoratedKey key = Util.dk("key" + i); cachedStore.getColumnFamily( key, Composites.EMPTY, Composites.EMPTY, false, 1, System.currentTimeMillis()); assert CacheService.instance.rowCache.size() == i + 1; assert cachedStore.containsCachedRow(key); // current key should be stored in the cache // checking if cell is read correctly after cache ColumnFamily cf = cachedStore.getColumnFamily( key, Composites.EMPTY, Composites.EMPTY, false, 1, System.currentTimeMillis()); Collection<Cell> cells = cf.getSortedColumns(); Cell cell = cells.iterator().next(); assert cells.size() == 1; assert cell.name().toByteBuffer().equals(ByteBufferUtil.bytes("col" + i)); assert cell.value().equals(ByteBufferUtil.bytes("val" + i)); } // insert 10 more keys SchemaLoader.insertData(KEYSPACE_CACHED, CF_CACHED, 100, 10); for (int i = 100; i < 110; i++) { DecoratedKey key = Util.dk("key" + i); cachedStore.getColumnFamily( key, Composites.EMPTY, Composites.EMPTY, false, 1, System.currentTimeMillis()); assert cachedStore.containsCachedRow( key); // cache should be populated with the latest rows read (old ones should be popped) // checking if cell is read correctly after cache ColumnFamily cf = cachedStore.getColumnFamily( key, Composites.EMPTY, Composites.EMPTY, false, 1, System.currentTimeMillis()); Collection<Cell> cells = cf.getSortedColumns(); Cell cell = cells.iterator().next(); assert cells.size() == 1; assert cell.name().toByteBuffer().equals(ByteBufferUtil.bytes("col" + i)); assert cell.value().equals(ByteBufferUtil.bytes("val" + i)); } // clear 100 rows from the cache int keysLeft = 109; for (int i = 109; i >= 10; i--) { cachedStore.invalidateCachedRow(Util.dk("key" + i)); assert CacheService.instance.rowCache.size() == keysLeft; keysLeft--; } CacheService.instance.setRowCacheCapacityInMB(0); }
@Test @Ignore("making ranges based on the keys, not on the tokens") public void testNeedsCleanup() { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore store = keyspace.getColumnFamilyStore("CF_STANDARD1"); store.clearUnsafe(); // disable compaction while flushing store.disableAutoCompaction(); // write three groups of 9 keys: 001, 002, ... 008, 009 // 101, 102, ... 108, 109 // 201, 202, ... 208, 209 for (int i = 1; i < 10; i++) { insertRowWithKey(i); insertRowWithKey(i + 100); insertRowWithKey(i + 200); } store.forceBlockingFlush(); assertEquals(1, store.getSSTables().size()); SSTableReader sstable = store.getSSTables().iterator().next(); // contiguous range spans all data assertFalse(CompactionManager.needsCleanup(sstable, makeRanges(0, 209))); assertFalse(CompactionManager.needsCleanup(sstable, makeRanges(0, 210))); // separate ranges span all data assertFalse( CompactionManager.needsCleanup( sstable, makeRanges( 0, 9, 100, 109, 200, 209))); assertFalse( CompactionManager.needsCleanup( sstable, makeRanges( 0, 109, 200, 210))); assertFalse( CompactionManager.needsCleanup( sstable, makeRanges( 0, 9, 100, 210))); // one range is missing completely assertTrue( CompactionManager.needsCleanup( sstable, makeRanges( 100, 109, 200, 209))); assertTrue( CompactionManager.needsCleanup( sstable, makeRanges( 0, 9, 200, 209))); assertTrue( CompactionManager.needsCleanup( sstable, makeRanges( 0, 9, 100, 109))); // the beginning of one range is missing assertTrue( CompactionManager.needsCleanup( sstable, makeRanges( 1, 9, 100, 109, 200, 209))); assertTrue( CompactionManager.needsCleanup( sstable, makeRanges( 0, 9, 101, 109, 200, 209))); assertTrue( CompactionManager.needsCleanup( sstable, makeRanges( 0, 9, 100, 109, 201, 209))); // the end of one range is missing assertTrue( CompactionManager.needsCleanup( sstable, makeRanges( 0, 8, 100, 109, 200, 209))); assertTrue( CompactionManager.needsCleanup( sstable, makeRanges( 0, 9, 100, 108, 200, 209))); assertTrue( CompactionManager.needsCleanup( sstable, makeRanges( 0, 9, 100, 109, 200, 208))); // some ranges don't contain any data assertFalse( CompactionManager.needsCleanup( sstable, makeRanges( 0, 0, 0, 9, 50, 51, 100, 109, 150, 199, 200, 209, 300, 301))); // same case, but with a middle range not covering some of the existing data assertFalse( CompactionManager.needsCleanup( sstable, makeRanges( 0, 0, 0, 9, 50, 51, 100, 103, 150, 199, 200, 209, 300, 301))); }
@Test public void testRowCacheRange() { CompactionManager.instance.disableAutoCompaction(); Keyspace keyspace = Keyspace.open(KEYSPACE_CACHED); String cf = "CachedIntCF"; ColumnFamilyStore cachedStore = keyspace.getColumnFamilyStore(cf); long startRowCacheHits = cachedStore.metric.rowCacheHit.getCount(); long startRowCacheOutOfRange = cachedStore.metric.rowCacheHitOutOfRange.getCount(); // empty the row cache CacheService.instance.invalidateRowCache(); // set global row cache size to 1 MB CacheService.instance.setRowCacheCapacityInMB(1); ByteBuffer key = ByteBufferUtil.bytes("rowcachekey"); DecoratedKey dk = cachedStore.partitioner.decorateKey(key); RowCacheKey rck = new RowCacheKey(cachedStore.metadata.ksAndCFName, dk); Mutation mutation = new Mutation(KEYSPACE_CACHED, key); for (int i = 0; i < 200; i++) mutation.add( cf, Util.cellname(i), ByteBufferUtil.bytes("val" + i), System.currentTimeMillis()); mutation.applyUnsafe(); // populate row cache, we should not get a row cache hit; cachedStore.getColumnFamily( QueryFilter.getSliceFilter( dk, cf, Composites.EMPTY, Composites.EMPTY, false, 10, System.currentTimeMillis())); assertEquals(startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); // do another query, limit is 20, which is < 100 that we cache, we should get a hit and it // should be in range cachedStore.getColumnFamily( QueryFilter.getSliceFilter( dk, cf, Composites.EMPTY, Composites.EMPTY, false, 20, System.currentTimeMillis())); assertEquals(++startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); assertEquals(startRowCacheOutOfRange, cachedStore.metric.rowCacheHitOutOfRange.getCount()); // get a slice from 95 to 105, 95->99 are in cache, we should not get a hit and then row cache // is out of range cachedStore.getColumnFamily( QueryFilter.getSliceFilter( dk, cf, CellNames.simpleDense(ByteBufferUtil.bytes(95)), CellNames.simpleDense(ByteBufferUtil.bytes(105)), false, 10, System.currentTimeMillis())); assertEquals(startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); assertEquals(++startRowCacheOutOfRange, cachedStore.metric.rowCacheHitOutOfRange.getCount()); // get a slice with limit > 100, we should get a hit out of range. cachedStore.getColumnFamily( QueryFilter.getSliceFilter( dk, cf, Composites.EMPTY, Composites.EMPTY, false, 101, System.currentTimeMillis())); assertEquals(startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); assertEquals(++startRowCacheOutOfRange, cachedStore.metric.rowCacheHitOutOfRange.getCount()); CacheService.instance.invalidateRowCache(); // try to populate row cache with a limit > rows to cache, we should still populate row cache; cachedStore.getColumnFamily( QueryFilter.getSliceFilter( dk, cf, Composites.EMPTY, Composites.EMPTY, false, 105, System.currentTimeMillis())); assertEquals(startRowCacheHits, cachedStore.metric.rowCacheHit.getCount()); // validate the stuff in cache; ColumnFamily cachedCf = (ColumnFamily) CacheService.instance.rowCache.get(rck); assertEquals(cachedCf.getColumnCount(), 100); int i = 0; for (Cell c : cachedCf) { assertEquals(c.name(), Util.cellname(i++)); } }
@Test public void testRangeTombstones() throws IOException, ExecutionException, InterruptedException { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2"); cfs.clearUnsafe(); // disable compaction while flushing cfs.disableAutoCompaction(); final CFMetaData cfmeta = cfs.metadata; Directories dir = cfs.directories; ArrayList<DecoratedKey> keys = new ArrayList<DecoratedKey>(); for (int i = 0; i < 4; i++) { keys.add(Util.dk("" + i)); } ArrayBackedSortedColumns cf = ArrayBackedSortedColumns.factory.create(cfmeta); cf.addColumn(Util.column("01", "a", 1)); // this must not resurrect cf.addColumn(Util.column("a", "a", 3)); cf.deletionInfo() .add( new RangeTombstone( Util.cellname("0"), Util.cellname("b"), 2, (int) (System.currentTimeMillis() / 1000)), cfmeta.comparator); SSTableWriter writer = new SSTableWriter( cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables()), 0, 0, cfs.metadata, StorageService.getPartitioner(), new MetadataCollector(cfs.metadata.comparator)); writer.append(Util.dk("0"), cf); writer.append(Util.dk("1"), cf); writer.append(Util.dk("3"), cf); cfs.addSSTable(writer.closeAndOpenReader()); writer = new SSTableWriter( cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables()), 0, 0, cfs.metadata, StorageService.getPartitioner(), new MetadataCollector(cfs.metadata.comparator)); writer.append(Util.dk("0"), cf); writer.append(Util.dk("1"), cf); writer.append(Util.dk("2"), cf); writer.append(Util.dk("3"), cf); cfs.addSSTable(writer.closeAndOpenReader()); Collection<SSTableReader> toCompact = cfs.getSSTables(); assert toCompact.size() == 2; // Force compaction on first sstables. Since each row is in only one sstable, we will be using // EchoedRow. Util.compact(cfs, toCompact); assertEquals(1, cfs.getSSTables().size()); // Now assert we do have the 4 keys assertEquals(4, Util.getRangeSlice(cfs).size()); ArrayList<DecoratedKey> k = new ArrayList<DecoratedKey>(); for (Row r : Util.getRangeSlice(cfs)) { k.add(r.key); assertEquals(ByteBufferUtil.bytes("a"), r.cf.getColumn(Util.cellname("a")).value()); assertNull(r.cf.getColumn(Util.cellname("01"))); assertEquals(3, r.cf.getColumn(Util.cellname("a")).timestamp()); } for (SSTableReader sstable : cfs.getSSTables()) { StatsMetadata stats = sstable.getSSTableMetadata(); assertEquals(ByteBufferUtil.bytes("0"), stats.minColumnNames.get(0)); assertEquals(ByteBufferUtil.bytes("b"), stats.maxColumnNames.get(0)); } assertEquals(keys, k); }