public void prepare(ColumnFamilyStore cfs) {
if (tree.partitioner() instanceof RandomPartitioner) {
// You can't beat an even tree distribution for md5
tree.init();
} else {
List<DecoratedKey> keys = new ArrayList<DecoratedKey>();
for (DecoratedKey sample : cfs.keySamples(request.range)) {
assert request.range.contains(sample.token)
: "Token " + sample.token + " is not within range " + request.range;
keys.add(sample);
}
if (keys.isEmpty()) {
// use an even tree distribution
tree.init();
} else {
int numkeys = keys.size();
Random random = new Random();
// sample the column family using random keys from the index
while (true) {
DecoratedKey dk = keys.get(random.nextInt(numkeys));
if (!tree.split(dk.token)) break;
}
}
}
logger.debug("Prepared AEService tree of size " + tree.size() + " for " + request);
ranges = tree.invalids();
}
public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1);
store.clearUnsafe();
store.metadata.gcGraceSeconds(1);
store.setCompactionStrategyClass(strategyClassName);
// disable compaction while flushing
store.disableAutoCompaction();
long timestamp = populate(KEYSPACE1, CF_STANDARD1, 0, 9, 3); // ttl=3s
store.forceBlockingFlush();
assertEquals(1, store.getSSTables().size());
long originalSize = store.getSSTables().iterator().next().uncompressedLength();
// wait enough to force single compaction
TimeUnit.SECONDS.sleep(5);
// enable compaction, submit background and wait for it to complete
store.enableAutoCompaction();
FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
while (CompactionManager.instance.getPendingTasks() > 0
|| CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);
// and sstable with ttl should be compacted
assertEquals(1, store.getSSTables().size());
long size = store.getSSTables().iterator().next().uncompressedLength();
assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize);
// make sure max timestamp of compacted sstables is recorded properly after compaction.
assertMaxTimestamp(store, timestamp);
return store;
}
@Test
public void testCompactionLog() throws Exception {
SystemKeyspace.discardCompactionsInProgress();
String cf = "Standard4";
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(cf);
SchemaLoader.insertData(KEYSPACE1, cf, 0, 1);
cfs.forceBlockingFlush();
Collection<SSTableReader> sstables = cfs.getSSTables();
assertFalse(sstables.isEmpty());
Set<Integer> generations =
Sets.newHashSet(
Iterables.transform(
sstables,
new Function<SSTableReader, Integer>() {
public Integer apply(SSTableReader sstable) {
return sstable.descriptor.generation;
}
}));
UUID taskId = SystemKeyspace.startCompaction(cfs, sstables);
Map<Pair<String, String>, Map<Integer, UUID>> compactionLogs =
SystemKeyspace.getUnfinishedCompactions();
Set<Integer> unfinishedCompactions = compactionLogs.get(Pair.create(KEYSPACE1, cf)).keySet();
assertTrue(unfinishedCompactions.containsAll(generations));
SystemKeyspace.finishCompaction(taskId);
compactionLogs = SystemKeyspace.getUnfinishedCompactions();
assertFalse(compactionLogs.containsKey(Pair.create(KEYSPACE1, cf)));
}
@Test
public void testSingleSSTableCompactionWithLeveledCompaction() throws Exception {
ColumnFamilyStore store =
testSingleSSTableCompaction(LeveledCompactionStrategy.class.getCanonicalName());
LeveledCompactionStrategy strategy = (LeveledCompactionStrategy) store.getCompactionStrategy();
// tombstone removal compaction should not promote level
assertEquals(1, strategy.getLevelSize(0));
}
@Test
public void testEchoedRow() throws IOException, ExecutionException, InterruptedException {
// This test check that EchoedRow doesn't skipp rows: see CASSANDRA-2653
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2");
// disable compaction while flushing
cfs.disableAutoCompaction();
// Insert 4 keys in two sstables. We need the sstables to have 2 rows
// at least to trigger what was causing CASSANDRA-2653
for (int i = 1; i < 5; i++) {
DecoratedKey key = Util.dk(String.valueOf(i));
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
rm.add(
"Standard2",
ByteBufferUtil.bytes(String.valueOf(i)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
i);
rm.apply();
if (i % 2 == 0) cfs.forceBlockingFlush();
}
Collection<SSTableReader> toCompact = cfs.getSSTables();
assert toCompact.size() == 2;
// Reinserting the same keys. We will compact only the previous sstable, but we need those new
// ones
// to make sure we use EchoedRow, otherwise it won't be used because purge can be done.
for (int i = 1; i < 5; i++) {
DecoratedKey key = Util.dk(String.valueOf(i));
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
rm.add(
"Standard2",
ByteBufferUtil.bytes(String.valueOf(i)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
i);
rm.apply();
}
cfs.forceBlockingFlush();
SSTableReader tmpSSTable = null;
for (SSTableReader sstable : cfs.getSSTables())
if (!toCompact.contains(sstable)) tmpSSTable = sstable;
assert tmpSSTable != null;
// Force compaction on first sstables. Since each row is in only one sstable, we will be using
// EchoedRow.
Util.compact(cfs, toCompact);
assertEquals(2, cfs.getSSTables().size());
// Now, we remove the sstable that was just created to force the use of EchoedRow (so that it
// doesn't hide the problem)
cfs.markObsolete(Collections.singleton(tmpSSTable), OperationType.UNKNOWN);
assertEquals(1, cfs.getSSTables().size());
// Now assert we do have the 4 keys
assertEquals(4, Util.getRangeSlice(cfs).size());
}
/**
* Writes out a bunch of mutations for a single column family.
*
* @param mutations A group of Mutations for the same keyspace and column family.
* @return The ColumnFamilyStore that was used.
*/
public static ColumnFamilyStore writeColumnFamily(List<Mutation> mutations) {
IMutation first = mutations.get(0);
String keyspaceName = first.getKeyspaceName();
UUID cfid = first.getColumnFamilyIds().iterator().next();
for (Mutation rm : mutations) rm.applyUnsafe();
ColumnFamilyStore store = Keyspace.open(keyspaceName).getColumnFamilyStore(cfid);
store.forceBlockingFlush();
return store;
}
/**
* Writes out a bunch of rows for a single column family.
*
* @param rows A group of RowMutations for the same table and column family.
* @return The ColumnFamilyStore that was used.
*/
public static ColumnFamilyStore writeColumnFamily(List<IMutation> rms)
throws IOException, ExecutionException, InterruptedException {
IMutation first = rms.get(0);
String tablename = first.getTable();
UUID cfid = first.getColumnFamilyIds().iterator().next();
for (IMutation rm : rms) rm.apply();
ColumnFamilyStore store = Table.open(tablename).getColumnFamilyStore(cfid);
store.forceBlockingFlush();
return store;
}
public static SSTableReader sstable(
int generation, int size, boolean keepRef, ColumnFamilyStore cfs) {
Descriptor descriptor =
new Descriptor(
cfs.getDirectories().getDirectoryForNewSSTables(),
cfs.keyspace.getName(),
cfs.getColumnFamilyName(),
generation);
Set<Component> components =
ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
for (Component component : components) {
File file = new File(descriptor.filenameFor(component));
try {
file.createNewFile();
} catch (IOException e) {
}
}
if (size > 0) {
try {
File file = new File(descriptor.filenameFor(Component.DATA));
try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
raf.setLength(size);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.emptyList());
StatsMetadata metadata =
(StatsMetadata)
new MetadataCollector(cfs.metadata.comparator)
.finalizeMetadata(
cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
.get(MetadataType.STATS);
SSTableReader reader =
SSTableReader.internalOpen(
descriptor,
components,
cfs.metadata,
segmentedFile.sharedCopy(),
segmentedFile.sharedCopy(),
indexSummary.sharedCopy(),
new AlwaysPresentFilter(),
1L,
metadata,
SSTableReader.OpenReason.NORMAL,
header);
reader.first = reader.last = readerBounds(generation);
if (!keepRef) reader.selfRef().release();
return reader;
}
private void testDontPurgeAccidentaly(String k, String cfname)
throws IOException, ExecutionException, InterruptedException {
// This test catches the regression of CASSANDRA-2786
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
// disable compaction while flushing
cfs.clearUnsafe();
cfs.disableAutoCompaction();
// Add test row
DecoratedKey key = Util.dk(k);
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
rm.add(
cfname,
CompositeType.build(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
0);
rm.apply();
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesBefore = cfs.getSSTables();
QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis());
assert !(cfs.getColumnFamily(filter).getColumnCount() == 0);
// Remove key
rm = new RowMutation(KEYSPACE1, key.key);
rm.delete(cfname, 2);
rm.apply();
ColumnFamily cf = cfs.getColumnFamily(filter);
assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;
// Sleep one second so that the removal is indeed purgeable even with gcgrace == 0
Thread.sleep(1000);
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesAfter = cfs.getSSTables();
Collection<SSTableReader> toCompact = new ArrayList<SSTableReader>();
for (SSTableReader sstable : sstablesAfter)
if (!sstablesBefore.contains(sstable)) toCompact.add(sstable);
Util.compact(cfs, toCompact);
cf = cfs.getColumnFamily(filter);
assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;
}
public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
store.clearUnsafe();
store.metadata.gcGraceSeconds(1);
store.setCompactionStrategyClass(strategyClassName);
// disable compaction while flushing
store.disableAutoCompaction();
long timestamp = System.currentTimeMillis();
for (int i = 0; i < 10; i++) {
DecoratedKey key = Util.dk(Integer.toString(i));
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
for (int j = 0; j < 10; j++)
rm.add(
"Standard1",
ByteBufferUtil.bytes(Integer.toString(j)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
timestamp,
j > 0
? 3
: 0); // let first column never expire, since deleting all columns does not produce
// sstable
rm.apply();
}
store.forceBlockingFlush();
assertEquals(1, store.getSSTables().size());
long originalSize = store.getSSTables().iterator().next().uncompressedLength();
// wait enough to force single compaction
TimeUnit.SECONDS.sleep(5);
// enable compaction, submit background and wait for it to complete
store.enableAutoCompaction();
FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
while (CompactionManager.instance.getPendingTasks() > 0
|| CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);
// and sstable with ttl should be compacted
assertEquals(1, store.getSSTables().size());
long size = store.getSSTables().iterator().next().uncompressedLength();
assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize);
// make sure max timestamp of compacted sstables is recorded properly after compaction.
assertMaxTimestamp(store, timestamp);
return store;
}
@Test
public void testCheckForExpiredSSTableBlockers() throws InterruptedException {
String KEYSPACE1 = "Keyspace1";
ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
cfs.truncateBlocking();
cfs.disableAutoCompaction();
cfs.metadata.gcGraceSeconds(0);
RowMutation rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
rm.add(
"Standard1",
ByteBufferUtil.bytes("col1"),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
System.currentTimeMillis());
rm.applyUnsafe();
cfs.forceBlockingFlush();
SSTableReader blockingSSTable = cfs.getSSTables().iterator().next();
for (int i = 0; i < 10; i++) {
rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
rm.delete("Standard1", System.currentTimeMillis());
rm.applyUnsafe();
cfs.forceBlockingFlush();
}
Multimap<SSTableReader, SSTableReader> blockers =
SSTableExpiredBlockers.checkForExpiredSSTableBlockers(
cfs.getSSTables(), (int) (System.currentTimeMillis() / 1000) + 100);
assertEquals(1, blockers.keySet().size());
assertTrue(blockers.keySet().contains(blockingSSTable));
assertEquals(10, blockers.get(blockingSSTable).size());
}
public SizeTieredCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options) {
super(cfs, options);
this.estimatedRemainingTasks = 0;
String optionValue = options.get(MIN_SSTABLE_SIZE_KEY);
minSSTableSize = optionValue == null ? DEFAULT_MIN_SSTABLE_SIZE : Long.parseLong(optionValue);
optionValue = options.get(BUCKET_LOW_KEY);
bucketLow = optionValue == null ? DEFAULT_BUCKET_LOW : Double.parseDouble(optionValue);
optionValue = options.get(BUCKET_HIGH_KEY);
bucketHigh = optionValue == null ? DEFAULT_BUCKET_HIGH : Double.parseDouble(optionValue);
if (bucketHigh <= bucketLow) {
logger.warn(
"Bucket low/high marks for {} incorrect, using defaults.", cfs.getColumnFamilyName());
bucketLow = DEFAULT_BUCKET_LOW;
bucketHigh = DEFAULT_BUCKET_HIGH;
}
cfs.setCompactionThresholds(
cfs.metadata.getMinCompactionThreshold(), cfs.metadata.getMaxCompactionThreshold());
}
@Test
public void testNoExpire() throws ExecutionException, InterruptedException {
ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
cfs.disableAutoCompaction();
cfs.metadata.gcGraceSeconds(0);
long timestamp = System.currentTimeMillis();
RowMutation rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.add(
"Standard1", ByteBufferUtil.bytes("col7"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col3"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
DecoratedKey noTTLKey = Util.dk("nottl");
rm = new RowMutation("Keyspace1", noTTLKey.key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col311"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp);
rm.apply();
cfs.forceBlockingFlush();
Thread.sleep(2000); // wait for ttl to expire
assertEquals(4, cfs.getSSTables().size());
cfs.enableAutoCompaction(true);
assertEquals(1, cfs.getSSTables().size());
SSTableReader sstable = cfs.getSSTables().iterator().next();
SSTableScanner scanner = sstable.getScanner(DataRange.allData(sstable.partitioner));
assertTrue(scanner.hasNext());
while (scanner.hasNext()) {
OnDiskAtomIterator iter = scanner.next();
assertEquals(noTTLKey, iter.getKey());
}
}
public static List<Row> getRangeSlice(ColumnFamilyStore cfs, ByteBuffer superColumn)
throws IOException, ExecutionException, InterruptedException {
Token min = StorageService.getPartitioner().getMinimumToken();
return cfs.getRangeSlice(
superColumn,
new Bounds<Token>(min, min).toRowBounds(),
10000,
new IdentityQueryFilter(),
null);
}
/*
* This excercise in particular the code of #4142
*/
@Test
public void testValidationMultipleSSTablePerLevel() throws Exception {
String ksname = "Keyspace1";
String cfname = "StandardLeveled";
Table table = Table.open(ksname);
ColumnFamilyStore store = table.getColumnFamilyStore(cfname);
ByteBuffer value =
ByteBuffer.wrap(new byte[100 * 1024]); // 100 KB value, make it easy to have multiple files
// Enough data to have a level 1 and 2
int rows = 20;
int columns = 10;
// Adds enough data to trigger multiple sstable per level
for (int r = 0; r < rows; r++) {
DecoratedKey key = Util.dk(String.valueOf(r));
RowMutation rm = new RowMutation(ksname, key.key);
for (int c = 0; c < columns; c++) {
rm.add(new QueryPath(cfname, null, ByteBufferUtil.bytes("column" + c)), value, 0);
}
rm.apply();
store.forceFlush();
}
LeveledCompactionStrategy strat = (LeveledCompactionStrategy) store.getCompactionStrategy();
while (strat.getLevelSize(0) > 0) {
store.forceMajorCompaction();
Thread.sleep(200);
}
// Checking we're not completely bad at math
assert strat.getLevelSize(1) > 0;
assert strat.getLevelSize(2) > 0;
AntiEntropyService.CFPair p = new AntiEntropyService.CFPair(ksname, cfname);
Range<Token> range = new Range<Token>(Util.token(""), Util.token(""));
AntiEntropyService.TreeRequest req =
new AntiEntropyService.TreeRequest("1", FBUtilities.getLocalAddress(), range, p);
AntiEntropyService.Validator validator = new AntiEntropyService.Validator(req);
CompactionManager.instance.submitValidation(store, validator).get();
}
/**
* Starts sending/receiving our list of differences to/from the remote endpoint: creates a
* callback that will be called out of band once the streams complete.
*/
void performStreamingRepair() throws IOException {
logger.info(
"Performing streaming repair of "
+ differences.size()
+ " ranges with "
+ remote
+ " for "
+ range);
ColumnFamilyStore cfstore = Table.open(tablename).getColumnFamilyStore(cfname);
try {
Collection<SSTableReader> sstables = cfstore.getSSTables();
Callback callback = new Callback();
// send ranges to the remote node
StreamOutSession outsession = StreamOutSession.create(tablename, remote, callback);
StreamOut.transferSSTables(outsession, sstables, differences, OperationType.AES);
// request ranges from the remote node
StreamIn.requestRanges(remote, tablename, differences, callback, OperationType.AES);
} catch (Exception e) {
throw new IOException("Streaming repair failed.", e);
}
}
@Test
public void testUserDefinedCompaction() throws Exception {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
final String cfname = "Standard3"; // use clean(no sstable) CF
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
// disable compaction while flushing
cfs.disableAutoCompaction();
final int ROWS_PER_SSTABLE = 10;
for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
DecoratedKey key = Util.dk(String.valueOf(i));
Mutation rm = new Mutation(KEYSPACE1, key.getKey());
rm.add(
cfname,
Util.cellname("col"),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
System.currentTimeMillis());
rm.applyUnsafe();
}
cfs.forceBlockingFlush();
Collection<SSTableReader> sstables = cfs.getSSTables();
assertEquals(1, sstables.size());
SSTableReader sstable = sstables.iterator().next();
int prevGeneration = sstable.descriptor.generation;
String file = new File(sstable.descriptor.filenameFor(Component.DATA)).getAbsolutePath();
// submit user defined compaction on flushed sstable
CompactionManager.instance.forceUserDefinedCompaction(file);
// wait until user defined compaction finishes
do {
Thread.sleep(100);
} while (CompactionManager.instance.getPendingTasks() > 0
|| CompactionManager.instance.getActiveCompactions() > 0);
// CF should have only one sstable with generation number advanced
sstables = cfs.getSSTables();
assertEquals(1, sstables.size());
assertEquals(prevGeneration + 1, sstables.iterator().next().descriptor.generation);
}
public static List<Row> getRangeSlice(ColumnFamilyStore cfs, ByteBuffer superColumn) {
IDiskAtomFilter filter =
superColumn == null
? new IdentityQueryFilter()
: new SliceQueryFilter(
SuperColumns.startOf(superColumn),
SuperColumns.endOf(superColumn),
false,
Integer.MAX_VALUE);
Token min = StorageService.getPartitioner().getMinimumToken();
return cfs.getRangeSlice(Bounds.makeRowBounds(min, min), null, filter, 10000);
}
public void applyModels() throws IOException {
ColumnFamilyStore cfs = Table.open(tableName).getColumnFamilyStore(cfName);
// reinitialize the table.
KSMetaData existing = DatabaseDescriptor.getTableDefinition(tableName);
CFMetaData cfm = existing.cfMetaData().get(cfName);
KSMetaData ksm = makeNewKeyspaceDefinition(existing);
CFMetaData.purge(cfm);
DatabaseDescriptor.setTableDefinition(ksm, newVersion);
if (!clientMode) {
cfs.snapshot(Table.getTimestampedSnapshotName(null));
CompactionManager.instance.getCompactionLock().lock();
cfs.flushLock.lock();
try {
Table.open(ksm.name).dropCf(cfm.cfId);
} finally {
cfs.flushLock.unlock();
CompactionManager.instance.getCompactionLock().unlock();
}
}
}
public CommitLogReplayer() {
this.tablesRecovered = new NonBlockingHashSet<Table>();
this.futures = new ArrayList<Future<?>>();
this.buffer = new byte[4096];
this.invalidMutations = new HashMap<Integer, AtomicInteger>();
// count the number of replayed mutation. We don't really care about atomicity, but we need it
// to be a reference.
this.replayedCount = new AtomicInteger();
// compute per-CF and global replay positions
this.cfPositions = new HashMap<Integer, ReplayPosition>();
for (ColumnFamilyStore cfs : ColumnFamilyStore.all()) {
// it's important to call RP.gRP per-cf, before aggregating all the positions w/ the
// Ordering.min call
// below: gRP will return NONE if there are no flushed sstables, which is important to have in
// the
// list (otherwise we'll just start replay from the first flush position that we do have,
// which is not correct).
ReplayPosition rp = ReplayPosition.getReplayPosition(cfs.getSSTables());
cfPositions.put(cfs.metadata.cfId, rp);
}
this.globalPosition = Ordering.from(ReplayPosition.comparator).min(cfPositions.values());
this.checksum = new PureJavaCrc32();
}
/** Gets the estimated total amount of data to write during compaction */
private static long getTotalWriteSize(
Iterable<SSTableReader> nonExpiredSSTables,
long estimatedTotalKeys,
ColumnFamilyStore cfs,
OperationType compactionType) {
long estimatedKeysBeforeCompaction = 0;
for (SSTableReader sstable : nonExpiredSSTables)
estimatedKeysBeforeCompaction += sstable.estimatedKeys();
estimatedKeysBeforeCompaction = Math.max(1, estimatedKeysBeforeCompaction);
double estimatedCompactionRatio = (double) estimatedTotalKeys / estimatedKeysBeforeCompaction;
return Math.round(
estimatedCompactionRatio
* cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType));
}
private static SSTableReader sstable(
File dataFolder, ColumnFamilyStore cfs, int generation, int size) throws IOException {
Descriptor descriptor =
new Descriptor(dataFolder, cfs.keyspace.getName(), cfs.getTableName(), generation);
Set<Component> components =
ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
for (Component component : components) {
File file = new File(descriptor.filenameFor(component));
if (!file.exists()) assertTrue(file.createNewFile());
try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
raf.setLength(size);
}
}
SegmentedFile dFile =
new BufferedSegmentedFile(
new ChannelProxy(new File(descriptor.filenameFor(Component.DATA))),
RandomAccessReader.DEFAULT_BUFFER_SIZE,
0);
SegmentedFile iFile =
new BufferedSegmentedFile(
new ChannelProxy(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))),
RandomAccessReader.DEFAULT_BUFFER_SIZE,
0);
SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.emptyList());
StatsMetadata metadata =
(StatsMetadata)
new MetadataCollector(cfs.metadata.comparator)
.finalizeMetadata(
cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
.get(MetadataType.STATS);
SSTableReader reader =
SSTableReader.internalOpen(
descriptor,
components,
cfs.metadata,
dFile,
iFile,
MockSchema.indexSummary.sharedCopy(),
new AlwaysPresentFilter(),
1L,
metadata,
SSTableReader.OpenReason.NORMAL,
header);
reader.first = reader.last = MockSchema.readerBounds(generation);
return reader;
}
@Test
public void testSimpleExpire() throws ExecutionException, InterruptedException {
ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
cfs.disableAutoCompaction();
cfs.metadata.gcGraceSeconds(0);
long timestamp = System.currentTimeMillis();
RowMutation rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.add(
"Standard1", ByteBufferUtil.bytes("col7"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1", ByteBufferUtil.bytes("col3"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
rm.apply();
cfs.forceBlockingFlush();
rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
rm.add(
"Standard1",
ByteBufferUtil.bytes("col311"),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
timestamp,
1);
rm.apply();
cfs.forceBlockingFlush();
Thread.sleep(2000); // wait for ttl to expire
assertEquals(4, cfs.getSSTables().size());
cfs.enableAutoCompaction(true);
assertEquals(0, cfs.getSSTables().size());
}
@Test
public void testAggressiveFullyExpired() {
String KEYSPACE1 = "Keyspace1";
ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
cfs.disableAutoCompaction();
cfs.metadata.gcGraceSeconds(0);
DecoratedKey ttlKey = Util.dk("ttl");
RowMutation rm = new RowMutation("Keyspace1", ttlKey.key);
rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 1, 1);
rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 3, 1);
rm.applyUnsafe();
cfs.forceBlockingFlush();
rm = new RowMutation(KEYSPACE1, ttlKey.key);
rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 2, 1);
rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 5, 1);
rm.applyUnsafe();
cfs.forceBlockingFlush();
rm = new RowMutation(KEYSPACE1, ttlKey.key);
rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 4, 1);
rm.add("Standard1", ByteBufferUtil.bytes("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 7, 1);
rm.applyUnsafe();
cfs.forceBlockingFlush();
rm = new RowMutation(KEYSPACE1, ttlKey.key);
rm.add("Standard1", ByteBufferUtil.bytes("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 6, 3);
rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 8, 1);
rm.applyUnsafe();
cfs.forceBlockingFlush();
Set<SSTableReader> sstables = Sets.newHashSet(cfs.getSSTables());
int now = (int) (System.currentTimeMillis() / 1000);
int gcBefore = now + 2;
Set<SSTableReader> expired =
CompactionController.getFullyExpiredSSTables(
cfs, sstables, Collections.EMPTY_SET, gcBefore);
assertEquals(2, expired.size());
cfs.clearUnsafe();
}
@Test
public void testSuperColumnTombstones()
throws IOException, ExecutionException, InterruptedException {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1");
cfs.disableAutoCompaction();
DecoratedKey key = Util.dk("tskey");
ByteBuffer scName = ByteBufferUtil.bytes("TestSuperColumn");
// a subcolumn
RowMutation rm = new RowMutation(KEYSPACE1, key.key);
rm.add(
"Super1",
CompositeType.build(scName, ByteBufferUtil.bytes(0)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
FBUtilities.timestampMicros());
rm.apply();
cfs.forceBlockingFlush();
// shadow the subcolumn with a supercolumn tombstone
rm = new RowMutation(KEYSPACE1, key.key);
rm.deleteRange(
"Super1",
SuperColumns.startOf(scName),
SuperColumns.endOf(scName),
FBUtilities.timestampMicros());
rm.apply();
cfs.forceBlockingFlush();
CompactionManager.instance.performMaximal(cfs);
assertEquals(1, cfs.getSSTables().size());
// check that the shadowed column is gone
SSTableReader sstable = cfs.getSSTables().iterator().next();
Range keyRange =
new Range<RowPosition>(key, sstable.partitioner.getMinimumToken().maxKeyBound());
SSTableScanner scanner = sstable.getScanner(DataRange.forKeyRange(keyRange));
OnDiskAtomIterator iter = scanner.next();
assertEquals(key, iter.getKey());
assert iter.next() instanceof RangeTombstone;
assert !iter.hasNext();
}
public static QueryFilter namesQueryFilter(
ColumnFamilyStore cfs, DecoratedKey key, CellName... names) {
SortedSet<CellName> s = new TreeSet<CellName>(cfs.getComparator());
for (CellName n : names) s.add(n);
return QueryFilter.getNamesFilter(key, cfs.name, s, System.currentTimeMillis());
}
public static NamesQueryFilter namesFilter(ColumnFamilyStore cfs, String... names) {
SortedSet<CellName> s = new TreeSet<CellName>(cfs.getComparator());
for (String str : names) s.add(cellname(str));
return new NamesQueryFilter(s);
}
/**
* Retrieves a local subBlock
*
* @param blockId row key
* @param sblockId SubBlock column name
* @param offset inside the sblock
* @return a local sublock
* @throws TException
*/
private LocalBlock getLocalSubBlock(
String subBlockCFName, ByteBuffer blockId, ByteBuffer sblockId, int offset)
throws TException {
DecoratedKey<Token<?>> decoratedKey =
new DecoratedKey<Token<?>>(StorageService.getPartitioner().getToken(blockId), blockId);
Table table = Table.open(cfsKeyspace);
ColumnFamilyStore sblockStore = table.getColumnFamilyStore(subBlockCFName);
Collection<SSTableReader> sstables = sblockStore.getSSTables();
for (SSTableReader sstable : sstables) {
long position = sstable.getPosition(decoratedKey, Operator.EQ);
if (position == -1) continue;
String filename = sstable.descriptor.filenameFor(Component.DATA);
RandomAccessFile raf = null;
int mappedLength = -1;
MappedByteBuffer mappedData = null;
MappedFileDataInput file = null;
try {
raf = new RandomAccessFile(filename, "r");
assert position < raf.length();
mappedLength =
(raf.length() - position) < Integer.MAX_VALUE
? (int) (raf.length() - position)
: Integer.MAX_VALUE;
mappedData = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, position, mappedLength);
file = new MappedFileDataInput(mappedData, filename, 0);
if (file == null) continue;
// Verify key was found in data file
DecoratedKey keyInDisk =
SSTableReader.decodeKey(
sstable.partitioner, sstable.descriptor, ByteBufferUtil.readWithShortLength(file));
assert keyInDisk.equals(decoratedKey)
: String.format("%s != %s in %s", keyInDisk, decoratedKey, file.getPath());
long rowSize = SSTableReader.readRowSize(file, sstable.descriptor);
assert rowSize > 0;
assert rowSize < mappedLength;
Filter bf = IndexHelper.defreezeBloomFilter(file, sstable.descriptor.usesOldBloomFilter);
// verify this column in in this version of the row.
if (!bf.isPresent(sblockId)) continue;
List<IndexHelper.IndexInfo> indexList = IndexHelper.deserializeIndex(file);
// we can stop early if bloom filter says none of the
// columns actually exist -- but,
// we can't stop before initializing the cf above, in
// case there's a relevant tombstone
ColumnFamilySerializer serializer = ColumnFamily.serializer();
try {
ColumnFamily cf =
serializer.deserializeFromSSTableNoColumns(
ColumnFamily.create(sstable.metadata), file);
if (cf.isMarkedForDelete()) continue;
} catch (Exception e) {
e.printStackTrace();
throw new IOException(
serializer
+ " failed to deserialize "
+ sstable.getColumnFamilyName()
+ " with "
+ sstable.metadata
+ " from "
+ file,
e);
}
Integer sblockLength = null;
if (indexList == null) sblockLength = seekToSubColumn(sstable.metadata, file, sblockId);
else sblockLength = seekToSubColumn(sstable.metadata, file, sblockId, indexList);
if (sblockLength == null || sblockLength < 0) continue;
int bytesReadFromStart = mappedLength - (int) file.bytesRemaining();
if (logger.isDebugEnabled())
logger.debug("BlockLength = " + sblockLength + " Availible " + file.bytesRemaining());
assert offset <= sblockLength : String.format("%d > %d", offset, sblockLength);
long dataOffset = position + bytesReadFromStart;
if (file.bytesRemaining() == 0 || sblockLength == 0) continue;
return new LocalBlock(file.getPath(), dataOffset + offset, sblockLength - offset);
} catch (IOException e) {
throw new TException(e);
} finally {
FileUtils.closeQuietly(raf);
}
}
return null;
}
public static void main(String args[]) throws IOException {
Options options = Options.parseArgs(args);
try {
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
String ksName = null;
String cfName = null;
Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
for (String filename : options.filenames) {
File file = new File(filename);
if (!file.exists()) {
System.out.println("Skipping inexisting file " + file);
continue;
}
Pair<Descriptor, Component> pair =
SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
if (pair == null) {
System.out.println("Skipping non sstable file " + file);
continue;
}
Descriptor desc = pair.left;
if (ksName == null) ksName = desc.ksname;
else if (!ksName.equals(desc.ksname))
throw new IllegalArgumentException("All sstables must be part of the same keyspace");
if (cfName == null) cfName = desc.cfname;
else if (!cfName.equals(desc.cfname))
throw new IllegalArgumentException("All sstables must be part of the same column family");
Set<Component> components =
new HashSet<Component>(
Arrays.asList(
new Component[] {
Component.DATA,
Component.PRIMARY_INDEX,
Component.FILTER,
Component.COMPRESSION_INFO,
Component.STATS
}));
Iterator<Component> iter = components.iterator();
while (iter.hasNext()) {
Component component = iter.next();
if (!(new File(desc.filenameFor(component)).exists())) iter.remove();
}
parsedFilenames.put(desc, components);
}
if (ksName == null || cfName == null) {
System.err.println("No valid sstables to split");
System.exit(1);
}
// Do not load sstables since they might be broken
Table table = Table.openWithoutSSTables(ksName);
ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
String snapshotName = "pre-split-" + System.currentTimeMillis();
List<SSTableReader> sstables = new ArrayList<SSTableReader>();
for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet()) {
try {
SSTableReader sstable =
SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs.metadata);
sstables.add(sstable);
if (options.snapshot) {
File snapshotDirectory =
Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
sstable.createLinks(snapshotDirectory.getPath());
}
} catch (Exception e) {
System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
if (options.snapshot)
System.out.println(
String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));
cfs.getDataTracker().markCompacting(sstables);
for (SSTableReader sstable : sstables) {
try {
new SSTableSplitter(cfs, sstable, options.sizeInMB).split();
// Remove the sstable
sstable.markCompacted();
sstable.releaseReference();
} catch (Exception e) {
System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
SSTableDeletingTask.waitForDeletions();
System.exit(0); // We need that to stop non daemonized threads
} catch (Exception e) {
System.err.println(e.getMessage());
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}
ColumnFamilyMetricNameFactory(ColumnFamilyStore cfs) {
this.keyspaceName = cfs.keyspace.getName();
this.columnFamilyName = cfs.name;
isIndex = cfs.isIndex();
}
