private static void insertRowWithKey(int key) {
long timestamp = System.currentTimeMillis();
DecoratedKey decoratedKey = Util.dk(String.format("%03d", key));
Mutation rm = new Mutation(KEYSPACE1, decoratedKey.getKey());
rm.add("CF_STANDARD1", Util.cellname("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1000);
rm.applyUnsafe();
}
@Test
public void testEchoedRow() {
// This test check that EchoedRow doesn't skipp rows: see CASSANDRA-2653
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2");
// disable compaction while flushing
cfs.disableAutoCompaction();
// Insert 4 keys in two sstables. We need the sstables to have 2 rows
// at least to trigger what was causing CASSANDRA-2653
for (int i = 1; i < 5; i++) {
DecoratedKey key = Util.dk(String.valueOf(i));
Mutation rm = new Mutation(KEYSPACE1, key.getKey());
rm.add("Standard2", Util.cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i);
rm.applyUnsafe();
if (i % 2 == 0) cfs.forceBlockingFlush();
}
Collection<SSTableReader> toCompact = cfs.getSSTables();
assertEquals(2, toCompact.size());
// Reinserting the same keys. We will compact only the previous sstable, but we need those new
// ones
// to make sure we use EchoedRow, otherwise it won't be used because purge can be done.
for (int i = 1; i < 5; i++) {
DecoratedKey key = Util.dk(String.valueOf(i));
Mutation rm = new Mutation(KEYSPACE1, key.getKey());
rm.add("Standard2", Util.cellname(String.valueOf(i)), ByteBufferUtil.EMPTY_BYTE_BUFFER, i);
rm.applyUnsafe();
}
cfs.forceBlockingFlush();
SSTableReader tmpSSTable = null;
for (SSTableReader sstable : cfs.getSSTables())
if (!toCompact.contains(sstable)) tmpSSTable = sstable;
assertNotNull(tmpSSTable);
// Force compaction on first sstables. Since each row is in only one sstable, we will be using
// EchoedRow.
Util.compact(cfs, toCompact);
assertEquals(2, cfs.getSSTables().size());
// Now, we remove the sstable that was just created to force the use of EchoedRow (so that it
// doesn't hide the problem)
cfs.markObsolete(Collections.singleton(tmpSSTable), OperationType.UNKNOWN);
assertEquals(1, cfs.getSSTables().size());
// Now assert we do have the 4 keys
assertEquals(4, Util.getRangeSlice(cfs).size());
}
private void testDontPurgeAccidentaly(String k, String cfname) throws InterruptedException {
// This test catches the regression of CASSANDRA-2786
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
// disable compaction while flushing
cfs.clearUnsafe();
cfs.disableAutoCompaction();
// Add test row
DecoratedKey key = Util.dk(k);
Mutation rm = new Mutation(KEYSPACE1, key.getKey());
rm.add(
cfname,
Util.cellname(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
0);
rm.applyUnsafe();
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesBefore = cfs.getSSTables();
QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis());
assertTrue(cfs.getColumnFamily(filter).hasColumns());
// Remove key
rm = new Mutation(KEYSPACE1, key.getKey());
rm.delete(cfname, 2);
rm.applyUnsafe();
ColumnFamily cf = cfs.getColumnFamily(filter);
assertTrue("should be empty: " + cf, cf == null || !cf.hasColumns());
// Sleep one second so that the removal is indeed purgeable even with gcgrace == 0
Thread.sleep(1000);
cfs.forceBlockingFlush();
Collection<SSTableReader> sstablesAfter = cfs.getSSTables();
Collection<SSTableReader> toCompact = new ArrayList<SSTableReader>();
for (SSTableReader sstable : sstablesAfter)
if (!sstablesBefore.contains(sstable)) toCompact.add(sstable);
Util.compact(cfs, toCompact);
cf = cfs.getColumnFamily(filter);
assertTrue("should be empty: " + cf, cf == null || !cf.hasColumns());
}
private long populate(String ks, String cf, int startRowKey, int endRowKey, int ttl) {
long timestamp = System.currentTimeMillis();
for (int i = startRowKey; i <= endRowKey; i++) {
DecoratedKey key = Util.dk(Integer.toString(i));
Mutation rm = new Mutation(ks, key.getKey());
for (int j = 0; j < 10; j++)
rm.add(
cf,
Util.cellname(Integer.toString(j)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
timestamp,
j > 0
? ttl
: 0); // let first column never expire, since deleting all columns does not produce
// sstable
rm.applyUnsafe();
}
return timestamp;
}
@Test
public void testSuperColumnTombstones() throws ExecutionException, InterruptedException {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1");
cfs.disableAutoCompaction();
DecoratedKey key = Util.dk("tskey");
ByteBuffer scName = ByteBufferUtil.bytes("TestSuperColumn");
// a subcolumn
Mutation rm = new Mutation(KEYSPACE1, key.getKey());
rm.add(
"Super1",
Util.cellname(scName, ByteBufferUtil.bytes(0)),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
FBUtilities.timestampMicros());
rm.applyUnsafe();
cfs.forceBlockingFlush();
// shadow the subcolumn with a supercolumn tombstone
rm = new Mutation(KEYSPACE1, key.getKey());
rm.deleteRange(
"Super1",
SuperColumns.startOf(scName),
SuperColumns.endOf(scName),
FBUtilities.timestampMicros());
rm.applyUnsafe();
cfs.forceBlockingFlush();
CompactionManager.instance.performMaximal(cfs);
assertEquals(1, cfs.getSSTables().size());
// check that the shadowed column is gone
SSTableReader sstable = cfs.getSSTables().iterator().next();
Range keyRange =
new Range<RowPosition>(key, sstable.partitioner.getMinimumToken().maxKeyBound());
SSTableScanner scanner = sstable.getScanner(DataRange.forKeyRange(keyRange));
OnDiskAtomIterator iter = scanner.next();
assertEquals(key, iter.getKey());
assertTrue(iter.next() instanceof RangeTombstone);
assertFalse(iter.hasNext());
}
@Test
public void testUserDefinedCompaction() throws Exception {
Keyspace keyspace = Keyspace.open(KEYSPACE1);
final String cfname = "Standard3"; // use clean(no sstable) CF
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
// disable compaction while flushing
cfs.disableAutoCompaction();
final int ROWS_PER_SSTABLE = 10;
for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
DecoratedKey key = Util.dk(String.valueOf(i));
Mutation rm = new Mutation(KEYSPACE1, key.getKey());
rm.add(
cfname,
Util.cellname("col"),
ByteBufferUtil.EMPTY_BYTE_BUFFER,
System.currentTimeMillis());
rm.applyUnsafe();
}
cfs.forceBlockingFlush();
Collection<SSTableReader> sstables = cfs.getSSTables();
assertEquals(1, sstables.size());
SSTableReader sstable = sstables.iterator().next();
int prevGeneration = sstable.descriptor.generation;
String file = new File(sstable.descriptor.filenameFor(Component.DATA)).getAbsolutePath();
// submit user defined compaction on flushed sstable
CompactionManager.instance.forceUserDefinedCompaction(file);
// wait until user defined compaction finishes
do {
Thread.sleep(100);
} while (CompactionManager.instance.getPendingTasks() > 0
|| CompactionManager.instance.getActiveCompactions() > 0);
// CF should have only one sstable with generation number advanced
sstables = cfs.getSSTables();
assertEquals(1, sstables.size());
assertEquals(prevGeneration + 1, sstables.iterator().next().descriptor.generation);
}
/**
* Called (in order) for every row present in the CF. Hashes the row, and adds it to the tree
* being built.
*
* <p>There are four possible cases: 1. Token is greater than range.right (we haven't generated
* a range for it yet), 2. Token is less than/equal to range.left (the range was valid), 3.
* Token is contained in the range (the range is in progress), 4. No more invalid ranges exist.
*
* <p>TODO: Because we only validate completely empty trees at the moment, we do not bother
* dealing with case 2 and case 4 should result in an error.
*
* <p>Additionally, there is a special case for the minimum token, because although it sorts
* first, it is contained in the last possible range.
*
* @param row The row.
*/
public void add(AbstractCompactedRow row) {
assert request.range.contains(row.key.token)
: row.key.token + " is not contained in " + request.range;
assert lastKey == null || lastKey.compareTo(row.key) < 0
: "row " + row.key + " received out of order wrt " + lastKey;
lastKey = row.key;
if (range == null) range = ranges.next();
// generate new ranges as long as case 1 is true
while (!range.contains(row.key.token)) {
// add the empty hash, and move to the next range
range.addHash(EMPTY_ROW);
range = ranges.next();
}
// case 3 must be true: mix in the hashed row
range.addHash(rowHash(row));
}
/**
* Retrieves a local subBlock
*
* @param blockId row key
* @param sblockId SubBlock column name
* @param offset inside the sblock
* @return a local sublock
* @throws TException
*/
private LocalBlock getLocalSubBlock(
String subBlockCFName, ByteBuffer blockId, ByteBuffer sblockId, int offset)
throws TException {
DecoratedKey<Token<?>> decoratedKey =
new DecoratedKey<Token<?>>(StorageService.getPartitioner().getToken(blockId), blockId);
Table table = Table.open(cfsKeyspace);
ColumnFamilyStore sblockStore = table.getColumnFamilyStore(subBlockCFName);
Collection<SSTableReader> sstables = sblockStore.getSSTables();
for (SSTableReader sstable : sstables) {
long position = sstable.getPosition(decoratedKey, Operator.EQ);
if (position == -1) continue;
String filename = sstable.descriptor.filenameFor(Component.DATA);
RandomAccessFile raf = null;
int mappedLength = -1;
MappedByteBuffer mappedData = null;
MappedFileDataInput file = null;
try {
raf = new RandomAccessFile(filename, "r");
assert position < raf.length();
mappedLength =
(raf.length() - position) < Integer.MAX_VALUE
? (int) (raf.length() - position)
: Integer.MAX_VALUE;
mappedData = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, position, mappedLength);
file = new MappedFileDataInput(mappedData, filename, 0);
if (file == null) continue;
// Verify key was found in data file
DecoratedKey keyInDisk =
SSTableReader.decodeKey(
sstable.partitioner, sstable.descriptor, ByteBufferUtil.readWithShortLength(file));
assert keyInDisk.equals(decoratedKey)
: String.format("%s != %s in %s", keyInDisk, decoratedKey, file.getPath());
long rowSize = SSTableReader.readRowSize(file, sstable.descriptor);
assert rowSize > 0;
assert rowSize < mappedLength;
Filter bf = IndexHelper.defreezeBloomFilter(file, sstable.descriptor.usesOldBloomFilter);
// verify this column in in this version of the row.
if (!bf.isPresent(sblockId)) continue;
List<IndexHelper.IndexInfo> indexList = IndexHelper.deserializeIndex(file);
// we can stop early if bloom filter says none of the
// columns actually exist -- but,
// we can't stop before initializing the cf above, in
// case there's a relevant tombstone
ColumnFamilySerializer serializer = ColumnFamily.serializer();
try {
ColumnFamily cf =
serializer.deserializeFromSSTableNoColumns(
ColumnFamily.create(sstable.metadata), file);
if (cf.isMarkedForDelete()) continue;
} catch (Exception e) {
e.printStackTrace();
throw new IOException(
serializer
+ " failed to deserialize "
+ sstable.getColumnFamilyName()
+ " with "
+ sstable.metadata
+ " from "
+ file,
e);
}
Integer sblockLength = null;
if (indexList == null) sblockLength = seekToSubColumn(sstable.metadata, file, sblockId);
else sblockLength = seekToSubColumn(sstable.metadata, file, sblockId, indexList);
if (sblockLength == null || sblockLength < 0) continue;
int bytesReadFromStart = mappedLength - (int) file.bytesRemaining();
if (logger.isDebugEnabled())
logger.debug("BlockLength = " + sblockLength + " Availible " + file.bytesRemaining());
assert offset <= sblockLength : String.format("%d > %d", offset, sblockLength);
long dataOffset = position + bytesReadFromStart;
if (file.bytesRemaining() == 0 || sblockLength == 0) continue;
return new LocalBlock(file.getPath(), dataOffset + offset, sblockLength - offset);
} catch (IOException e) {
throw new TException(e);
} finally {
FileUtils.closeQuietly(raf);
}
}
return null;
}
public void collectReducedColumns(
ColumnFamily container,
Iterator<Cell> reducedColumns,
DecoratedKey key,
int gcBefore,
long now) {
columnCounter = columnCounter(container.getComparator(), now);
DeletionInfo.InOrderTester tester = container.deletionInfo().inOrderTester(reversed);
while (reducedColumns.hasNext()) {
Cell cell = reducedColumns.next();
if (logger.isTraceEnabled())
logger.trace(
"collecting {} of {}: {}",
columnCounter.live(),
count,
cell.getString(container.getComparator()));
// An expired tombstone will be immediately discarded in memory, and needn't be counted.
// Neither should be any cell shadowed by a range- or a partition tombstone.
if (cell.getLocalDeletionTime() < gcBefore || !columnCounter.count(cell, tester)) continue;
if (columnCounter.live() > count) break;
if (respectTombstoneThresholds()
&& columnCounter.tombstones() > DatabaseDescriptor.getTombstoneFailureThreshold()) {
Tracing.trace(
"Scanned over {} tombstones; query aborted (see tombstone_failure_threshold); slices={}",
DatabaseDescriptor.getTombstoneFailureThreshold(),
getSlicesInfo(container));
throw new TombstoneOverwhelmingException(
columnCounter.tombstones(),
count,
container.metadata().ksName,
container.metadata().cfName,
container.getComparator().getString(cell.name()),
getSlicesInfo(container));
}
container.appendColumn(cell);
}
boolean warnTombstones =
logger.isWarnEnabled()
&& respectTombstoneThresholds()
&& columnCounter.tombstones() > DatabaseDescriptor.getTombstoneWarnThreshold();
if (warnTombstones) {
String msg =
String.format(
"Read %d live and %d tombstone cells in %s.%s for key: %1.512s (see tombstone_warn_threshold). %d columns were requested, slices=%1.512s",
columnCounter.live(),
columnCounter.tombstones(),
container.metadata().ksName,
container.metadata().cfName,
container.metadata().getKeyValidator().getString(key.getKey()),
count,
getSlicesInfo(container));
ClientWarn.warn(msg);
logger.warn(msg);
}
Tracing.trace(
"Read {} live and {} tombstone cells{}",
columnCounter.live(),
columnCounter.tombstones(),
warnTombstones ? " (see tombstone_warn_threshold)" : "");
}