/**
* Tests that the system.peers table is not updated after a node has been removed. (See
* CASSANDRA-6053)
*/
@Test
public void testStateChangeOnRemovedNode() throws UnknownHostException {
StorageService ss = StorageService.instance;
VersionedValue.VersionedValueFactory valueFactory =
new VersionedValue.VersionedValueFactory(partitioner);
// create a ring of 2 nodes
ArrayList<Token> endpointTokens = new ArrayList<>();
List<InetAddress> hosts = new ArrayList<>();
Util.createInitialRing(
ss, partitioner, endpointTokens, new ArrayList<Token>(), hosts, new ArrayList<UUID>(), 2);
InetAddress toRemove = hosts.get(1);
SystemKeyspace.updatePeerInfo(toRemove, "data_center", "dc42");
SystemKeyspace.updatePeerInfo(toRemove, "rack", "rack42");
assertEquals("rack42", SystemKeyspace.loadDcRackInfo().get(toRemove).get("rack"));
// mark the node as removed
Gossiper.instance.injectApplicationState(
toRemove,
ApplicationState.STATUS,
valueFactory.left(
Collections.singleton(endpointTokens.get(1)), Gossiper.computeExpireTime()));
assertTrue(
Gossiper.instance.isDeadState(Gossiper.instance.getEndpointStateForEndpoint(hosts.get(1))));
// state changes made after the endpoint has left should be ignored
ss.onChange(hosts.get(1), ApplicationState.RACK, valueFactory.rack("rack9999"));
assertEquals("rack42", SystemKeyspace.loadDcRackInfo().get(toRemove).get("rack"));
}
@Test
public void testCompactionLog() throws Exception {
SystemKeyspace.discardCompactionsInProgress();
String cf = "Standard4";
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(cf);
SchemaLoader.insertData(KEYSPACE1, cf, 0, 1);
cfs.forceBlockingFlush();
Collection<SSTableReader> sstables = cfs.getSSTables();
assertFalse(sstables.isEmpty());
Set<Integer> generations =
Sets.newHashSet(
Iterables.transform(
sstables,
new Function<SSTableReader, Integer>() {
public Integer apply(SSTableReader sstable) {
return sstable.descriptor.generation;
}
}));
UUID taskId = SystemKeyspace.startCompaction(cfs, sstables);
Map<Pair<String, String>, Map<Integer, UUID>> compactionLogs =
SystemKeyspace.getUnfinishedCompactions();
Set<Integer> unfinishedCompactions = compactionLogs.get(Pair.create(KEYSPACE1, cf)).keySet();
assertTrue(unfinishedCompactions.containsAll(generations));
SystemKeyspace.finishCompaction(taskId);
compactionLogs = SystemKeyspace.getUnfinishedCompactions();
assertFalse(compactionLogs.containsKey(Pair.create(KEYSPACE1, cf)));
}
/** Drop peers info and token from system table */
void onPostStart() throws Exception {
super.onPostStart();
for (String peer : obsoletePeers) {
log.info("Remove node {} from cassandra peer table", peer);
SystemKeyspace.removeEndpoint(InetAddress.getByName(peer));
}
removeFlag(Constants.OBSOLETE_CASSANDRA_PEERS);
}
public static String updateCompactionHistory(
String keyspaceName,
String columnFamilyName,
AbstractCompactionIterable ci,
long startSize,
long endSize) {
long[] counts = ci.getMergedRowCounts();
StringBuilder mergeSummary = new StringBuilder(counts.length * 10);
Map<Integer, Long> mergedRows = new HashMap<>();
for (int i = 0; i < counts.length; i++) {
long count = counts[i];
if (count == 0) continue;
int rows = i + 1;
mergeSummary.append(String.format("%d:%d, ", rows, count));
mergedRows.put(rows, count);
}
SystemKeyspace.updateCompactionHistory(
keyspaceName, columnFamilyName, System.currentTimeMillis(), startSize, endSize, mergedRows);
return mergeSummary.toString();
}
/**
* For internal use and testing only. The rest of the system should go through the submit*
* methods, which are properly serialized. Caller is in charge of marking/unmarking the sstables
* as compacting.
*/
protected void runMayThrow() throws Exception {
// The collection of sstables passed may be empty (but not null); even if
// it is not empty, it may compact down to nothing if all rows are deleted.
assert transaction != null;
if (transaction.originals().isEmpty()) return;
// Note that the current compaction strategy, is not necessarily the one this task was created
// under.
// This should be harmless; see comments to CFS.maybeReloadCompactionStrategy.
AbstractCompactionStrategy strategy = cfs.getCompactionStrategy();
if (DatabaseDescriptor.isSnapshotBeforeCompaction())
cfs.snapshotWithoutFlush(System.currentTimeMillis() + "-compact-" + cfs.name);
// note that we need to do a rough estimate early if we can fit the compaction on disk - this is
// pessimistic, but
// since we might remove sstables from the compaction in checkAvailableDiskSpace it needs to be
// done here
long expectedWriteSize =
cfs.getExpectedCompactedFileSize(transaction.originals(), compactionType);
long earlySSTableEstimate = Math.max(1, expectedWriteSize / strategy.getMaxSSTableBytes());
checkAvailableDiskSpace(earlySSTableEstimate, expectedWriteSize);
// sanity check: all sstables must belong to the same cfs
assert !Iterables.any(
transaction.originals(),
new Predicate<SSTableReader>() {
@Override
public boolean apply(SSTableReader sstable) {
return !sstable.descriptor.cfname.equals(cfs.name);
}
});
UUID taskId = SystemKeyspace.startCompaction(cfs, transaction.originals());
// new sstables from flush can be added during a compaction, but only the compaction can remove
// them,
// so in our single-threaded compaction world this is a valid way of determining if we're
// compacting
// all the sstables (that existed when we started)
StringBuilder ssTableLoggerMsg = new StringBuilder("[");
for (SSTableReader sstr : transaction.originals()) {
ssTableLoggerMsg.append(
String.format("%s:level=%d, ", sstr.getFilename(), sstr.getSSTableLevel()));
}
ssTableLoggerMsg.append("]");
String taskIdLoggerMsg = taskId == null ? UUIDGen.getTimeUUID().toString() : taskId.toString();
logger.info("Compacting ({}) {}", taskIdLoggerMsg, ssTableLoggerMsg);
long start = System.nanoTime();
long totalKeysWritten = 0;
long estimatedKeys = 0;
try (CompactionController controller = getCompactionController(transaction.originals())) {
Set<SSTableReader> actuallyCompact =
Sets.difference(transaction.originals(), controller.getFullyExpiredSSTables());
SSTableFormat.Type sstableFormat = getFormatType(transaction.originals());
List<SSTableReader> newSStables;
AbstractCompactionIterable ci;
// SSTableScanners need to be closed before markCompactedSSTablesReplaced call as scanners
// contain references
// to both ifile and dfile and SSTR will throw deletion errors on Windows if it tries to
// delete before scanner is closed.
// See CASSANDRA-8019 and CASSANDRA-8399
try (Refs<SSTableReader> refs = Refs.ref(actuallyCompact);
AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact)) {
ci =
new CompactionIterable(
compactionType, scanners.scanners, controller, sstableFormat, taskId);
try (CloseableIterator<AbstractCompactedRow> iter = ci.iterator()) {
if (collector != null) collector.beginCompaction(ci);
long lastCheckObsoletion = start;
if (!controller.cfs.getCompactionStrategy().isActive)
throw new CompactionInterruptedException(ci.getCompactionInfo());
try (CompactionAwareWriter writer =
getCompactionAwareWriter(cfs, transaction, actuallyCompact)) {
estimatedKeys = writer.estimatedKeys();
while (iter.hasNext()) {
if (ci.isStopRequested())
throw new CompactionInterruptedException(ci.getCompactionInfo());
try (AbstractCompactedRow row = iter.next()) {
if (writer.append(row)) totalKeysWritten++;
if (System.nanoTime() - lastCheckObsoletion > TimeUnit.MINUTES.toNanos(1L)) {
controller.maybeRefreshOverlaps();
lastCheckObsoletion = System.nanoTime();
}
}
}
// don't replace old sstables yet, as we need to mark the compaction finished in the
// system table
newSStables = writer.finish();
} finally {
// point of no return -- the new sstables are live on disk; next we'll start deleting
// the old ones
// (in replaceCompactedSSTables)
if (taskId != null) SystemKeyspace.finishCompaction(taskId);
if (collector != null) collector.finishCompaction(ci);
}
}
}
// log a bunch of statistics about the result and save to system table compaction_history
long dTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
long startsize = SSTableReader.getTotalBytes(transaction.originals());
long endsize = SSTableReader.getTotalBytes(newSStables);
double ratio = (double) endsize / (double) startsize;
StringBuilder newSSTableNames = new StringBuilder();
for (SSTableReader reader : newSStables)
newSSTableNames.append(reader.descriptor.baseFilename()).append(",");
double mbps = dTime > 0 ? (double) endsize / (1024 * 1024) / ((double) dTime / 1000) : 0;
long totalSourceRows = 0;
String mergeSummary =
updateCompactionHistory(
cfs.keyspace.getName(), cfs.getColumnFamilyName(), ci, startsize, endsize);
logger.info(
String.format(
"Compacted (%s) %d sstables to [%s] to level=%d. %,d bytes to %,d (~%d%% of original) in %,dms = %fMB/s. %,d total partitions merged to %,d. Partition merge counts were {%s}",
taskIdLoggerMsg,
transaction.originals().size(),
newSSTableNames.toString(),
getLevel(),
startsize,
endsize,
(int) (ratio * 100),
dTime,
mbps,
totalSourceRows,
totalKeysWritten,
mergeSummary));
logger.debug(
String.format(
"CF Total Bytes Compacted: %,d", CompactionTask.addToTotalBytesCompacted(endsize)));
logger.debug(
"Actual #keys: {}, Estimated #keys:{}, Err%: {}",
totalKeysWritten,
estimatedKeys,
((double) (totalKeysWritten - estimatedKeys) / totalKeysWritten));
if (offline) Refs.release(Refs.selfRefs(newSStables));
}
}