/**
* Modifies the per-CF dirty cursors of any commit log segments for the column family according to
* the position given. Discards any commit log segments that are no longer used.
*
* @param cfId the column family ID that was flushed
* @param context the replay position of the flush
*/
public void discardCompletedSegments(final UUID cfId, final ReplayPosition context) {
logger.trace("discard completed log segments for {}, table {}", context, cfId);
// Go thru the active segment files, which are ordered oldest to newest, marking the
// flushed CF as clean, until we reach the segment file containing the ReplayPosition passed
// in the arguments. Any segments that become unused after they are marked clean will be
// recycled or discarded.
for (Iterator<CommitLogSegment> iter = allocator.getActiveSegments().iterator();
iter.hasNext(); ) {
CommitLogSegment segment = iter.next();
segment.markClean(cfId, context);
if (segment.isUnused()) {
logger.trace("Commit log segment {} is unused", segment);
allocator.recycleSegment(segment);
} else {
logger.trace(
"Not safe to delete{} commit log segment {}; dirty is {}",
(iter.hasNext() ? "" : " active"),
segment,
segment.dirtyString());
}
// Don't mark or try to delete any newer segments once we've reached the one containing the
// position of the flush.
if (segment.contains(context)) break;
}
}
/** Forces a disk flush on the commit log files that need it. Blocking. */
public void sync(boolean syncAllSegments) {
CommitLogSegment current = allocator.allocatingFrom();
for (CommitLogSegment segment : allocator.getActiveSegments()) {
if (!syncAllSegments && segment.id > current.id) return;
segment.sync();
}
}
/**
* Perform recovery on commit logs located in the directory specified by the config file.
*
* @return the number of mutations replayed
*/
public int recover() throws IOException {
// If createReserveSegments is already flipped, the CLSM is running and recovery has already
// taken place.
if (allocator.createReserveSegments) return 0;
// Allocator could be in the process of initial startup with 0 active and available segments. We
// need to wait for
// the allocation manager to finish allocation and add it to available segments so we don't get
// an invalid response
// on allocator.manages(...) below by grabbing a file off the filesystem before it's added to
// the CLQ.
allocator.allocatingFrom();
FilenameFilter unmanagedFilesFilter =
new FilenameFilter() {
public boolean accept(File dir, String name) {
// we used to try to avoid instantiating commitlog (thus creating an empty segment ready
// for writes)
// until after recover was finished. this turns out to be fragile; it is less
// error-prone to go
// ahead and allow writes before recover(), and just skip active segments when we do.
return CommitLogDescriptor.isValid(name) && !allocator.manages(name);
}
};
// submit all existing files in the commit log dir for archiving prior to recovery -
// CASSANDRA-6904
for (File file :
new File(DatabaseDescriptor.getCommitLogLocation()).listFiles(unmanagedFilesFilter)) {
archiver.maybeArchive(file.getPath(), file.getName());
archiver.maybeWaitForArchiving(file.getName());
}
assert archiver.archivePending.isEmpty()
: "Not all commit log archive tasks were completed before restore";
archiver.maybeRestoreArchive();
File[] files =
new File(DatabaseDescriptor.getCommitLogLocation()).listFiles(unmanagedFilesFilter);
int replayed = 0;
if (files.length == 0) {
logger.info("No commitlog files found; skipping replay");
} else {
Arrays.sort(files, new CommitLogSegmentFileComparator());
logger.info("Replaying {}", StringUtils.join(files, ", "));
replayed = recover(files);
logger.info("Log replay complete, {} replayed mutations", replayed);
for (File f : files) allocator.recycleSegment(f);
}
allocator.enableReserveSegmentCreation();
return replayed;
}
@Override
public Map<String, Double> getActiveSegmentCompressionRatios() {
Map<String, Double> segmentRatios = new TreeMap<>();
for (CommitLogSegment segment : allocator.getActiveSegments())
segmentRatios.put(segment.getName(), 1.0 * segment.onDiskSize() / segment.contentSize());
return segmentRatios;
}
/** FOR TESTING PURPOSES. See CommitLogAllocator. */
public void stopUnsafe(boolean deleteSegments) {
executor.shutdown();
try {
executor.awaitTermination();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
allocator.stopUnsafe(deleteSegments);
}
/** FOR TESTING PURPOSES. See CommitLogAllocator */
public int restartUnsafe() throws IOException {
allocator.start();
executor.restartUnsafe();
try {
return recover();
} catch (FSWriteError e) {
// Workaround for a class of races that keeps showing up on Windows tests.
// stop/start/reset path on Windows with segment deletion is very touchy/brittle
// and the timing keeps getting screwed up. Rather than chasing our tail further
// or rewriting the CLSM, just report that we didn't recover anything back up
// the chain. This will silence most intermittent test failures on Windows
// and appropriately fail tests that expected segments to be recovered that
// were not.
return 0;
}
}
/**
* Add a Mutation to the commit log.
*
* @param mutation the Mutation to add to the log
*/
public ReplayPosition add(Mutation mutation) {
assert mutation != null;
long size = Mutation.serializer.serializedSize(mutation, MessagingService.current_version);
long totalSize = size + ENTRY_OVERHEAD_SIZE;
if (totalSize > MAX_MUTATION_SIZE) {
throw new IllegalArgumentException(
String.format(
"Mutation of %s bytes is too large for the maxiumum size of %s",
totalSize, MAX_MUTATION_SIZE));
}
Allocation alloc = allocator.allocate(mutation, (int) totalSize);
ICRC32 checksum = CRC32Factory.instance.create();
final ByteBuffer buffer = alloc.getBuffer();
try (BufferedDataOutputStreamPlus dos = new DataOutputBufferFixed(buffer)) {
// checksummed length
dos.writeInt((int) size);
checksum.update(buffer, buffer.position() - 4, 4);
buffer.putInt(checksum.getCrc());
int start = buffer.position();
// checksummed mutation
Mutation.serializer.serialize(mutation, dos, MessagingService.current_version);
checksum.update(buffer, start, (int) size);
buffer.putInt(checksum.getCrc());
} catch (IOException e) {
throw new FSWriteError(e, alloc.getSegment().getPath());
} finally {
alloc.markWritten();
}
executor.finishWriteFor(alloc);
return alloc.getReplayPosition();
}
/**
* Used by tests.
*
* @return the number of active segments (segments with unflushed data in them)
*/
public int activeSegments() {
return allocator.getActiveSegments().size();
}
/** Shuts down the threads used by the commit log, blocking until completion. */
public void shutdownBlocking() throws InterruptedException {
executor.shutdown();
executor.awaitTermination();
allocator.shutdown();
allocator.awaitTermination();
}
@Override
public long getActiveOnDiskSize() {
return allocator.onDiskSize();
}
@Override
public long getActiveContentSize() {
long size = 0;
for (CommitLogSegment segment : allocator.getActiveSegments()) size += segment.contentSize();
return size;
}
public List<String> getActiveSegmentNames() {
List<String> segmentNames = new ArrayList<>();
for (CommitLogSegment segment : allocator.getActiveSegments())
segmentNames.add(segment.getName());
return segmentNames;
}
/**
* Flushes all dirty CFs, waiting for them to free and recycle any segments they were retaining
*/
public void forceRecycleAllSegments() {
allocator.forceRecycleAll(Collections.<UUID>emptyList());
}
/**
* Flushes all dirty CFs, waiting for them to free and recycle any segments they were retaining
*/
public void forceRecycleAllSegments(Iterable<UUID> droppedCfs) {
allocator.forceRecycleAll(droppedCfs);
}
/**
* @return a ReplayPosition which, if >= one returned from add(), implies add() was started (but
* not necessarily finished) prior to this call
*/
public ReplayPosition getContext() {
return allocator.allocatingFrom().getContext();
}
CommitLog start() {
executor.start();
allocator.start();
return this;
}