Exemplo n.º 1
0
  /**
   * Modifies the per-CF dirty cursors of any commit log segments for the column family according to
   * the position given. Discards any commit log segments that are no longer used.
   *
   * @param cfId the column family ID that was flushed
   * @param context the replay position of the flush
   */
  public void discardCompletedSegments(final UUID cfId, final ReplayPosition context) {
    logger.trace("discard completed log segments for {}, table {}", context, cfId);

    // Go thru the active segment files, which are ordered oldest to newest, marking the
    // flushed CF as clean, until we reach the segment file containing the ReplayPosition passed
    // in the arguments. Any segments that become unused after they are marked clean will be
    // recycled or discarded.
    for (Iterator<CommitLogSegment> iter = allocator.getActiveSegments().iterator();
        iter.hasNext(); ) {
      CommitLogSegment segment = iter.next();
      segment.markClean(cfId, context);

      if (segment.isUnused()) {
        logger.trace("Commit log segment {} is unused", segment);
        allocator.recycleSegment(segment);
      } else {
        logger.trace(
            "Not safe to delete{} commit log segment {}; dirty is {}",
            (iter.hasNext() ? "" : " active"),
            segment,
            segment.dirtyString());
      }

      // Don't mark or try to delete any newer segments once we've reached the one containing the
      // position of the flush.
      if (segment.contains(context)) break;
    }
  }
Exemplo n.º 2
0
 /** Forces a disk flush on the commit log files that need it. Blocking. */
 public void sync(boolean syncAllSegments) {
   CommitLogSegment current = allocator.allocatingFrom();
   for (CommitLogSegment segment : allocator.getActiveSegments()) {
     if (!syncAllSegments && segment.id > current.id) return;
     segment.sync();
   }
 }
Exemplo n.º 3
0
  /**
   * Perform recovery on commit logs located in the directory specified by the config file.
   *
   * @return the number of mutations replayed
   */
  public int recover() throws IOException {
    // If createReserveSegments is already flipped, the CLSM is running and recovery has already
    // taken place.
    if (allocator.createReserveSegments) return 0;

    // Allocator could be in the process of initial startup with 0 active and available segments. We
    // need to wait for
    // the allocation manager to finish allocation and add it to available segments so we don't get
    // an invalid response
    // on allocator.manages(...) below by grabbing a file off the filesystem before it's added to
    // the CLQ.
    allocator.allocatingFrom();

    FilenameFilter unmanagedFilesFilter =
        new FilenameFilter() {
          public boolean accept(File dir, String name) {
            // we used to try to avoid instantiating commitlog (thus creating an empty segment ready
            // for writes)
            // until after recover was finished.  this turns out to be fragile; it is less
            // error-prone to go
            // ahead and allow writes before recover(), and just skip active segments when we do.
            return CommitLogDescriptor.isValid(name) && !allocator.manages(name);
          }
        };

    // submit all existing files in the commit log dir for archiving prior to recovery -
    // CASSANDRA-6904
    for (File file :
        new File(DatabaseDescriptor.getCommitLogLocation()).listFiles(unmanagedFilesFilter)) {
      archiver.maybeArchive(file.getPath(), file.getName());
      archiver.maybeWaitForArchiving(file.getName());
    }

    assert archiver.archivePending.isEmpty()
        : "Not all commit log archive tasks were completed before restore";
    archiver.maybeRestoreArchive();

    File[] files =
        new File(DatabaseDescriptor.getCommitLogLocation()).listFiles(unmanagedFilesFilter);
    int replayed = 0;
    if (files.length == 0) {
      logger.info("No commitlog files found; skipping replay");
    } else {
      Arrays.sort(files, new CommitLogSegmentFileComparator());
      logger.info("Replaying {}", StringUtils.join(files, ", "));
      replayed = recover(files);
      logger.info("Log replay complete, {} replayed mutations", replayed);

      for (File f : files) allocator.recycleSegment(f);
    }

    allocator.enableReserveSegmentCreation();
    return replayed;
  }
Exemplo n.º 4
0
 @Override
 public Map<String, Double> getActiveSegmentCompressionRatios() {
   Map<String, Double> segmentRatios = new TreeMap<>();
   for (CommitLogSegment segment : allocator.getActiveSegments())
     segmentRatios.put(segment.getName(), 1.0 * segment.onDiskSize() / segment.contentSize());
   return segmentRatios;
 }
Exemplo n.º 5
0
 /** FOR TESTING PURPOSES. See CommitLogAllocator. */
 public void stopUnsafe(boolean deleteSegments) {
   executor.shutdown();
   try {
     executor.awaitTermination();
   } catch (InterruptedException e) {
     throw new RuntimeException(e);
   }
   allocator.stopUnsafe(deleteSegments);
 }
Exemplo n.º 6
0
 /** FOR TESTING PURPOSES. See CommitLogAllocator */
 public int restartUnsafe() throws IOException {
   allocator.start();
   executor.restartUnsafe();
   try {
     return recover();
   } catch (FSWriteError e) {
     // Workaround for a class of races that keeps showing up on Windows tests.
     // stop/start/reset path on Windows with segment deletion is very touchy/brittle
     // and the timing keeps getting screwed up. Rather than chasing our tail further
     // or rewriting the CLSM, just report that we didn't recover anything back up
     // the chain. This will silence most intermittent test failures on Windows
     // and appropriately fail tests that expected segments to be recovered that
     // were not.
     return 0;
   }
 }
Exemplo n.º 7
0
  /**
   * Add a Mutation to the commit log.
   *
   * @param mutation the Mutation to add to the log
   */
  public ReplayPosition add(Mutation mutation) {
    assert mutation != null;

    long size = Mutation.serializer.serializedSize(mutation, MessagingService.current_version);

    long totalSize = size + ENTRY_OVERHEAD_SIZE;
    if (totalSize > MAX_MUTATION_SIZE) {
      throw new IllegalArgumentException(
          String.format(
              "Mutation of %s bytes is too large for the maxiumum size of %s",
              totalSize, MAX_MUTATION_SIZE));
    }

    Allocation alloc = allocator.allocate(mutation, (int) totalSize);
    ICRC32 checksum = CRC32Factory.instance.create();
    final ByteBuffer buffer = alloc.getBuffer();
    try (BufferedDataOutputStreamPlus dos = new DataOutputBufferFixed(buffer)) {
      // checksummed length
      dos.writeInt((int) size);
      checksum.update(buffer, buffer.position() - 4, 4);
      buffer.putInt(checksum.getCrc());

      int start = buffer.position();
      // checksummed mutation
      Mutation.serializer.serialize(mutation, dos, MessagingService.current_version);
      checksum.update(buffer, start, (int) size);
      buffer.putInt(checksum.getCrc());
    } catch (IOException e) {
      throw new FSWriteError(e, alloc.getSegment().getPath());
    } finally {
      alloc.markWritten();
    }

    executor.finishWriteFor(alloc);
    return alloc.getReplayPosition();
  }
Exemplo n.º 8
0
 /**
  * Used by tests.
  *
  * @return the number of active segments (segments with unflushed data in them)
  */
 public int activeSegments() {
   return allocator.getActiveSegments().size();
 }
Exemplo n.º 9
0
 /** Shuts down the threads used by the commit log, blocking until completion. */
 public void shutdownBlocking() throws InterruptedException {
   executor.shutdown();
   executor.awaitTermination();
   allocator.shutdown();
   allocator.awaitTermination();
 }
Exemplo n.º 10
0
 @Override
 public long getActiveOnDiskSize() {
   return allocator.onDiskSize();
 }
Exemplo n.º 11
0
 @Override
 public long getActiveContentSize() {
   long size = 0;
   for (CommitLogSegment segment : allocator.getActiveSegments()) size += segment.contentSize();
   return size;
 }
Exemplo n.º 12
0
 public List<String> getActiveSegmentNames() {
   List<String> segmentNames = new ArrayList<>();
   for (CommitLogSegment segment : allocator.getActiveSegments())
     segmentNames.add(segment.getName());
   return segmentNames;
 }
Exemplo n.º 13
0
 /**
  * Flushes all dirty CFs, waiting for them to free and recycle any segments they were retaining
  */
 public void forceRecycleAllSegments() {
   allocator.forceRecycleAll(Collections.<UUID>emptyList());
 }
Exemplo n.º 14
0
 /**
  * Flushes all dirty CFs, waiting for them to free and recycle any segments they were retaining
  */
 public void forceRecycleAllSegments(Iterable<UUID> droppedCfs) {
   allocator.forceRecycleAll(droppedCfs);
 }
Exemplo n.º 15
0
 /**
  * @return a ReplayPosition which, if >= one returned from add(), implies add() was started (but
  *     not necessarily finished) prior to this call
  */
 public ReplayPosition getContext() {
   return allocator.allocatingFrom().getContext();
 }
Exemplo n.º 16
0
 CommitLog start() {
   executor.start();
   allocator.start();
   return this;
 }