public static long getMinRepairedAt(Set<SSTableReader> actuallyCompact) {
   long minRepairedAt = Long.MAX_VALUE;
   for (SSTableReader sstable : actuallyCompact)
     minRepairedAt = Math.min(minRepairedAt, sstable.getSSTableMetadata().repairedAt);
   if (minRepairedAt == Long.MAX_VALUE) return ActiveRepairService.UNREPAIRED_SSTABLE;
   return minRepairedAt;
 }
  @Test
  public void testGetTemporaryFilesThrowsIfCompletingAfterObsoletion() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);

    LogTransaction logs = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(logs);

    LogTransaction.SSTableTidier tidier = logs.obsoleted(sstable);

    sstable.markObsolete(tidier);
    sstable.selfRef().release();

    LogTransaction.waitForDeletions();

    try {
      // This should race with the asynchronous deletion of txn log files
      // it should throw because we are violating the requirement that a transaction must
      // finish before deleting files (i.e. releasing sstables)
      getTemporaryFiles(dataFolder);
      fail("Expected runtime exception");
    } catch (RuntimeException e) {
      // pass as long as the cause is not an assertion
      assertFalse(e.getCause() instanceof AssertionError);
    }

    logs.finish();
  }
 void assertInProgress() throws Exception {
   assertFiles(
       dataFolder.getPath(),
       Sets.newHashSet(
           Iterables.concat(
               sstableNew.getAllFilePaths(),
               sstableOld.getAllFilePaths(),
               txnLogs.logFilePaths())));
 }
      protected Throwable doAbort(Throwable accumulate) {
        tidier.abort();
        LogTransaction.waitForDeletions();

        Throwable ret = txnLogs.abort(accumulate);

        sstableNew.selfRef().release();
        sstableOld.selfRef().release();
        return ret;
      }
      protected Throwable doCommit(Throwable accumulate) {
        sstableOld.markObsolete(tidier);
        sstableOld.selfRef().release();
        LogTransaction.waitForDeletions();

        Throwable ret = txnLogs.commit(accumulate);

        sstableNew.selfRef().release();
        return ret;
      }
  @Test
  public void testCommitSameDesc() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstableOld1 = sstable(dataFolder, cfs, 0, 128);
    SSTableReader sstableOld2 = sstable(dataFolder, cfs, 0, 256);
    SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);

    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    log.trackNew(sstableNew);

    sstableOld1.setReplaced();

    LogTransaction.SSTableTidier tidier = log.obsoleted(sstableOld2);
    assertNotNull(tidier);

    log.finish();

    sstableOld2.markObsolete(tidier);

    sstableOld1.selfRef().release();
    sstableOld2.selfRef().release();

    assertFiles(dataFolder.getPath(), new HashSet<>(sstableNew.getAllFilePaths()));

    sstableNew.selfRef().release();
  }
 public static SSTableReader sstable(
     int generation, int size, boolean keepRef, ColumnFamilyStore cfs) {
   Descriptor descriptor =
       new Descriptor(
           cfs.getDirectories().getDirectoryForNewSSTables(),
           cfs.keyspace.getName(),
           cfs.getColumnFamilyName(),
           generation);
   Set<Component> components =
       ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
   for (Component component : components) {
     File file = new File(descriptor.filenameFor(component));
     try {
       file.createNewFile();
     } catch (IOException e) {
     }
   }
   if (size > 0) {
     try {
       File file = new File(descriptor.filenameFor(Component.DATA));
       try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
         raf.setLength(size);
       }
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
   }
   SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.emptyList());
   StatsMetadata metadata =
       (StatsMetadata)
           new MetadataCollector(cfs.metadata.comparator)
               .finalizeMetadata(
                   cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
               .get(MetadataType.STATS);
   SSTableReader reader =
       SSTableReader.internalOpen(
           descriptor,
           components,
           cfs.metadata,
           segmentedFile.sharedCopy(),
           segmentedFile.sharedCopy(),
           indexSummary.sharedCopy(),
           new AlwaysPresentFilter(),
           1L,
           metadata,
           SSTableReader.OpenReason.NORMAL,
           header);
   reader.first = reader.last = readerBounds(generation);
   if (!keepRef) reader.selfRef().release();
   return reader;
 }
 /** Gets the estimated total amount of data to write during compaction */
 private static long getTotalWriteSize(
     Iterable<SSTableReader> nonExpiredSSTables,
     long estimatedTotalKeys,
     ColumnFamilyStore cfs,
     OperationType compactionType) {
   long estimatedKeysBeforeCompaction = 0;
   for (SSTableReader sstable : nonExpiredSSTables)
     estimatedKeysBeforeCompaction += sstable.estimatedKeys();
   estimatedKeysBeforeCompaction = Math.max(1, estimatedKeysBeforeCompaction);
   double estimatedCompactionRatio = (double) estimatedTotalKeys / estimatedKeysBeforeCompaction;
   return Math.round(
       estimatedCompactionRatio
           * cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType));
 }
  private static SSTableReader sstable(
      File dataFolder, ColumnFamilyStore cfs, int generation, int size) throws IOException {
    Descriptor descriptor =
        new Descriptor(dataFolder, cfs.keyspace.getName(), cfs.getTableName(), generation);
    Set<Component> components =
        ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
    for (Component component : components) {
      File file = new File(descriptor.filenameFor(component));
      if (!file.exists()) assertTrue(file.createNewFile());
      try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
        raf.setLength(size);
      }
    }

    SegmentedFile dFile =
        new BufferedSegmentedFile(
            new ChannelProxy(new File(descriptor.filenameFor(Component.DATA))),
            RandomAccessReader.DEFAULT_BUFFER_SIZE,
            0);
    SegmentedFile iFile =
        new BufferedSegmentedFile(
            new ChannelProxy(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))),
            RandomAccessReader.DEFAULT_BUFFER_SIZE,
            0);

    SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.emptyList());
    StatsMetadata metadata =
        (StatsMetadata)
            new MetadataCollector(cfs.metadata.comparator)
                .finalizeMetadata(
                    cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
                .get(MetadataType.STATS);
    SSTableReader reader =
        SSTableReader.internalOpen(
            descriptor,
            components,
            cfs.metadata,
            dFile,
            iFile,
            MockSchema.indexSummary.sharedCopy(),
            new AlwaysPresentFilter(),
            1L,
            metadata,
            SSTableReader.OpenReason.NORMAL,
            header);
    reader.first = reader.last = MockSchema.readerBounds(generation);
    return reader;
  }
  @Test
  public void testAbortOnlyNew() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);

    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    log.trackNew(sstable);
    log.abort();

    sstable.selfRef().release();

    assertFiles(dataFolder.getPath(), new HashSet<>());
  }
  @Test
  public void testCommitOnlyOld() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);

    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    LogTransaction.SSTableTidier tidier = log.obsoleted(sstable);
    assertNotNull(tidier);

    log.finish();
    sstable.markObsolete(tidier);
    sstable.selfRef().release();

    assertFiles(dataFolder.getPath(), new HashSet<>());
  }
  @Test
  public void testGetTemporaryFilesSafeAfterObsoletion() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);

    LogTransaction logs = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(logs);

    LogTransaction.SSTableTidier tidier = logs.obsoleted(sstable);

    logs.finish();

    sstable.markObsolete(tidier);
    sstable.selfRef().release();

    // This should race with the asynchronous deletion of txn log files
    // It doesn't matter what it returns but it should not throw because the txn
    // was completed before deleting files (i.e. releasing sstables)
    for (int i = 0; i < 200; i++) getTemporaryFiles(dataFolder);
  }
  @Test
  public void testUntrack() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);

    // complete a transaction without keep the new files since they were untracked
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    log.trackNew(sstableNew);
    log.untrackNew(sstableNew);

    log.finish();

    sstableNew.selfRef().release();
    Thread.sleep(1);
    LogTransaction.waitForDeletions();

    assertFiles(dataFolder.getPath(), Collections.<String>emptySet());
  }
  public boolean shouldInclude(SSTableReader sstable) {
    List<ByteBuffer> minColumnNames = sstable.getSSTableMetadata().minColumnNames;
    List<ByteBuffer> maxColumnNames = sstable.getSSTableMetadata().maxColumnNames;
    CellNameType comparator = sstable.metadata.comparator;

    if (minColumnNames.isEmpty() || maxColumnNames.isEmpty()) return true;

    for (ColumnSlice slice : slices)
      if (slice.intersects(minColumnNames, maxColumnNames, comparator, reversed)) return true;

    return false;
  }
 /**
  * update a reader: if !original, this is a reader that is being introduced by this transaction;
  * otherwise it must be in the originals() set, i.e. a reader guarded by this transaction
  */
 public void update(SSTableReader reader, boolean original) {
   assert !staged.update.contains(reader)
       : "each reader may only be updated once per checkpoint: " + reader;
   assert !identities.contains(reader.instanceId)
       : "each reader instance may only be provided as an update once: " + reader;
   // check it isn't obsolete, and that it matches the original flag
   assert !(logged.obsolete.contains(reader) || staged.obsolete.contains(reader))
       : "may not update a reader that has been obsoleted";
   assert original == originals.contains(reader)
       : String.format(
           "the 'original' indicator was incorrect (%s provided): %s", original, reader);
   staged.update.add(reader);
   identities.add(reader.instanceId);
   if (!isOffline()) reader.setupKeyCache();
 }
  private static void testCorruptRecord(
      BiConsumer<LogTransaction, SSTableReader> modifier, boolean isRecoverable)
      throws IOException {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
    SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);

    // simulate tracking sstables with a committed transaction except the checksum will be wrong
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    log.trackNew(sstableNew);
    log.obsoleted(sstableOld);

    // Modify the transaction log or disk state for sstableOld
    modifier.accept(log, sstableOld);

    assertNull(log.complete(null));

    sstableOld.selfRef().release();
    sstableNew.selfRef().release();

    // The files on disk, for old files make sure to exclude the files that were deleted by the
    // modifier
    Set<String> newFiles = sstableNew.getAllFilePaths().stream().collect(Collectors.toSet());
    Set<String> oldFiles =
        sstableOld
            .getAllFilePaths()
            .stream()
            .filter(p -> new File(p).exists())
            .collect(Collectors.toSet());

    // This should filter as in progress since the last record is corrupt
    assertFiles(newFiles, getTemporaryFiles(dataFolder));
    assertFiles(oldFiles, getFinalFiles(dataFolder));

    if (isRecoverable) { // the corruption is recoverable but the commit record is unreadable so the
      // transaction is still in progress

      // This should remove new files
      LogTransaction.removeUnfinishedLeftovers(cfs.metadata);

      // make sure to exclude the old files that were deleted by the modifier
      assertFiles(dataFolder.getPath(), oldFiles);
    } else { // if an intermediate line was also modified, it should ignore the tx log file

      // This should not remove any files
      LogTransaction.removeUnfinishedLeftovers(cfs.metadata);

      assertFiles(
          dataFolder.getPath(),
          Sets.newHashSet(Iterables.concat(newFiles, oldFiles, log.logFilePaths())));
    }
  }
  private static void testObsoletedFilesChanged(Consumer<SSTableReader> modifier)
      throws IOException {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
    SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);

    // simulate tracking sstables with a committed transaction except the checksum will be wrong
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    log.trackNew(sstableNew);
    /*TransactionLog.SSTableTidier tidier =*/ log.obsoleted(sstableOld);

    // modify the old sstable files
    modifier.accept(sstableOld);

    // Fake a commit
    log.txnFile().commit();

    // This should not remove the old files
    LogTransaction.removeUnfinishedLeftovers(cfs.metadata);

    assertFiles(
        dataFolder.getPath(),
        Sets.newHashSet(
            Iterables.concat(
                sstableNew.getAllFilePaths(), sstableOld.getAllFilePaths(), log.logFilePaths())));

    sstableOld.selfRef().release();
    sstableNew.selfRef().release();

    // complete the transaction to avoid LEAK errors
    assertNull(log.complete(null));

    assertFiles(
        dataFolder.getPath(),
        Sets.newHashSet(
            Iterables.concat(
                sstableNew.getAllFilePaths(), sstableOld.getAllFilePaths(), log.logFilePaths())));
  }
  @Test
  public void testRemoveUnfinishedLeftovers_commit() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
    SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);

    // simulate tracking sstables with a committed transaction (new log file deleted)
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    log.trackNew(sstableNew);
    LogTransaction.SSTableTidier tidier = log.obsoleted(sstableOld);

    // Fake a commit
    log.txnFile().commit();

    Set<File> tmpFiles =
        sstableOld.getAllFilePaths().stream().map(File::new).collect(Collectors.toSet());

    sstableNew.selfRef().release();
    sstableOld.selfRef().release();

    Assert.assertEquals(tmpFiles, getTemporaryFiles(sstableOld.descriptor.directory));

    // normally called at startup
    LogTransaction.removeUnfinishedLeftovers(cfs.metadata);

    // sstableNew should be only table left
    Directories directories = new Directories(cfs.metadata);
    Map<Descriptor, Set<Component>> sstables =
        directories.sstableLister(Directories.OnTxnErr.THROW).list();
    assertEquals(1, sstables.size());

    assertFiles(dataFolder.getPath(), new HashSet<>(sstableNew.getAllFilePaths()));

    // complete the transaction to avoid LEAK errors
    tidier.run();
    assertNull(log.complete(null));
  }
  /**
   * Given arguments specifying an SSTable, and optionally an output file, export the contents of
   * the SSTable to JSON.
   *
   * @param args command lines arguments
   * @throws ConfigurationException on configuration failure (wrong params given)
   */
  public static void main(String[] args) throws ConfigurationException {
    CommandLineParser parser = new PosixParser();
    try {
      cmd = parser.parse(options, args);
    } catch (ParseException e1) {
      System.err.println(e1.getMessage());
      printUsage();
      System.exit(1);
    }

    if (cmd.getArgs().length != 1) {
      System.err.println("You must supply exactly one sstable");
      printUsage();
      System.exit(1);
    }

    String[] keys = cmd.getOptionValues(KEY_OPTION);
    HashSet<String> excludes =
        new HashSet<>(
            Arrays.asList(
                cmd.getOptionValues(EXCLUDE_KEY_OPTION) == null
                    ? new String[0]
                    : cmd.getOptionValues(EXCLUDE_KEY_OPTION)));
    String ssTableFileName = new File(cmd.getArgs()[0]).getAbsolutePath();

    if (Descriptor.isLegacyFile(new File(ssTableFileName))) {
      System.err.println("Unsupported legacy sstable");
      System.exit(1);
    }
    if (!new File(ssTableFileName).exists()) {
      System.err.println("Cannot find file " + ssTableFileName);
      System.exit(1);
    }
    Descriptor desc = Descriptor.fromFilename(ssTableFileName);
    try {
      CFMetaData metadata = metadataFromSSTable(desc);
      if (cmd.hasOption(ENUMERATE_KEYS_OPTION)) {
        JsonTransformer.keysToJson(
            null,
            iterToStream(new KeyIterator(desc, metadata)),
            cmd.hasOption(RAW_TIMESTAMPS),
            metadata,
            System.out);
      } else {
        SSTableReader sstable = SSTableReader.openNoValidation(desc, metadata);
        IPartitioner partitioner = sstable.getPartitioner();
        final ISSTableScanner currentScanner;
        if ((keys != null) && (keys.length > 0)) {
          List<AbstractBounds<PartitionPosition>> bounds =
              Arrays.stream(keys)
                  .filter(key -> !excludes.contains(key))
                  .map(metadata.getKeyValidator()::fromString)
                  .map(partitioner::decorateKey)
                  .sorted()
                  .map(DecoratedKey::getToken)
                  .map(token -> new Bounds<>(token.minKeyBound(), token.maxKeyBound()))
                  .collect(Collectors.toList());
          currentScanner = sstable.getScanner(bounds.iterator());
        } else {
          currentScanner = sstable.getScanner();
        }
        Stream<UnfilteredRowIterator> partitions =
            iterToStream(currentScanner)
                .filter(
                    i ->
                        excludes.isEmpty()
                            || !excludes.contains(
                                metadata.getKeyValidator().getString(i.partitionKey().getKey())));
        if (cmd.hasOption(DEBUG_OUTPUT_OPTION)) {
          AtomicLong position = new AtomicLong();
          partitions.forEach(
              partition -> {
                position.set(currentScanner.getCurrentPosition());

                if (!partition.partitionLevelDeletion().isLive()) {
                  System.out.println(
                      "["
                          + metadata.getKeyValidator().getString(partition.partitionKey().getKey())
                          + "]@"
                          + position.get()
                          + " "
                          + partition.partitionLevelDeletion());
                }
                if (!partition.staticRow().isEmpty()) {
                  System.out.println(
                      "["
                          + metadata.getKeyValidator().getString(partition.partitionKey().getKey())
                          + "]@"
                          + position.get()
                          + " "
                          + partition.staticRow().toString(metadata, true));
                }
                partition.forEachRemaining(
                    row -> {
                      System.out.println(
                          "["
                              + metadata
                                  .getKeyValidator()
                                  .getString(partition.partitionKey().getKey())
                              + "]@"
                              + position.get()
                              + " "
                              + row.toString(metadata, false, true));
                      position.set(currentScanner.getCurrentPosition());
                    });
              });
        } else {
          JsonTransformer.toJson(
              currentScanner, partitions, cmd.hasOption(RAW_TIMESTAMPS), metadata, System.out);
        }
      }
    } catch (IOException e) {
      // throwing exception outside main with broken pipe causes windows cmd to hang
      e.printStackTrace(System.err);
    }

    System.exit(0);
  }
 public OnDiskAtomIterator getSSTableColumnIterator(
     SSTableReader sstable, FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry) {
   return sstable.iterator(file, key, slices, reversed, indexEntry);
 }
 public OnDiskAtomIterator getSSTableColumnIterator(SSTableReader sstable, DecoratedKey key) {
   return sstable.iterator(key, slices, reversed);
 }
  @Test
  public void testGetTemporaryFiles() throws IOException {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstable1 = sstable(dataFolder, cfs, 0, 128);

    Set<File> tmpFiles = getTemporaryFiles(dataFolder);
    assertNotNull(tmpFiles);
    assertEquals(0, tmpFiles.size());

    try (LogTransaction log = new LogTransaction(OperationType.WRITE)) {
      Directories directories = new Directories(cfs.metadata);

      File[] beforeSecondSSTable = dataFolder.listFiles(pathname -> !pathname.isDirectory());

      SSTableReader sstable2 = sstable(dataFolder, cfs, 1, 128);
      log.trackNew(sstable2);

      Map<Descriptor, Set<Component>> sstables =
          directories.sstableLister(Directories.OnTxnErr.THROW).list();
      assertEquals(2, sstables.size());

      // this should contain sstable1, sstable2 and the transaction log file
      File[] afterSecondSSTable = dataFolder.listFiles(pathname -> !pathname.isDirectory());

      int numNewFiles = afterSecondSSTable.length - beforeSecondSSTable.length;
      assertEquals(
          numNewFiles - 1,
          sstable2.getAllFilePaths().size()); // new files except for transaction log file

      tmpFiles = getTemporaryFiles(dataFolder);
      assertNotNull(tmpFiles);
      assertEquals(numNewFiles - 1, tmpFiles.size());

      File ssTable2DataFile = new File(sstable2.descriptor.filenameFor(Component.DATA));
      File ssTable2IndexFile = new File(sstable2.descriptor.filenameFor(Component.PRIMARY_INDEX));

      assertTrue(tmpFiles.contains(ssTable2DataFile));
      assertTrue(tmpFiles.contains(ssTable2IndexFile));

      List<File> files = directories.sstableLister(Directories.OnTxnErr.THROW).listFiles();
      List<File> filesNoTmp =
          directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true).listFiles();
      assertNotNull(files);
      assertNotNull(filesNoTmp);

      assertTrue(files.contains(ssTable2DataFile));
      assertTrue(files.contains(ssTable2IndexFile));

      assertFalse(filesNoTmp.contains(ssTable2DataFile));
      assertFalse(filesNoTmp.contains(ssTable2IndexFile));

      log.finish();

      // Now it should be empty since the transaction has finished
      tmpFiles = getTemporaryFiles(dataFolder);
      assertNotNull(tmpFiles);
      assertEquals(0, tmpFiles.size());

      filesNoTmp =
          directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true).listFiles();
      assertNotNull(filesNoTmp);
      assertTrue(filesNoTmp.contains(ssTable2DataFile));
      assertTrue(filesNoTmp.contains(ssTable2IndexFile));

      sstable1.selfRef().release();
      sstable2.selfRef().release();
    }
  }
 protected AbstractTransactionalTest.TestableTransaction newTest() throws Exception {
   LogTransaction.waitForDeletions();
   SSTableReader.resetTidying();
   return new TxnTest();
 }
 void assertAborted() throws Exception {
   assertFiles(dataFolder.getPath(), new HashSet<>(sstableOld.getAllFilePaths()));
 }
 void assertCommitted() throws Exception {
   assertFiles(dataFolder.getPath(), new HashSet<>(sstableNew.getAllFilePaths()));
 }
Beispiel #26
0
  /**
   * For internal use and testing only. The rest of the system should go through the submit*
   * methods, which are properly serialized. Caller is in charge of marking/unmarking the sstables
   * as compacting.
   */
  protected void runMayThrow() throws Exception {
    // The collection of sstables passed may be empty (but not null); even if
    // it is not empty, it may compact down to nothing if all rows are deleted.
    assert transaction != null;

    if (transaction.originals().isEmpty()) return;

    // Note that the current compaction strategy, is not necessarily the one this task was created
    // under.
    // This should be harmless; see comments to CFS.maybeReloadCompactionStrategy.
    AbstractCompactionStrategy strategy = cfs.getCompactionStrategy();

    if (DatabaseDescriptor.isSnapshotBeforeCompaction())
      cfs.snapshotWithoutFlush(System.currentTimeMillis() + "-compact-" + cfs.name);

    // note that we need to do a rough estimate early if we can fit the compaction on disk - this is
    // pessimistic, but
    // since we might remove sstables from the compaction in checkAvailableDiskSpace it needs to be
    // done here
    long expectedWriteSize =
        cfs.getExpectedCompactedFileSize(transaction.originals(), compactionType);
    long earlySSTableEstimate = Math.max(1, expectedWriteSize / strategy.getMaxSSTableBytes());
    checkAvailableDiskSpace(earlySSTableEstimate, expectedWriteSize);

    // sanity check: all sstables must belong to the same cfs
    assert !Iterables.any(
        transaction.originals(),
        new Predicate<SSTableReader>() {
          @Override
          public boolean apply(SSTableReader sstable) {
            return !sstable.descriptor.cfname.equals(cfs.name);
          }
        });

    UUID taskId = SystemKeyspace.startCompaction(cfs, transaction.originals());

    // new sstables from flush can be added during a compaction, but only the compaction can remove
    // them,
    // so in our single-threaded compaction world this is a valid way of determining if we're
    // compacting
    // all the sstables (that existed when we started)
    StringBuilder ssTableLoggerMsg = new StringBuilder("[");
    for (SSTableReader sstr : transaction.originals()) {
      ssTableLoggerMsg.append(
          String.format("%s:level=%d, ", sstr.getFilename(), sstr.getSSTableLevel()));
    }
    ssTableLoggerMsg.append("]");
    String taskIdLoggerMsg = taskId == null ? UUIDGen.getTimeUUID().toString() : taskId.toString();
    logger.info("Compacting ({}) {}", taskIdLoggerMsg, ssTableLoggerMsg);

    long start = System.nanoTime();

    long totalKeysWritten = 0;

    long estimatedKeys = 0;
    try (CompactionController controller = getCompactionController(transaction.originals())) {
      Set<SSTableReader> actuallyCompact =
          Sets.difference(transaction.originals(), controller.getFullyExpiredSSTables());

      SSTableFormat.Type sstableFormat = getFormatType(transaction.originals());

      List<SSTableReader> newSStables;
      AbstractCompactionIterable ci;

      // SSTableScanners need to be closed before markCompactedSSTablesReplaced call as scanners
      // contain references
      // to both ifile and dfile and SSTR will throw deletion errors on Windows if it tries to
      // delete before scanner is closed.
      // See CASSANDRA-8019 and CASSANDRA-8399
      try (Refs<SSTableReader> refs = Refs.ref(actuallyCompact);
          AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact)) {
        ci =
            new CompactionIterable(
                compactionType, scanners.scanners, controller, sstableFormat, taskId);
        try (CloseableIterator<AbstractCompactedRow> iter = ci.iterator()) {
          if (collector != null) collector.beginCompaction(ci);
          long lastCheckObsoletion = start;

          if (!controller.cfs.getCompactionStrategy().isActive)
            throw new CompactionInterruptedException(ci.getCompactionInfo());

          try (CompactionAwareWriter writer =
              getCompactionAwareWriter(cfs, transaction, actuallyCompact)) {
            estimatedKeys = writer.estimatedKeys();
            while (iter.hasNext()) {
              if (ci.isStopRequested())
                throw new CompactionInterruptedException(ci.getCompactionInfo());

              try (AbstractCompactedRow row = iter.next()) {
                if (writer.append(row)) totalKeysWritten++;

                if (System.nanoTime() - lastCheckObsoletion > TimeUnit.MINUTES.toNanos(1L)) {
                  controller.maybeRefreshOverlaps();
                  lastCheckObsoletion = System.nanoTime();
                }
              }
            }

            // don't replace old sstables yet, as we need to mark the compaction finished in the
            // system table
            newSStables = writer.finish();
          } finally {
            // point of no return -- the new sstables are live on disk; next we'll start deleting
            // the old ones
            // (in replaceCompactedSSTables)
            if (taskId != null) SystemKeyspace.finishCompaction(taskId);

            if (collector != null) collector.finishCompaction(ci);
          }
        }
      }

      // log a bunch of statistics about the result and save to system table compaction_history
      long dTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
      long startsize = SSTableReader.getTotalBytes(transaction.originals());
      long endsize = SSTableReader.getTotalBytes(newSStables);
      double ratio = (double) endsize / (double) startsize;

      StringBuilder newSSTableNames = new StringBuilder();
      for (SSTableReader reader : newSStables)
        newSSTableNames.append(reader.descriptor.baseFilename()).append(",");

      double mbps = dTime > 0 ? (double) endsize / (1024 * 1024) / ((double) dTime / 1000) : 0;
      long totalSourceRows = 0;
      String mergeSummary =
          updateCompactionHistory(
              cfs.keyspace.getName(), cfs.getColumnFamilyName(), ci, startsize, endsize);
      logger.info(
          String.format(
              "Compacted (%s) %d sstables to [%s] to level=%d.  %,d bytes to %,d (~%d%% of original) in %,dms = %fMB/s.  %,d total partitions merged to %,d.  Partition merge counts were {%s}",
              taskIdLoggerMsg,
              transaction.originals().size(),
              newSSTableNames.toString(),
              getLevel(),
              startsize,
              endsize,
              (int) (ratio * 100),
              dTime,
              mbps,
              totalSourceRows,
              totalKeysWritten,
              mergeSummary));
      logger.debug(
          String.format(
              "CF Total Bytes Compacted: %,d", CompactionTask.addToTotalBytesCompacted(endsize)));
      logger.debug(
          "Actual #keys: {}, Estimated #keys:{}, Err%: {}",
          totalKeysWritten,
          estimatedKeys,
          ((double) (totalKeysWritten - estimatedKeys) / totalKeysWritten));

      if (offline) Refs.release(Refs.selfRefs(newSStables));
    }
  }