コード例 #1
0
  @Test
  public void testCommitSameDesc() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstableOld1 = sstable(dataFolder, cfs, 0, 128);
    SSTableReader sstableOld2 = sstable(dataFolder, cfs, 0, 256);
    SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);

    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    log.trackNew(sstableNew);

    sstableOld1.setReplaced();

    LogTransaction.SSTableTidier tidier = log.obsoleted(sstableOld2);
    assertNotNull(tidier);

    log.finish();

    sstableOld2.markObsolete(tidier);

    sstableOld1.selfRef().release();
    sstableOld2.selfRef().release();

    assertFiles(dataFolder.getPath(), new HashSet<>(sstableNew.getAllFilePaths()));

    sstableNew.selfRef().release();
  }
コード例 #2
0
  private static void testCorruptRecord(
      BiConsumer<LogTransaction, SSTableReader> modifier, boolean isRecoverable)
      throws IOException {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
    SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);

    // simulate tracking sstables with a committed transaction except the checksum will be wrong
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    log.trackNew(sstableNew);
    log.obsoleted(sstableOld);

    // Modify the transaction log or disk state for sstableOld
    modifier.accept(log, sstableOld);

    assertNull(log.complete(null));

    sstableOld.selfRef().release();
    sstableNew.selfRef().release();

    // The files on disk, for old files make sure to exclude the files that were deleted by the
    // modifier
    Set<String> newFiles = sstableNew.getAllFilePaths().stream().collect(Collectors.toSet());
    Set<String> oldFiles =
        sstableOld
            .getAllFilePaths()
            .stream()
            .filter(p -> new File(p).exists())
            .collect(Collectors.toSet());

    // This should filter as in progress since the last record is corrupt
    assertFiles(newFiles, getTemporaryFiles(dataFolder));
    assertFiles(oldFiles, getFinalFiles(dataFolder));

    if (isRecoverable) { // the corruption is recoverable but the commit record is unreadable so the
      // transaction is still in progress

      // This should remove new files
      LogTransaction.removeUnfinishedLeftovers(cfs.metadata);

      // make sure to exclude the old files that were deleted by the modifier
      assertFiles(dataFolder.getPath(), oldFiles);
    } else { // if an intermediate line was also modified, it should ignore the tx log file

      // This should not remove any files
      LogTransaction.removeUnfinishedLeftovers(cfs.metadata);

      assertFiles(
          dataFolder.getPath(),
          Sets.newHashSet(Iterables.concat(newFiles, oldFiles, log.logFilePaths())));
    }
  }
コード例 #3
0
  @Test
  public void testRemoveUnfinishedLeftovers_abort_multipleFolders() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);

    File origiFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    File dataFolder1 = new File(origiFolder, "1");
    File dataFolder2 = new File(origiFolder, "2");
    Files.createDirectories(dataFolder1.toPath());
    Files.createDirectories(dataFolder2.toPath());

    SSTableReader[] sstables = {
      sstable(dataFolder1, cfs, 0, 128),
      sstable(dataFolder1, cfs, 1, 128),
      sstable(dataFolder2, cfs, 2, 128),
      sstable(dataFolder2, cfs, 3, 128)
    };

    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    LogTransaction.SSTableTidier[] tidiers = {
      log.obsoleted(sstables[0]), log.obsoleted(sstables[2])
    };

    log.trackNew(sstables[1]);
    log.trackNew(sstables[3]);

    Collection<File> logFiles = log.logFiles();
    Assert.assertEquals(2, logFiles.size());

    // fake an abort
    log.txnFile().abort();

    Arrays.stream(sstables).forEach(s -> s.selfRef().release());

    // test listing
    Assert.assertEquals(
        sstables[1].getAllFilePaths().stream().map(File::new).collect(Collectors.toSet()),
        getTemporaryFiles(dataFolder1));
    Assert.assertEquals(
        sstables[3].getAllFilePaths().stream().map(File::new).collect(Collectors.toSet()),
        getTemporaryFiles(dataFolder2));

    // normally called at startup
    LogTransaction.removeUnfinishedLeftovers(Arrays.asList(dataFolder1, dataFolder2));

    // old tables should be only table left
    assertFiles(dataFolder1.getPath(), new HashSet<>(sstables[0].getAllFilePaths()));
    assertFiles(dataFolder2.getPath(), new HashSet<>(sstables[2].getAllFilePaths()));

    // complete the transaction to avoid LEAK errors
    Arrays.stream(tidiers).forEach(LogTransaction.SSTableTidier::run);
    assertNull(log.complete(null));
  }
コード例 #4
0
      Transaction(ColumnFamilyStore cfs, LogTransaction txnLogs) throws IOException {
        this.cfs = cfs;
        this.txnLogs = txnLogs;
        this.dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
        this.sstableOld = sstable(dataFolder, cfs, 0, 128);
        this.sstableNew = sstable(dataFolder, cfs, 1, 128);

        assertNotNull(txnLogs);
        assertNotNull(txnLogs.id());
        Assert.assertEquals(OperationType.COMPACTION, txnLogs.type());

        txnLogs.trackNew(sstableNew);
        tidier = txnLogs.obsoleted(sstableOld);
        assertNotNull(tidier);
      }
コード例 #5
0
  @Test
  public void testAbortOnlyNew() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);

    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    log.trackNew(sstable);
    log.abort();

    sstable.selfRef().release();

    assertFiles(dataFolder.getPath(), new HashSet<>());
  }
コード例 #6
0
  private static void testObsoletedFilesChanged(Consumer<SSTableReader> modifier)
      throws IOException {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
    SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);

    // simulate tracking sstables with a committed transaction except the checksum will be wrong
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    log.trackNew(sstableNew);
    /*TransactionLog.SSTableTidier tidier =*/ log.obsoleted(sstableOld);

    // modify the old sstable files
    modifier.accept(sstableOld);

    // Fake a commit
    log.txnFile().commit();

    // This should not remove the old files
    LogTransaction.removeUnfinishedLeftovers(cfs.metadata);

    assertFiles(
        dataFolder.getPath(),
        Sets.newHashSet(
            Iterables.concat(
                sstableNew.getAllFilePaths(), sstableOld.getAllFilePaths(), log.logFilePaths())));

    sstableOld.selfRef().release();
    sstableNew.selfRef().release();

    // complete the transaction to avoid LEAK errors
    assertNull(log.complete(null));

    assertFiles(
        dataFolder.getPath(),
        Sets.newHashSet(
            Iterables.concat(
                sstableNew.getAllFilePaths(), sstableOld.getAllFilePaths(), log.logFilePaths())));
  }
コード例 #7
0
  @Test
  public void testUntrack() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);

    // complete a transaction without keep the new files since they were untracked
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    log.trackNew(sstableNew);
    log.untrackNew(sstableNew);

    log.finish();

    sstableNew.selfRef().release();
    Thread.sleep(1);
    LogTransaction.waitForDeletions();

    assertFiles(dataFolder.getPath(), Collections.<String>emptySet());
  }
コード例 #8
0
  @Test
  public void testRemoveUnfinishedLeftovers_commit() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
    SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);

    // simulate tracking sstables with a committed transaction (new log file deleted)
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    log.trackNew(sstableNew);
    LogTransaction.SSTableTidier tidier = log.obsoleted(sstableOld);

    // Fake a commit
    log.txnFile().commit();

    Set<File> tmpFiles =
        sstableOld.getAllFilePaths().stream().map(File::new).collect(Collectors.toSet());

    sstableNew.selfRef().release();
    sstableOld.selfRef().release();

    Assert.assertEquals(tmpFiles, getTemporaryFiles(sstableOld.descriptor.directory));

    // normally called at startup
    LogTransaction.removeUnfinishedLeftovers(cfs.metadata);

    // sstableNew should be only table left
    Directories directories = new Directories(cfs.metadata);
    Map<Descriptor, Set<Component>> sstables =
        directories.sstableLister(Directories.OnTxnErr.THROW).list();
    assertEquals(1, sstables.size());

    assertFiles(dataFolder.getPath(), new HashSet<>(sstableNew.getAllFilePaths()));

    // complete the transaction to avoid LEAK errors
    tidier.run();
    assertNull(log.complete(null));
  }
コード例 #9
0
  @Test
  public void testCommitMultipleFolders() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);

    File origiFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    File dataFolder1 = new File(origiFolder, "1");
    File dataFolder2 = new File(origiFolder, "2");
    Files.createDirectories(dataFolder1.toPath());
    Files.createDirectories(dataFolder2.toPath());

    SSTableReader[] sstables = {
      sstable(dataFolder1, cfs, 0, 128),
      sstable(dataFolder1, cfs, 1, 128),
      sstable(dataFolder2, cfs, 2, 128),
      sstable(dataFolder2, cfs, 3, 128)
    };

    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    LogTransaction.SSTableTidier[] tidiers = {
      log.obsoleted(sstables[0]), log.obsoleted(sstables[2])
    };

    log.trackNew(sstables[1]);
    log.trackNew(sstables[3]);

    log.finish();

    sstables[0].markObsolete(tidiers[0]);
    sstables[2].markObsolete(tidiers[1]);

    Arrays.stream(sstables).forEach(s -> s.selfRef().release());
    LogTransaction.waitForDeletions();

    assertFiles(dataFolder1.getPath(), new HashSet<>(sstables[1].getAllFilePaths()));
    assertFiles(dataFolder2.getPath(), new HashSet<>(sstables[3].getAllFilePaths()));
  }
コード例 #10
0
  @Test
  public void testGetTemporaryFilesMultipleFolders() throws IOException {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);

    File origiFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    File dataFolder1 = new File(origiFolder, "1");
    File dataFolder2 = new File(origiFolder, "2");
    Files.createDirectories(dataFolder1.toPath());
    Files.createDirectories(dataFolder2.toPath());

    SSTableReader[] sstables = {
      sstable(dataFolder1, cfs, 0, 128),
      sstable(dataFolder1, cfs, 1, 128),
      sstable(dataFolder2, cfs, 2, 128),
      sstable(dataFolder2, cfs, 3, 128)
    };

    // they should all have the same number of files since they are created in the same way
    int numSStableFiles = sstables[0].getAllFilePaths().size();

    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    for (File dataFolder : new File[] {dataFolder1, dataFolder2}) {
      Set<File> tmpFiles = getTemporaryFiles(dataFolder);
      assertNotNull(tmpFiles);
      assertEquals(0, tmpFiles.size());
    }

    LogTransaction.SSTableTidier[] tidiers = {
      log.obsoleted(sstables[0]), log.obsoleted(sstables[2])
    };

    log.trackNew(sstables[1]);
    log.trackNew(sstables[3]);

    for (File dataFolder : new File[] {dataFolder1, dataFolder2}) {
      Set<File> tmpFiles = getTemporaryFiles(dataFolder);
      assertNotNull(tmpFiles);
      assertEquals(numSStableFiles, tmpFiles.size());
    }

    log.finish();

    for (File dataFolder : new File[] {dataFolder1, dataFolder2}) {
      Set<File> tmpFiles = getTemporaryFiles(dataFolder);
      assertNotNull(tmpFiles);
      assertEquals(numSStableFiles, tmpFiles.size());
    }

    sstables[0].markObsolete(tidiers[0]);
    sstables[2].markObsolete(tidiers[1]);

    Arrays.stream(sstables).forEach(s -> s.selfRef().release());
    LogTransaction.waitForDeletions();

    for (File dataFolder : new File[] {dataFolder1, dataFolder2}) {
      Set<File> tmpFiles = getTemporaryFiles(dataFolder);
      assertNotNull(tmpFiles);
      assertEquals(0, tmpFiles.size());
    }
  }
コード例 #11
0
  @Test
  public void testGetTemporaryFiles() throws IOException {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    SSTableReader sstable1 = sstable(dataFolder, cfs, 0, 128);

    Set<File> tmpFiles = getTemporaryFiles(dataFolder);
    assertNotNull(tmpFiles);
    assertEquals(0, tmpFiles.size());

    try (LogTransaction log = new LogTransaction(OperationType.WRITE)) {
      Directories directories = new Directories(cfs.metadata);

      File[] beforeSecondSSTable = dataFolder.listFiles(pathname -> !pathname.isDirectory());

      SSTableReader sstable2 = sstable(dataFolder, cfs, 1, 128);
      log.trackNew(sstable2);

      Map<Descriptor, Set<Component>> sstables =
          directories.sstableLister(Directories.OnTxnErr.THROW).list();
      assertEquals(2, sstables.size());

      // this should contain sstable1, sstable2 and the transaction log file
      File[] afterSecondSSTable = dataFolder.listFiles(pathname -> !pathname.isDirectory());

      int numNewFiles = afterSecondSSTable.length - beforeSecondSSTable.length;
      assertEquals(
          numNewFiles - 1,
          sstable2.getAllFilePaths().size()); // new files except for transaction log file

      tmpFiles = getTemporaryFiles(dataFolder);
      assertNotNull(tmpFiles);
      assertEquals(numNewFiles - 1, tmpFiles.size());

      File ssTable2DataFile = new File(sstable2.descriptor.filenameFor(Component.DATA));
      File ssTable2IndexFile = new File(sstable2.descriptor.filenameFor(Component.PRIMARY_INDEX));

      assertTrue(tmpFiles.contains(ssTable2DataFile));
      assertTrue(tmpFiles.contains(ssTable2IndexFile));

      List<File> files = directories.sstableLister(Directories.OnTxnErr.THROW).listFiles();
      List<File> filesNoTmp =
          directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true).listFiles();
      assertNotNull(files);
      assertNotNull(filesNoTmp);

      assertTrue(files.contains(ssTable2DataFile));
      assertTrue(files.contains(ssTable2IndexFile));

      assertFalse(filesNoTmp.contains(ssTable2DataFile));
      assertFalse(filesNoTmp.contains(ssTable2IndexFile));

      log.finish();

      // Now it should be empty since the transaction has finished
      tmpFiles = getTemporaryFiles(dataFolder);
      assertNotNull(tmpFiles);
      assertEquals(0, tmpFiles.size());

      filesNoTmp =
          directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true).listFiles();
      assertNotNull(filesNoTmp);
      assertTrue(filesNoTmp.contains(ssTable2DataFile));
      assertTrue(filesNoTmp.contains(ssTable2IndexFile));

      sstable1.selfRef().release();
      sstable2.selfRef().release();
    }
  }
コード例 #12
0
  private static void testRemoveUnfinishedLeftovers_multipleFolders_errorConditions(
      Consumer<LogTransaction> modifier, boolean shouldCommit) throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);

    File origiFolder = new Directories(cfs.metadata).getDirectoryForNewSSTables();
    File dataFolder1 = new File(origiFolder, "1");
    File dataFolder2 = new File(origiFolder, "2");
    Files.createDirectories(dataFolder1.toPath());
    Files.createDirectories(dataFolder2.toPath());

    SSTableReader[] sstables = {
      sstable(dataFolder1, cfs, 0, 128),
      sstable(dataFolder1, cfs, 1, 128),
      sstable(dataFolder2, cfs, 2, 128),
      sstable(dataFolder2, cfs, 3, 128)
    };

    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);

    LogTransaction.SSTableTidier[] tidiers = {
      log.obsoleted(sstables[0]), log.obsoleted(sstables[2])
    };

    log.trackNew(sstables[1]);
    log.trackNew(sstables[3]);

    // fake some error condition on the txn logs
    modifier.accept(log);

    Arrays.stream(sstables).forEach(s -> s.selfRef().release());

    LogTransaction.removeUnfinishedLeftovers(Arrays.asList(dataFolder1, dataFolder2));
    LogTransaction.waitForDeletions();

    if (shouldCommit) {
      // only new sstables should still be there
      assertFiles(dataFolder1.getPath(), new HashSet<>(sstables[1].getAllFilePaths()));
      assertFiles(dataFolder2.getPath(), new HashSet<>(sstables[3].getAllFilePaths()));
    } else {
      // all files should still be there
      assertFiles(
          dataFolder1.getPath(),
          Sets.newHashSet(
              Iterables.concat(
                  sstables[0].getAllFilePaths(),
                  sstables[1].getAllFilePaths(),
                  Collections.singleton(log.logFilePaths().get(0)))));
      assertFiles(
          dataFolder2.getPath(),
          Sets.newHashSet(
              Iterables.concat(
                  sstables[2].getAllFilePaths(),
                  sstables[3].getAllFilePaths(),
                  Collections.singleton(log.logFilePaths().get(1)))));
    }

    // complete the transaction to avoid LEAK errors
    Arrays.stream(tidiers).forEach(LogTransaction.SSTableTidier::run);
    log.txnFile().commit(); // just anything to make sure transaction tidier will finish
    assertNull(log.complete(null));
  }