@Test
  public void testRemoveUnfinishedLeftovers_multipleFolders_tooManyFinalRecords() throws Throwable {
    testRemoveUnfinishedLeftovers_multipleFolders_errorConditions(
        txn -> {
          List<File> logFiles = txn.logFiles();
          Assert.assertEquals(2, logFiles.size());

          // insert mismatched records
          FileUtils.append(logFiles.get(0), LogRecord.makeCommit(System.currentTimeMillis()).raw);
          FileUtils.append(logFiles.get(1), LogRecord.makeCommit(System.currentTimeMillis()).raw);
          FileUtils.append(logFiles.get(1), LogRecord.makeCommit(System.currentTimeMillis()).raw);
        },
        false);
  }
 @Test
 public void testMissingChecksumSecondFromLastLine() throws IOException {
   testCorruptRecord(
       (t, s) -> { // Fake two lines without a checksum
         long now = System.currentTimeMillis();
         t.logFiles().forEach(f -> FileUtils.append(f, String.format("add:[ma-3-big,%d,4]", now)));
         t.logFiles().forEach(f -> FileUtils.append(f, String.format("commit:[%d,0,0]", now)));
       },
       false);
 }
 @Test
 public void testMissingChecksumLastLine() throws IOException {
   testCorruptRecord(
       (t, s) -> {
         // Fake a commit without a checksum
         long now = System.currentTimeMillis();
         t.logFiles().forEach(f -> FileUtils.append(f, String.format("commit:[%d,0,0]", now)));
       },
       true);
 }
 @Test
 public void testWrongChecksumLastLineWrongRecordFormat() throws IOException {
   testCorruptRecord(
       (t, s) -> { // Fake a commit with invalid checksum and a wrong record format (extra spaces)
         long now = System.currentTimeMillis();
         t.logFiles()
             .forEach(
                 f ->
                     FileUtils.append(f, String.format("commit:[%d ,0 ,0 ][%d]", now, 12345678L)));
       },
       true);
 }
  @Test
  public void testRemoveUnfinishedLeftovers_multipleFolders_missingFinalRecords_second()
      throws Throwable {
    testRemoveUnfinishedLeftovers_multipleFolders_errorConditions(
        txn -> {
          List<File> logFiles = txn.logFiles();
          Assert.assertEquals(2, logFiles.size());

          // insert only one commit record
          FileUtils.append(logFiles.get(1), LogRecord.makeCommit(System.currentTimeMillis()).raw);
        },
        true);
  }
 @Test
 public void testObsoletedDataFileUpdateTimeChanged() throws IOException {
   testObsoletedFilesChanged(
       sstable -> {
         // increase the modification time of the Data file
         for (String filePath : sstable.getAllFilePaths()) {
           if (filePath.endsWith("Data.db"))
             assertTrue(
                 new File(filePath)
                     .setLastModified(System.currentTimeMillis() + 60000)); // one minute later
         }
       });
 }
  @Test
  public void testRemoveUnfinishedLeftovers_multipleFolders_partialFinalRecords_second()
      throws Throwable {
    testRemoveUnfinishedLeftovers_multipleFolders_errorConditions(
        txn -> {
          List<File> logFiles = txn.logFiles();
          Assert.assertEquals(2, logFiles.size());

          // insert a full record and a partial one
          String finalRecord = LogRecord.makeCommit(System.currentTimeMillis()).raw;
          int toChop = finalRecord.length() / 2;
          FileUtils.append(logFiles.get(0), finalRecord);
          FileUtils.append(
              logFiles.get(1), finalRecord.substring(0, finalRecord.length() - toChop));
        },
        true);
  }
  @Test
  public void testWrongChecksumLastLineMissingFile() throws IOException {
    testCorruptRecord(
        (t, s) -> { // Fake a commit with invalid checksum and also delete one of the old files
          for (String filePath : s.getAllFilePaths()) {
            if (filePath.endsWith("Data.db")) {
              assertTrue(FileUtils.delete(filePath));
              assertNull(t.txnFile().syncFolder(null));
              break;
            }
          }

          long now = System.currentTimeMillis();
          t.logFiles()
              .forEach(
                  f -> FileUtils.append(f, String.format("commit:[%d,0,0][%d]", now, 12345678L)));
        },
        false);
  }
  @Test
  public void testRemoveUnfinishedLeftovers_multipleFolders_partialNonFinalRecord_second()
      throws Throwable {
    testRemoveUnfinishedLeftovers_multipleFolders_errorConditions(
        txn -> {
          List<File> logFiles = txn.logFiles();
          Assert.assertEquals(2, logFiles.size());

          // insert a partial sstable record and a full commit record
          String sstableRecord =
              LogRecord.make(LogRecord.Type.ADD, Collections.emptyList(), 0, "abc").raw;
          int toChop = sstableRecord.length() / 2;
          FileUtils.append(logFiles.get(0), sstableRecord);
          FileUtils.append(
              logFiles.get(1), sstableRecord.substring(0, sstableRecord.length() - toChop));
          String finalRecord = LogRecord.makeCommit(System.currentTimeMillis()).raw;
          FileUtils.append(logFiles.get(0), finalRecord);
          FileUtils.append(logFiles.get(1), finalRecord);
        },
        false);
  }
Пример #10
0
  public static void main(String args[]) throws IOException {
    Options options = Options.parseArgs(args);
    try {
      // load keyspace descriptions.
      DatabaseDescriptor.loadSchemas();

      String ksName = null;
      String cfName = null;
      Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
      for (String filename : options.filenames) {
        File file = new File(filename);
        if (!file.exists()) {
          System.out.println("Skipping inexisting file " + file);
          continue;
        }

        Pair<Descriptor, Component> pair =
            SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
        if (pair == null) {
          System.out.println("Skipping non sstable file " + file);
          continue;
        }
        Descriptor desc = pair.left;

        if (ksName == null) ksName = desc.ksname;
        else if (!ksName.equals(desc.ksname))
          throw new IllegalArgumentException("All sstables must be part of the same keyspace");

        if (cfName == null) cfName = desc.cfname;
        else if (!cfName.equals(desc.cfname))
          throw new IllegalArgumentException("All sstables must be part of the same column family");

        Set<Component> components =
            new HashSet<Component>(
                Arrays.asList(
                    new Component[] {
                      Component.DATA,
                      Component.PRIMARY_INDEX,
                      Component.FILTER,
                      Component.COMPRESSION_INFO,
                      Component.STATS
                    }));

        Iterator<Component> iter = components.iterator();
        while (iter.hasNext()) {
          Component component = iter.next();
          if (!(new File(desc.filenameFor(component)).exists())) iter.remove();
        }
        parsedFilenames.put(desc, components);
      }

      if (ksName == null || cfName == null) {
        System.err.println("No valid sstables to split");
        System.exit(1);
      }

      // Do not load sstables since they might be broken
      Table table = Table.openWithoutSSTables(ksName);
      ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);

      String snapshotName = "pre-split-" + System.currentTimeMillis();

      List<SSTableReader> sstables = new ArrayList<SSTableReader>();
      for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet()) {
        try {
          SSTableReader sstable =
              SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs.metadata);
          sstables.add(sstable);

          if (options.snapshot) {
            File snapshotDirectory =
                Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
            sstable.createLinks(snapshotDirectory.getPath());
          }

        } catch (Exception e) {
          System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
          if (options.debug) e.printStackTrace(System.err);
        }
      }
      if (options.snapshot)
        System.out.println(
            String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));

      cfs.getDataTracker().markCompacting(sstables);
      for (SSTableReader sstable : sstables) {
        try {
          new SSTableSplitter(cfs, sstable, options.sizeInMB).split();

          // Remove the sstable
          sstable.markCompacted();
          sstable.releaseReference();
        } catch (Exception e) {
          System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
          if (options.debug) e.printStackTrace(System.err);
        }
      }
      SSTableDeletingTask.waitForDeletions();
      System.exit(0); // We need that to stop non daemonized threads
    } catch (Exception e) {
      System.err.println(e.getMessage());
      if (options.debug) e.printStackTrace(System.err);
      System.exit(1);
    }
  }