public static SSTableReader sstable(
     int generation, int size, boolean keepRef, ColumnFamilyStore cfs) {
   Descriptor descriptor =
       new Descriptor(
           cfs.getDirectories().getDirectoryForNewSSTables(),
           cfs.keyspace.getName(),
           cfs.getColumnFamilyName(),
           generation);
   Set<Component> components =
       ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
   for (Component component : components) {
     File file = new File(descriptor.filenameFor(component));
     try {
       file.createNewFile();
     } catch (IOException e) {
     }
   }
   if (size > 0) {
     try {
       File file = new File(descriptor.filenameFor(Component.DATA));
       try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
         raf.setLength(size);
       }
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
   }
   SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.emptyList());
   StatsMetadata metadata =
       (StatsMetadata)
           new MetadataCollector(cfs.metadata.comparator)
               .finalizeMetadata(
                   cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
               .get(MetadataType.STATS);
   SSTableReader reader =
       SSTableReader.internalOpen(
           descriptor,
           components,
           cfs.metadata,
           segmentedFile.sharedCopy(),
           segmentedFile.sharedCopy(),
           indexSummary.sharedCopy(),
           new AlwaysPresentFilter(),
           1L,
           metadata,
           SSTableReader.OpenReason.NORMAL,
           header);
   reader.first = reader.last = readerBounds(generation);
   if (!keepRef) reader.selfRef().release();
   return reader;
 }
  private static SSTableReader sstable(
      File dataFolder, ColumnFamilyStore cfs, int generation, int size) throws IOException {
    Descriptor descriptor =
        new Descriptor(dataFolder, cfs.keyspace.getName(), cfs.getTableName(), generation);
    Set<Component> components =
        ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
    for (Component component : components) {
      File file = new File(descriptor.filenameFor(component));
      if (!file.exists()) assertTrue(file.createNewFile());
      try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
        raf.setLength(size);
      }
    }

    SegmentedFile dFile =
        new BufferedSegmentedFile(
            new ChannelProxy(new File(descriptor.filenameFor(Component.DATA))),
            RandomAccessReader.DEFAULT_BUFFER_SIZE,
            0);
    SegmentedFile iFile =
        new BufferedSegmentedFile(
            new ChannelProxy(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))),
            RandomAccessReader.DEFAULT_BUFFER_SIZE,
            0);

    SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.emptyList());
    StatsMetadata metadata =
        (StatsMetadata)
            new MetadataCollector(cfs.metadata.comparator)
                .finalizeMetadata(
                    cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
                .get(MetadataType.STATS);
    SSTableReader reader =
        SSTableReader.internalOpen(
            descriptor,
            components,
            cfs.metadata,
            dFile,
            iFile,
            MockSchema.indexSummary.sharedCopy(),
            new AlwaysPresentFilter(),
            1L,
            metadata,
            SSTableReader.OpenReason.NORMAL,
            header);
    reader.first = reader.last = MockSchema.readerBounds(generation);
    return reader;
  }
Example #3
0
  @SuppressWarnings("resource") // log and writer closed by SSTableTxnWriter
  public SSTableTxnWriter createFlushWriter(
      String filename, PartitionColumns columns, EncodingStats stats) {
    // we operate "offline" here, as we expose the resulting reader consciously when done
    // (although we may want to modify this behaviour in future, to encapsulate full flush behaviour
    // in LifecycleTransaction)
    LifecycleTransaction txn = null;
    try {
      txn = LifecycleTransaction.offline(OperationType.FLUSH);
      MetadataCollector sstableMetadataCollector =
          new MetadataCollector(cfs.metadata.comparator)
              .commitLogIntervals(
                  new IntervalSet(commitLogLowerBound.get(), commitLogUpperBound.get()));

      return new SSTableTxnWriter(
          txn,
          cfs.createSSTableMultiWriter(
              Descriptor.fromFilename(filename),
              (long) partitions.size(),
              ActiveRepairService.UNREPAIRED_SSTABLE,
              sstableMetadataCollector,
              new SerializationHeader(true, cfs.metadata, columns, stats),
              txn));
    } catch (Throwable t) {
      if (txn != null) txn.close();
      throw t;
    }
  }
  public void switchCompactionLocation(Directories.DataDirectory location) {
    @SuppressWarnings("resource")
    SSTableWriter writer =
        SSTableWriter.create(
            Descriptor.fromFilename(
                cfs.getSSTablePath(getDirectories().getLocationForDisk(location))),
            estimatedTotalKeys / estimatedSSTables,
            minRepairedAt,
            cfs.metadata,
            new MetadataCollector(allSSTables, cfs.metadata.comparator, level),
            SerializationHeader.make(cfs.metadata, nonExpiredSSTables),
            txn);

    sstableWriter.switchWriter(writer);
  }
  /**
   * Construct table schema from info stored in SSTable's Stats.db
   *
   * @param desc SSTable's descriptor
   * @return Restored CFMetaData
   * @throws IOException when Stats.db cannot be read
   */
  public static CFMetaData metadataFromSSTable(Descriptor desc) throws IOException {
    if (!desc.version.storeRows()) throw new IOException("pre-3.0 SSTable is not supported.");

    EnumSet<MetadataType> types =
        EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
    Map<MetadataType, MetadataComponent> sstableMetadata =
        desc.getMetadataSerializer().deserialize(desc, types);
    ValidationMetadata validationMetadata =
        (ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
    SerializationHeader.Component header =
        (SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);

    IPartitioner partitioner =
        SecondaryIndexManager.isIndexColumnFamily(desc.cfname)
            ? new LocalPartitioner(header.getKeyType())
            : FBUtilities.newPartitioner(validationMetadata.partitioner);

    CFMetaData.Builder builder =
        CFMetaData.Builder.create("keyspace", "table").withPartitioner(partitioner);
    header
        .getStaticColumns()
        .entrySet()
        .stream()
        .forEach(
            entry -> {
              ColumnIdentifier ident =
                  ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
              builder.addStaticColumn(ident, entry.getValue());
            });
    header
        .getRegularColumns()
        .entrySet()
        .stream()
        .forEach(
            entry -> {
              ColumnIdentifier ident =
                  ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
              builder.addRegularColumn(ident, entry.getValue());
            });
    builder.addPartitionKey("PartitionKey", header.getKeyType());
    for (int i = 0; i < header.getClusteringTypes().size(); i++) {
      builder.addClusteringColumn(
          "clustering" + (i > 0 ? i : ""), header.getClusteringTypes().get(i));
    }
    return builder.build();
  }
  @Test
  public void testGetNewNames() throws IOException {
    Descriptor desc =
        Descriptor.fromFilename(
            new File("Keyspace1", "Keyspace1-Standard1-ia-500-Data.db").toString());
    // assert !desc.isLatestVersion; // minimum compatible version -- for now it is the latest as
    // well
    PendingFile inContext =
        new PendingFile(
            null, desc, "Data.db", Arrays.asList(Pair.create(0L, 1L)), OperationType.BOOTSTRAP);

    PendingFile outContext = StreamIn.getContextMapping(inContext);
    // filename and generation are expected to have changed
    assert !inContext.getFilename().equals(outContext.getFilename());

    // nothing else should
    assertEquals(inContext.component, outContext.component);
    assertEquals(desc.ksname, outContext.desc.ksname);
    assertEquals(desc.cfname, outContext.desc.cfname);
    assertEquals(desc.version, outContext.desc.version);
  }
  /**
   * Given arguments specifying an SSTable, and optionally an output file, export the contents of
   * the SSTable to JSON.
   *
   * @param args command lines arguments
   * @throws ConfigurationException on configuration failure (wrong params given)
   */
  public static void main(String[] args) throws ConfigurationException {
    CommandLineParser parser = new PosixParser();
    try {
      cmd = parser.parse(options, args);
    } catch (ParseException e1) {
      System.err.println(e1.getMessage());
      printUsage();
      System.exit(1);
    }

    if (cmd.getArgs().length != 1) {
      System.err.println("You must supply exactly one sstable");
      printUsage();
      System.exit(1);
    }

    String[] keys = cmd.getOptionValues(KEY_OPTION);
    HashSet<String> excludes =
        new HashSet<>(
            Arrays.asList(
                cmd.getOptionValues(EXCLUDE_KEY_OPTION) == null
                    ? new String[0]
                    : cmd.getOptionValues(EXCLUDE_KEY_OPTION)));
    String ssTableFileName = new File(cmd.getArgs()[0]).getAbsolutePath();

    if (Descriptor.isLegacyFile(new File(ssTableFileName))) {
      System.err.println("Unsupported legacy sstable");
      System.exit(1);
    }
    if (!new File(ssTableFileName).exists()) {
      System.err.println("Cannot find file " + ssTableFileName);
      System.exit(1);
    }
    Descriptor desc = Descriptor.fromFilename(ssTableFileName);
    try {
      CFMetaData metadata = metadataFromSSTable(desc);
      if (cmd.hasOption(ENUMERATE_KEYS_OPTION)) {
        JsonTransformer.keysToJson(
            null,
            iterToStream(new KeyIterator(desc, metadata)),
            cmd.hasOption(RAW_TIMESTAMPS),
            metadata,
            System.out);
      } else {
        SSTableReader sstable = SSTableReader.openNoValidation(desc, metadata);
        IPartitioner partitioner = sstable.getPartitioner();
        final ISSTableScanner currentScanner;
        if ((keys != null) && (keys.length > 0)) {
          List<AbstractBounds<PartitionPosition>> bounds =
              Arrays.stream(keys)
                  .filter(key -> !excludes.contains(key))
                  .map(metadata.getKeyValidator()::fromString)
                  .map(partitioner::decorateKey)
                  .sorted()
                  .map(DecoratedKey::getToken)
                  .map(token -> new Bounds<>(token.minKeyBound(), token.maxKeyBound()))
                  .collect(Collectors.toList());
          currentScanner = sstable.getScanner(bounds.iterator());
        } else {
          currentScanner = sstable.getScanner();
        }
        Stream<UnfilteredRowIterator> partitions =
            iterToStream(currentScanner)
                .filter(
                    i ->
                        excludes.isEmpty()
                            || !excludes.contains(
                                metadata.getKeyValidator().getString(i.partitionKey().getKey())));
        if (cmd.hasOption(DEBUG_OUTPUT_OPTION)) {
          AtomicLong position = new AtomicLong();
          partitions.forEach(
              partition -> {
                position.set(currentScanner.getCurrentPosition());

                if (!partition.partitionLevelDeletion().isLive()) {
                  System.out.println(
                      "["
                          + metadata.getKeyValidator().getString(partition.partitionKey().getKey())
                          + "]@"
                          + position.get()
                          + " "
                          + partition.partitionLevelDeletion());
                }
                if (!partition.staticRow().isEmpty()) {
                  System.out.println(
                      "["
                          + metadata.getKeyValidator().getString(partition.partitionKey().getKey())
                          + "]@"
                          + position.get()
                          + " "
                          + partition.staticRow().toString(metadata, true));
                }
                partition.forEachRemaining(
                    row -> {
                      System.out.println(
                          "["
                              + metadata
                                  .getKeyValidator()
                                  .getString(partition.partitionKey().getKey())
                              + "]@"
                              + position.get()
                              + " "
                              + row.toString(metadata, false, true));
                      position.set(currentScanner.getCurrentPosition());
                    });
              });
        } else {
          JsonTransformer.toJson(
              currentScanner, partitions, cmd.hasOption(RAW_TIMESTAMPS), metadata, System.out);
        }
      }
    } catch (IOException e) {
      // throwing exception outside main with broken pipe causes windows cmd to hang
      e.printStackTrace(System.err);
    }

    System.exit(0);
  }
  public static void main(String args[]) throws IOException {
    Options options = Options.parseArgs(args);
    try {
      // load keyspace descriptions.
      DatabaseDescriptor.loadSchemas();

      String ksName = null;
      String cfName = null;
      Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
      for (String filename : options.filenames) {
        File file = new File(filename);
        if (!file.exists()) {
          System.out.println("Skipping inexisting file " + file);
          continue;
        }

        Pair<Descriptor, Component> pair =
            SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
        if (pair == null) {
          System.out.println("Skipping non sstable file " + file);
          continue;
        }
        Descriptor desc = pair.left;

        if (ksName == null) ksName = desc.ksname;
        else if (!ksName.equals(desc.ksname))
          throw new IllegalArgumentException("All sstables must be part of the same keyspace");

        if (cfName == null) cfName = desc.cfname;
        else if (!cfName.equals(desc.cfname))
          throw new IllegalArgumentException("All sstables must be part of the same column family");

        Set<Component> components =
            new HashSet<Component>(
                Arrays.asList(
                    new Component[] {
                      Component.DATA,
                      Component.PRIMARY_INDEX,
                      Component.FILTER,
                      Component.COMPRESSION_INFO,
                      Component.STATS
                    }));

        Iterator<Component> iter = components.iterator();
        while (iter.hasNext()) {
          Component component = iter.next();
          if (!(new File(desc.filenameFor(component)).exists())) iter.remove();
        }
        parsedFilenames.put(desc, components);
      }

      if (ksName == null || cfName == null) {
        System.err.println("No valid sstables to split");
        System.exit(1);
      }

      // Do not load sstables since they might be broken
      Table table = Table.openWithoutSSTables(ksName);
      ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);

      String snapshotName = "pre-split-" + System.currentTimeMillis();

      List<SSTableReader> sstables = new ArrayList<SSTableReader>();
      for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet()) {
        try {
          SSTableReader sstable =
              SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs.metadata);
          sstables.add(sstable);

          if (options.snapshot) {
            File snapshotDirectory =
                Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
            sstable.createLinks(snapshotDirectory.getPath());
          }

        } catch (Exception e) {
          System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
          if (options.debug) e.printStackTrace(System.err);
        }
      }
      if (options.snapshot)
        System.out.println(
            String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));

      cfs.getDataTracker().markCompacting(sstables);
      for (SSTableReader sstable : sstables) {
        try {
          new SSTableSplitter(cfs, sstable, options.sizeInMB).split();

          // Remove the sstable
          sstable.markCompacted();
          sstable.releaseReference();
        } catch (Exception e) {
          System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
          if (options.debug) e.printStackTrace(System.err);
        }
      }
      SSTableDeletingTask.waitForDeletions();
      System.exit(0); // We need that to stop non daemonized threads
    } catch (Exception e) {
      System.err.println(e.getMessage());
      if (options.debug) e.printStackTrace(System.err);
      System.exit(1);
    }
  }