public static SSTableReader sstable(
int generation, int size, boolean keepRef, ColumnFamilyStore cfs) {
Descriptor descriptor =
new Descriptor(
cfs.getDirectories().getDirectoryForNewSSTables(),
cfs.keyspace.getName(),
cfs.getColumnFamilyName(),
generation);
Set<Component> components =
ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
for (Component component : components) {
File file = new File(descriptor.filenameFor(component));
try {
file.createNewFile();
} catch (IOException e) {
}
}
if (size > 0) {
try {
File file = new File(descriptor.filenameFor(Component.DATA));
try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
raf.setLength(size);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.emptyList());
StatsMetadata metadata =
(StatsMetadata)
new MetadataCollector(cfs.metadata.comparator)
.finalizeMetadata(
cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
.get(MetadataType.STATS);
SSTableReader reader =
SSTableReader.internalOpen(
descriptor,
components,
cfs.metadata,
segmentedFile.sharedCopy(),
segmentedFile.sharedCopy(),
indexSummary.sharedCopy(),
new AlwaysPresentFilter(),
1L,
metadata,
SSTableReader.OpenReason.NORMAL,
header);
reader.first = reader.last = readerBounds(generation);
if (!keepRef) reader.selfRef().release();
return reader;
}
private static SSTableReader sstable(
File dataFolder, ColumnFamilyStore cfs, int generation, int size) throws IOException {
Descriptor descriptor =
new Descriptor(dataFolder, cfs.keyspace.getName(), cfs.getTableName(), generation);
Set<Component> components =
ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
for (Component component : components) {
File file = new File(descriptor.filenameFor(component));
if (!file.exists()) assertTrue(file.createNewFile());
try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
raf.setLength(size);
}
}
SegmentedFile dFile =
new BufferedSegmentedFile(
new ChannelProxy(new File(descriptor.filenameFor(Component.DATA))),
RandomAccessReader.DEFAULT_BUFFER_SIZE,
0);
SegmentedFile iFile =
new BufferedSegmentedFile(
new ChannelProxy(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))),
RandomAccessReader.DEFAULT_BUFFER_SIZE,
0);
SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.emptyList());
StatsMetadata metadata =
(StatsMetadata)
new MetadataCollector(cfs.metadata.comparator)
.finalizeMetadata(
cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
.get(MetadataType.STATS);
SSTableReader reader =
SSTableReader.internalOpen(
descriptor,
components,
cfs.metadata,
dFile,
iFile,
MockSchema.indexSummary.sharedCopy(),
new AlwaysPresentFilter(),
1L,
metadata,
SSTableReader.OpenReason.NORMAL,
header);
reader.first = reader.last = MockSchema.readerBounds(generation);
return reader;
}
@SuppressWarnings("resource") // log and writer closed by SSTableTxnWriter
public SSTableTxnWriter createFlushWriter(
String filename, PartitionColumns columns, EncodingStats stats) {
// we operate "offline" here, as we expose the resulting reader consciously when done
// (although we may want to modify this behaviour in future, to encapsulate full flush behaviour
// in LifecycleTransaction)
LifecycleTransaction txn = null;
try {
txn = LifecycleTransaction.offline(OperationType.FLUSH);
MetadataCollector sstableMetadataCollector =
new MetadataCollector(cfs.metadata.comparator)
.commitLogIntervals(
new IntervalSet(commitLogLowerBound.get(), commitLogUpperBound.get()));
return new SSTableTxnWriter(
txn,
cfs.createSSTableMultiWriter(
Descriptor.fromFilename(filename),
(long) partitions.size(),
ActiveRepairService.UNREPAIRED_SSTABLE,
sstableMetadataCollector,
new SerializationHeader(true, cfs.metadata, columns, stats),
txn));
} catch (Throwable t) {
if (txn != null) txn.close();
throw t;
}
}
public void switchCompactionLocation(Directories.DataDirectory location) {
@SuppressWarnings("resource")
SSTableWriter writer =
SSTableWriter.create(
Descriptor.fromFilename(
cfs.getSSTablePath(getDirectories().getLocationForDisk(location))),
estimatedTotalKeys / estimatedSSTables,
minRepairedAt,
cfs.metadata,
new MetadataCollector(allSSTables, cfs.metadata.comparator, level),
SerializationHeader.make(cfs.metadata, nonExpiredSSTables),
txn);
sstableWriter.switchWriter(writer);
}
/**
* Construct table schema from info stored in SSTable's Stats.db
*
* @param desc SSTable's descriptor
* @return Restored CFMetaData
* @throws IOException when Stats.db cannot be read
*/
public static CFMetaData metadataFromSSTable(Descriptor desc) throws IOException {
if (!desc.version.storeRows()) throw new IOException("pre-3.0 SSTable is not supported.");
EnumSet<MetadataType> types =
EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS, MetadataType.HEADER);
Map<MetadataType, MetadataComponent> sstableMetadata =
desc.getMetadataSerializer().deserialize(desc, types);
ValidationMetadata validationMetadata =
(ValidationMetadata) sstableMetadata.get(MetadataType.VALIDATION);
SerializationHeader.Component header =
(SerializationHeader.Component) sstableMetadata.get(MetadataType.HEADER);
IPartitioner partitioner =
SecondaryIndexManager.isIndexColumnFamily(desc.cfname)
? new LocalPartitioner(header.getKeyType())
: FBUtilities.newPartitioner(validationMetadata.partitioner);
CFMetaData.Builder builder =
CFMetaData.Builder.create("keyspace", "table").withPartitioner(partitioner);
header
.getStaticColumns()
.entrySet()
.stream()
.forEach(
entry -> {
ColumnIdentifier ident =
ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
builder.addStaticColumn(ident, entry.getValue());
});
header
.getRegularColumns()
.entrySet()
.stream()
.forEach(
entry -> {
ColumnIdentifier ident =
ColumnIdentifier.getInterned(UTF8Type.instance.getString(entry.getKey()), true);
builder.addRegularColumn(ident, entry.getValue());
});
builder.addPartitionKey("PartitionKey", header.getKeyType());
for (int i = 0; i < header.getClusteringTypes().size(); i++) {
builder.addClusteringColumn(
"clustering" + (i > 0 ? i : ""), header.getClusteringTypes().get(i));
}
return builder.build();
}
@Test
public void testGetNewNames() throws IOException {
Descriptor desc =
Descriptor.fromFilename(
new File("Keyspace1", "Keyspace1-Standard1-ia-500-Data.db").toString());
// assert !desc.isLatestVersion; // minimum compatible version -- for now it is the latest as
// well
PendingFile inContext =
new PendingFile(
null, desc, "Data.db", Arrays.asList(Pair.create(0L, 1L)), OperationType.BOOTSTRAP);
PendingFile outContext = StreamIn.getContextMapping(inContext);
// filename and generation are expected to have changed
assert !inContext.getFilename().equals(outContext.getFilename());
// nothing else should
assertEquals(inContext.component, outContext.component);
assertEquals(desc.ksname, outContext.desc.ksname);
assertEquals(desc.cfname, outContext.desc.cfname);
assertEquals(desc.version, outContext.desc.version);
}
/**
* Given arguments specifying an SSTable, and optionally an output file, export the contents of
* the SSTable to JSON.
*
* @param args command lines arguments
* @throws ConfigurationException on configuration failure (wrong params given)
*/
public static void main(String[] args) throws ConfigurationException {
CommandLineParser parser = new PosixParser();
try {
cmd = parser.parse(options, args);
} catch (ParseException e1) {
System.err.println(e1.getMessage());
printUsage();
System.exit(1);
}
if (cmd.getArgs().length != 1) {
System.err.println("You must supply exactly one sstable");
printUsage();
System.exit(1);
}
String[] keys = cmd.getOptionValues(KEY_OPTION);
HashSet<String> excludes =
new HashSet<>(
Arrays.asList(
cmd.getOptionValues(EXCLUDE_KEY_OPTION) == null
? new String[0]
: cmd.getOptionValues(EXCLUDE_KEY_OPTION)));
String ssTableFileName = new File(cmd.getArgs()[0]).getAbsolutePath();
if (Descriptor.isLegacyFile(new File(ssTableFileName))) {
System.err.println("Unsupported legacy sstable");
System.exit(1);
}
if (!new File(ssTableFileName).exists()) {
System.err.println("Cannot find file " + ssTableFileName);
System.exit(1);
}
Descriptor desc = Descriptor.fromFilename(ssTableFileName);
try {
CFMetaData metadata = metadataFromSSTable(desc);
if (cmd.hasOption(ENUMERATE_KEYS_OPTION)) {
JsonTransformer.keysToJson(
null,
iterToStream(new KeyIterator(desc, metadata)),
cmd.hasOption(RAW_TIMESTAMPS),
metadata,
System.out);
} else {
SSTableReader sstable = SSTableReader.openNoValidation(desc, metadata);
IPartitioner partitioner = sstable.getPartitioner();
final ISSTableScanner currentScanner;
if ((keys != null) && (keys.length > 0)) {
List<AbstractBounds<PartitionPosition>> bounds =
Arrays.stream(keys)
.filter(key -> !excludes.contains(key))
.map(metadata.getKeyValidator()::fromString)
.map(partitioner::decorateKey)
.sorted()
.map(DecoratedKey::getToken)
.map(token -> new Bounds<>(token.minKeyBound(), token.maxKeyBound()))
.collect(Collectors.toList());
currentScanner = sstable.getScanner(bounds.iterator());
} else {
currentScanner = sstable.getScanner();
}
Stream<UnfilteredRowIterator> partitions =
iterToStream(currentScanner)
.filter(
i ->
excludes.isEmpty()
|| !excludes.contains(
metadata.getKeyValidator().getString(i.partitionKey().getKey())));
if (cmd.hasOption(DEBUG_OUTPUT_OPTION)) {
AtomicLong position = new AtomicLong();
partitions.forEach(
partition -> {
position.set(currentScanner.getCurrentPosition());
if (!partition.partitionLevelDeletion().isLive()) {
System.out.println(
"["
+ metadata.getKeyValidator().getString(partition.partitionKey().getKey())
+ "]@"
+ position.get()
+ " "
+ partition.partitionLevelDeletion());
}
if (!partition.staticRow().isEmpty()) {
System.out.println(
"["
+ metadata.getKeyValidator().getString(partition.partitionKey().getKey())
+ "]@"
+ position.get()
+ " "
+ partition.staticRow().toString(metadata, true));
}
partition.forEachRemaining(
row -> {
System.out.println(
"["
+ metadata
.getKeyValidator()
.getString(partition.partitionKey().getKey())
+ "]@"
+ position.get()
+ " "
+ row.toString(metadata, false, true));
position.set(currentScanner.getCurrentPosition());
});
});
} else {
JsonTransformer.toJson(
currentScanner, partitions, cmd.hasOption(RAW_TIMESTAMPS), metadata, System.out);
}
}
} catch (IOException e) {
// throwing exception outside main with broken pipe causes windows cmd to hang
e.printStackTrace(System.err);
}
System.exit(0);
}
public static void main(String args[]) throws IOException {
Options options = Options.parseArgs(args);
try {
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
String ksName = null;
String cfName = null;
Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
for (String filename : options.filenames) {
File file = new File(filename);
if (!file.exists()) {
System.out.println("Skipping inexisting file " + file);
continue;
}
Pair<Descriptor, Component> pair =
SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
if (pair == null) {
System.out.println("Skipping non sstable file " + file);
continue;
}
Descriptor desc = pair.left;
if (ksName == null) ksName = desc.ksname;
else if (!ksName.equals(desc.ksname))
throw new IllegalArgumentException("All sstables must be part of the same keyspace");
if (cfName == null) cfName = desc.cfname;
else if (!cfName.equals(desc.cfname))
throw new IllegalArgumentException("All sstables must be part of the same column family");
Set<Component> components =
new HashSet<Component>(
Arrays.asList(
new Component[] {
Component.DATA,
Component.PRIMARY_INDEX,
Component.FILTER,
Component.COMPRESSION_INFO,
Component.STATS
}));
Iterator<Component> iter = components.iterator();
while (iter.hasNext()) {
Component component = iter.next();
if (!(new File(desc.filenameFor(component)).exists())) iter.remove();
}
parsedFilenames.put(desc, components);
}
if (ksName == null || cfName == null) {
System.err.println("No valid sstables to split");
System.exit(1);
}
// Do not load sstables since they might be broken
Table table = Table.openWithoutSSTables(ksName);
ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
String snapshotName = "pre-split-" + System.currentTimeMillis();
List<SSTableReader> sstables = new ArrayList<SSTableReader>();
for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet()) {
try {
SSTableReader sstable =
SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs.metadata);
sstables.add(sstable);
if (options.snapshot) {
File snapshotDirectory =
Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
sstable.createLinks(snapshotDirectory.getPath());
}
} catch (Exception e) {
System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
if (options.snapshot)
System.out.println(
String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));
cfs.getDataTracker().markCompacting(sstables);
for (SSTableReader sstable : sstables) {
try {
new SSTableSplitter(cfs, sstable, options.sizeInMB).split();
// Remove the sstable
sstable.markCompacted();
sstable.releaseReference();
} catch (Exception e) {
System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
SSTableDeletingTask.waitForDeletions();
System.exit(0); // We need that to stop non daemonized threads
} catch (Exception e) {
System.err.println(e.getMessage());
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}