private static List<Pair<SSTableReader, Long>> createSSTableAndLengthPairs( Collection<SSTableReader> collection) { List<Pair<SSTableReader, Long>> tableLengthPairs = new ArrayList<Pair<SSTableReader, Long>>(collection.size()); for (SSTableReader table : collection) tableLengthPairs.add(Pair.create(table, table.onDiskLength())); return tableLengthPairs; }
/** * @param sstables * @return */ public static List<Pair<SSTableReader, Long>> createSSTableAndMinTimestampPairs( Iterable<SSTableReader> sstables) { List<Pair<SSTableReader, Long>> sstableMinTimestampPairs = Lists.newArrayListWithCapacity(Iterables.size(sstables)); for (SSTableReader sstable : sstables) sstableMinTimestampPairs.add(Pair.create(sstable, sstable.getMinTimestamp())); return sstableMinTimestampPairs; }
private void setToRowStart(SSTableReader reader, RowIndexEntry indexEntry, FileDataInput input) throws IOException { if (input == null) { this.file = sstable.getFileDataInput(indexEntry.position); } else { this.file = input; input.seek(indexEntry.position); } sstable.decodeKey(ByteBufferUtil.readWithShortLength(file)); SSTableReader.readRowSize(file, sstable.descriptor); }
private SSTableReader writeSortedContents() throws IOException { logger.info("Writing " + this); SSTableWriter writer = cfs.createFlushWriter(columnFamilies.size()); for (Map.Entry<DecoratedKey, ColumnFamily> entry : columnFamilies.entrySet()) writer.append(entry.getKey(), entry.getValue()); SSTableReader ssTable = writer.closeAndOpenReader(); logger.info( String.format( "Completed flushing %s (%d bytes)", ssTable.getFilename(), new File(ssTable.getFilename()).length())); return ssTable; }
@Before public void setup() throws IOException { job = Job.getInstance(); conf = job.getConfiguration(); attemptId = new TaskAttemptID(); Path inputPath = new Path(TABLE_PATH_STR); inputSplit = new FileSplit(inputPath, 0, 1, null); Descriptor desc = new Descriptor(new File(TABLE_PATH_STR), "keyspace", "columnFamily", 1, false); doReturn(desc).when(ssTableColumnRecordReader).getDescriptor(); doReturn(desc).when(ssTableRowRecordReader).getDescriptor(); doNothing() .when(ssTableColumnRecordReader) .copyTablesToLocal(any(FileSplit.class), any(TaskAttemptContext.class)); doNothing() .when(ssTableRowRecordReader) .copyTablesToLocal(any(FileSplit.class), any(TaskAttemptContext.class)); doReturn(ssTableReader) .when(ssTableColumnRecordReader) .openSSTableReader(any(IPartitioner.class), any(CFMetaData.class)); doReturn(ssTableReader) .when(ssTableRowRecordReader) .openSSTableReader(any(IPartitioner.class), any(CFMetaData.class)); when(ssTableReader.getDirectScanner(null)).thenReturn(tableScanner); }
@Test public void testSuperColumnTombstones() throws IOException, ExecutionException, InterruptedException { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1"); cfs.disableAutoCompaction(); DecoratedKey key = Util.dk("tskey"); ByteBuffer scName = ByteBufferUtil.bytes("TestSuperColumn"); // a subcolumn RowMutation rm = new RowMutation(KEYSPACE1, key.key); rm.add( "Super1", CompositeType.build(scName, ByteBufferUtil.bytes(0)), ByteBufferUtil.EMPTY_BYTE_BUFFER, FBUtilities.timestampMicros()); rm.apply(); cfs.forceBlockingFlush(); // shadow the subcolumn with a supercolumn tombstone rm = new RowMutation(KEYSPACE1, key.key); rm.deleteRange( "Super1", SuperColumns.startOf(scName), SuperColumns.endOf(scName), FBUtilities.timestampMicros()); rm.apply(); cfs.forceBlockingFlush(); CompactionManager.instance.performMaximal(cfs); assertEquals(1, cfs.getSSTables().size()); // check that the shadowed column is gone SSTableReader sstable = cfs.getSSTables().iterator().next(); Range keyRange = new Range<RowPosition>(key, sstable.partitioner.getMinimumToken().maxKeyBound()); SSTableScanner scanner = sstable.getScanner(DataRange.forKeyRange(keyRange)); OnDiskAtomIterator iter = scanner.next(); assertEquals(key, iter.getKey()); assert iter.next() instanceof RangeTombstone; assert !iter.hasNext(); }
@Test public void testNoExpire() throws ExecutionException, InterruptedException { ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1"); cfs.disableAutoCompaction(); cfs.metadata.gcGraceSeconds(0); long timestamp = System.currentTimeMillis(); RowMutation rm = new RowMutation("Keyspace1", Util.dk("ttl").key); rm.add( "Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1); rm.add( "Standard1", ByteBufferUtil.bytes("col7"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation("Keyspace1", Util.dk("ttl").key); rm.add( "Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation("Keyspace1", Util.dk("ttl").key); rm.add( "Standard1", ByteBufferUtil.bytes("col3"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1); rm.apply(); cfs.forceBlockingFlush(); DecoratedKey noTTLKey = Util.dk("nottl"); rm = new RowMutation("Keyspace1", noTTLKey.key); rm.add( "Standard1", ByteBufferUtil.bytes("col311"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp); rm.apply(); cfs.forceBlockingFlush(); Thread.sleep(2000); // wait for ttl to expire assertEquals(4, cfs.getSSTables().size()); cfs.enableAutoCompaction(true); assertEquals(1, cfs.getSSTables().size()); SSTableReader sstable = cfs.getSSTables().iterator().next(); SSTableScanner scanner = sstable.getScanner(DataRange.allData(sstable.partitioner)); assertTrue(scanner.hasNext()); while (scanner.hasNext()) { OnDiskAtomIterator iter = scanner.next(); assertEquals(noTTLKey, iter.getKey()); } }
@Test public void testImportSimpleCf() throws IOException, ParseException { // Import JSON to temp SSTable file String jsonUrl = getClass().getClassLoader().getResource("SimpleCF.json").getPath(); File tempSS = tempSSTableFile("Keyspace1", "Standard1"); SSTableImport.importJson(jsonUrl, "Keyspace1", "Standard1", tempSS.getPath()); // Verify results SSTableReader reader = SSTableReader.open(tempSS.getPath(), DatabaseDescriptor.getPartitioner()); QueryFilter qf = QueryFilter.getNamesFilter( "rowA", new QueryPath("Standard1", null, null), "colAA".getBytes()); ColumnFamily cf = qf.getSSTableColumnIterator(reader).getColumnFamily(); assert Arrays.equals(cf.getColumn("colAA".getBytes()).value(), hexToBytes("76616c4141")); }
public IndexedSliceReader( SSTableReader sstable, RowIndexEntry indexEntry, FileDataInput input, ByteBuffer startColumn, ByteBuffer finishColumn, boolean reversed) { this.sstable = sstable; this.originalInput = input; this.startColumn = startColumn; this.finishColumn = finishColumn; this.reversed = reversed; this.comparator = sstable.metadata.comparator; try { if (sstable.descriptor.hasPromotedIndexes) { this.indexes = indexEntry.columnsIndex(); if (indexes.isEmpty()) { setToRowStart(sstable, indexEntry, input); this.emptyColumnFamily = ColumnFamily.serializer() .deserializeFromSSTableNoColumns(ColumnFamily.create(sstable.metadata), file); fetcher = new SimpleBlockFetcher(); } else { this.emptyColumnFamily = ColumnFamily.create(sstable.metadata); emptyColumnFamily.delete(indexEntry.deletionInfo()); fetcher = new IndexedBlockFetcher(indexEntry); } } else { setToRowStart(sstable, indexEntry, input); IndexHelper.skipBloomFilter(file); this.indexes = IndexHelper.deserializeIndex(file); this.emptyColumnFamily = ColumnFamily.serializer() .deserializeFromSSTableNoColumns(ColumnFamily.create(sstable.metadata), file); fetcher = indexes.isEmpty() ? new SimpleBlockFetcher() : new IndexedBlockFetcher(); } } catch (IOException e) { sstable.markSuspect(); throw new IOError(e); } }
protected List<Long> getMaxTimestampList(ColumnFamilyStore cfs) { List<Long> list = new LinkedList<Long>(); for (SSTableReader sstable : cfs.getSSTables()) list.add(sstable.getMaxTimestamp()); return list; }
/** * Retrieves a local subBlock * * @param blockId row key * @param sblockId SubBlock column name * @param offset inside the sblock * @return a local sublock * @throws TException */ private LocalBlock getLocalSubBlock( String subBlockCFName, ByteBuffer blockId, ByteBuffer sblockId, int offset) throws TException { DecoratedKey<Token<?>> decoratedKey = new DecoratedKey<Token<?>>(StorageService.getPartitioner().getToken(blockId), blockId); Table table = Table.open(cfsKeyspace); ColumnFamilyStore sblockStore = table.getColumnFamilyStore(subBlockCFName); Collection<SSTableReader> sstables = sblockStore.getSSTables(); for (SSTableReader sstable : sstables) { long position = sstable.getPosition(decoratedKey, Operator.EQ); if (position == -1) continue; String filename = sstable.descriptor.filenameFor(Component.DATA); RandomAccessFile raf = null; int mappedLength = -1; MappedByteBuffer mappedData = null; MappedFileDataInput file = null; try { raf = new RandomAccessFile(filename, "r"); assert position < raf.length(); mappedLength = (raf.length() - position) < Integer.MAX_VALUE ? (int) (raf.length() - position) : Integer.MAX_VALUE; mappedData = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, position, mappedLength); file = new MappedFileDataInput(mappedData, filename, 0); if (file == null) continue; // Verify key was found in data file DecoratedKey keyInDisk = SSTableReader.decodeKey( sstable.partitioner, sstable.descriptor, ByteBufferUtil.readWithShortLength(file)); assert keyInDisk.equals(decoratedKey) : String.format("%s != %s in %s", keyInDisk, decoratedKey, file.getPath()); long rowSize = SSTableReader.readRowSize(file, sstable.descriptor); assert rowSize > 0; assert rowSize < mappedLength; Filter bf = IndexHelper.defreezeBloomFilter(file, sstable.descriptor.usesOldBloomFilter); // verify this column in in this version of the row. if (!bf.isPresent(sblockId)) continue; List<IndexHelper.IndexInfo> indexList = IndexHelper.deserializeIndex(file); // we can stop early if bloom filter says none of the // columns actually exist -- but, // we can't stop before initializing the cf above, in // case there's a relevant tombstone ColumnFamilySerializer serializer = ColumnFamily.serializer(); try { ColumnFamily cf = serializer.deserializeFromSSTableNoColumns( ColumnFamily.create(sstable.metadata), file); if (cf.isMarkedForDelete()) continue; } catch (Exception e) { e.printStackTrace(); throw new IOException( serializer + " failed to deserialize " + sstable.getColumnFamilyName() + " with " + sstable.metadata + " from " + file, e); } Integer sblockLength = null; if (indexList == null) sblockLength = seekToSubColumn(sstable.metadata, file, sblockId); else sblockLength = seekToSubColumn(sstable.metadata, file, sblockId, indexList); if (sblockLength == null || sblockLength < 0) continue; int bytesReadFromStart = mappedLength - (int) file.bytesRemaining(); if (logger.isDebugEnabled()) logger.debug("BlockLength = " + sblockLength + " Availible " + file.bytesRemaining()); assert offset <= sblockLength : String.format("%d > %d", offset, sblockLength); long dataOffset = position + bytesReadFromStart; if (file.bytesRemaining() == 0 || sblockLength == 0) continue; return new LocalBlock(file.getPath(), dataOffset + offset, sblockLength - offset); } catch (IOException e) { throw new TException(e); } finally { FileUtils.closeQuietly(raf); } } return null; }
public static void main(String args[]) throws IOException { Options options = Options.parseArgs(args); try { // load keyspace descriptions. DatabaseDescriptor.loadSchemas(); String ksName = null; String cfName = null; Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>(); for (String filename : options.filenames) { File file = new File(filename); if (!file.exists()) { System.out.println("Skipping inexisting file " + file); continue; } Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(), file.getName()); if (pair == null) { System.out.println("Skipping non sstable file " + file); continue; } Descriptor desc = pair.left; if (ksName == null) ksName = desc.ksname; else if (!ksName.equals(desc.ksname)) throw new IllegalArgumentException("All sstables must be part of the same keyspace"); if (cfName == null) cfName = desc.cfname; else if (!cfName.equals(desc.cfname)) throw new IllegalArgumentException("All sstables must be part of the same column family"); Set<Component> components = new HashSet<Component>( Arrays.asList( new Component[] { Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.COMPRESSION_INFO, Component.STATS })); Iterator<Component> iter = components.iterator(); while (iter.hasNext()) { Component component = iter.next(); if (!(new File(desc.filenameFor(component)).exists())) iter.remove(); } parsedFilenames.put(desc, components); } if (ksName == null || cfName == null) { System.err.println("No valid sstables to split"); System.exit(1); } // Do not load sstables since they might be broken Table table = Table.openWithoutSSTables(ksName); ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName); String snapshotName = "pre-split-" + System.currentTimeMillis(); List<SSTableReader> sstables = new ArrayList<SSTableReader>(); for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet()) { try { SSTableReader sstable = SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs.metadata); sstables.add(sstable); if (options.snapshot) { File snapshotDirectory = Directories.getSnapshotDirectory(sstable.descriptor, snapshotName); sstable.createLinks(snapshotDirectory.getPath()); } } catch (Exception e) { System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage())); if (options.debug) e.printStackTrace(System.err); } } if (options.snapshot) System.out.println( String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName)); cfs.getDataTracker().markCompacting(sstables); for (SSTableReader sstable : sstables) { try { new SSTableSplitter(cfs, sstable, options.sizeInMB).split(); // Remove the sstable sstable.markCompacted(); sstable.releaseReference(); } catch (Exception e) { System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage())); if (options.debug) e.printStackTrace(System.err); } } SSTableDeletingTask.waitForDeletions(); System.exit(0); // We need that to stop non daemonized threads } catch (Exception e) { System.err.println(e.getMessage()); if (options.debug) e.printStackTrace(System.err); System.exit(1); } }
@Test public void testRangeTombstones() throws IOException, ExecutionException, InterruptedException { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2"); cfs.clearUnsafe(); // disable compaction while flushing cfs.disableAutoCompaction(); final CFMetaData cfmeta = cfs.metadata; Directories dir = cfs.directories; ArrayList<DecoratedKey> keys = new ArrayList<DecoratedKey>(); for (int i = 0; i < 4; i++) { keys.add(Util.dk("" + i)); } ArrayBackedSortedColumns cf = ArrayBackedSortedColumns.factory.create(cfmeta); cf.addColumn(Util.column("01", "a", 1)); // this must not resurrect cf.addColumn(Util.column("a", "a", 3)); cf.deletionInfo() .add( new RangeTombstone( Util.cellname("0"), Util.cellname("b"), 2, (int) (System.currentTimeMillis() / 1000)), cfmeta.comparator); SSTableWriter writer = new SSTableWriter( cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables()), 0, 0, cfs.metadata, StorageService.getPartitioner(), new MetadataCollector(cfs.metadata.comparator)); writer.append(Util.dk("0"), cf); writer.append(Util.dk("1"), cf); writer.append(Util.dk("3"), cf); cfs.addSSTable(writer.closeAndOpenReader()); writer = new SSTableWriter( cfs.getTempSSTablePath(dir.getDirectoryForNewSSTables()), 0, 0, cfs.metadata, StorageService.getPartitioner(), new MetadataCollector(cfs.metadata.comparator)); writer.append(Util.dk("0"), cf); writer.append(Util.dk("1"), cf); writer.append(Util.dk("2"), cf); writer.append(Util.dk("3"), cf); cfs.addSSTable(writer.closeAndOpenReader()); Collection<SSTableReader> toCompact = cfs.getSSTables(); assert toCompact.size() == 2; // Force compaction on first sstables. Since each row is in only one sstable, we will be using // EchoedRow. Util.compact(cfs, toCompact); assertEquals(1, cfs.getSSTables().size()); // Now assert we do have the 4 keys assertEquals(4, Util.getRangeSlice(cfs).size()); ArrayList<DecoratedKey> k = new ArrayList<DecoratedKey>(); for (Row r : Util.getRangeSlice(cfs)) { k.add(r.key); assertEquals(ByteBufferUtil.bytes("a"), r.cf.getColumn(Util.cellname("a")).value()); assertNull(r.cf.getColumn(Util.cellname("01"))); assertEquals(3, r.cf.getColumn(Util.cellname("a")).timestamp()); } for (SSTableReader sstable : cfs.getSSTables()) { StatsMetadata stats = sstable.getSSTableMetadata(); assertEquals(ByteBufferUtil.bytes("0"), stats.minColumnNames.get(0)); assertEquals(ByteBufferUtil.bytes("b"), stats.maxColumnNames.get(0)); } assertEquals(keys, k); }
public static void assertMaxTimestamp(ColumnFamilyStore cfs, long maxTimestampExpected) { long maxTimestampObserved = Long.MIN_VALUE; for (SSTableReader sstable : cfs.getSSTables()) maxTimestampObserved = Math.max(sstable.getMaxTimestamp(), maxTimestampObserved); assertEquals(maxTimestampExpected, maxTimestampObserved); }
@Test public void testParallelLeveledCompaction() throws Exception { String ksname = "Keyspace1"; String cfname = "StandardLeveled"; Keyspace keyspace = Keyspace.open(ksname); ColumnFamilyStore store = keyspace.getColumnFamilyStore(cfname); store.disableAutoCompaction(); LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) store.getCompactionStrategy(); ByteBuffer value = ByteBuffer.wrap(new byte[100 * 1024]); // 100 KB value, make it easy to have multiple files // Enough data to have a level 1 and 2 int rows = 128; int columns = 10; // Adds enough data to trigger multiple sstable per level for (int r = 0; r < rows; r++) { DecoratedKey key = Util.dk(String.valueOf(r)); RowMutation rm = new RowMutation(ksname, key.key); for (int c = 0; c < columns; c++) { rm.add(cfname, ByteBufferUtil.bytes("column" + c), value, 0); } rm.apply(); store.forceBlockingFlush(); } // Execute LCS in parallel ExecutorService executor = new ThreadPoolExecutor( 4, 4, Long.MAX_VALUE, TimeUnit.SECONDS, new LinkedBlockingDeque<Runnable>()); List<Runnable> tasks = new ArrayList<Runnable>(); while (true) { while (true) { final AbstractCompactionTask t = lcs.getMaximalTask(Integer.MIN_VALUE); if (t == null) break; tasks.add( new Runnable() { public void run() { t.execute(null); } }); } if (tasks.isEmpty()) break; List<Future<?>> futures = new ArrayList<Future<?>>(tasks.size()); for (Runnable r : tasks) futures.add(executor.submit(r)); FBUtilities.waitOnFutures(futures); tasks.clear(); } // Assert all SSTables are lined up correctly. LeveledManifest manifest = lcs.manifest; int levels = manifest.getLevelCount(); for (int level = 0; level < levels; level++) { List<SSTableReader> sstables = manifest.getLevel(level); // score check assert (double) SSTable.getTotalBytes(sstables) / manifest.maxBytesForLevel(level) < 1.00; // overlap check for levels greater than 0 if (level > 0) { for (SSTableReader sstable : sstables) { Set<SSTableReader> overlaps = LeveledManifest.overlapping(sstable, sstables); assert overlaps.size() == 1 && overlaps.contains(sstable); } } } for (SSTableReader sstable : store.getSSTables()) { assert sstable.getSSTableLevel() == sstable.getSSTableLevel(); } }
public static void main(String args[]) throws IOException { Options options = Options.parseArgs(args); try { // load keyspace descriptions. DatabaseDescriptor.loadSchemas(false); if (Schema.instance.getCFMetaData(options.keyspace, options.cf) == null) throw new IllegalArgumentException( String.format("Unknown keyspace/columnFamily %s.%s", options.keyspace, options.cf)); Keyspace keyspace = Keyspace.openWithoutSSTables(options.keyspace); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(options.cf); OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug); Directories.SSTableLister lister = cfs.directories.sstableLister(); if (options.snapshot != null) lister.onlyBackups(true).snapshots(options.snapshot); else lister.includeBackups(false); Collection<SSTableReader> readers = new ArrayList<SSTableReader>(); // Upgrade sstables for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet()) { Set<Component> components = entry.getValue(); if (!components.contains(Component.DATA) || !components.contains(Component.PRIMARY_INDEX)) continue; try { SSTableReader sstable = SSTableReader.openNoValidation(entry.getKey(), components, cfs.metadata); if (sstable.descriptor.version.equals(Descriptor.Version.CURRENT)) continue; readers.add(sstable); } catch (Exception e) { JVMStabilityInspector.inspectThrowable(e); System.err.println(String.format("Error Loading %s: %s", entry.getKey(), e.getMessage())); if (options.debug) e.printStackTrace(System.err); continue; } } int numSSTables = readers.size(); handler.output("Found " + numSSTables + " sstables that need upgrading."); for (SSTableReader sstable : readers) { try { Upgrader upgrader = new Upgrader(cfs, sstable, handler); upgrader.upgrade(); if (!options.keepSource) { // Remove the sstable (it's been copied by upgrade) System.out.format("Deleting table %s.%n", sstable.descriptor.baseFilename()); sstable.markObsolete(); sstable.selfRef().release(); } } catch (Exception e) { System.err.println(String.format("Error upgrading %s: %s", sstable, e.getMessage())); if (options.debug) e.printStackTrace(System.err); } } CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES); SSTableDeletingTask.waitForDeletions(); System.exit(0); } catch (Exception e) { System.err.println(e.getMessage()); if (options.debug) e.printStackTrace(System.err); System.exit(1); } }