@Test public void testGetSliceFromLarge() throws Throwable { // tests slicing against 1000 columns in an sstable Table table = Table.open("Keyspace1"); ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1"); String key = "row3"; RowMutation rm = new RowMutation("Keyspace1", key); ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1"); for (int i = 1000; i < 2000; i++) cf.addColumn(column("col" + i, ("v" + i), 1L)); rm.add(cf); rm.apply(); cfStore.forceBlockingFlush(); validateSliceLarge(cfStore); // compact so we have a big row with more than the minimum index count if (cfStore.getSSTables().size() > 1) { CompactionManager.instance.submitMajor(cfStore).get(); } SSTableReader sstable = cfStore.getSSTables().iterator().next(); DecoratedKey decKey = sstable.getPartitioner().decorateKey(key); SSTable.PositionSize info = sstable.getPosition(decKey); BufferedRandomAccessFile file = new BufferedRandomAccessFile(sstable.getFilename(), "r"); file.seek(info.position); assert file.readUTF().equals(key); file.readInt(); IndexHelper.skipBloomFilter(file); ArrayList<IndexHelper.IndexInfo> indexes = IndexHelper.deserializeIndex(file); assert indexes.size() > 2; validateSliceLarge(cfStore); }
@Test public void testRemoveColumn() throws IOException, ExecutionException, InterruptedException { Table table = Table.open("Keyspace1"); ColumnFamilyStore store = table.getColumnFamilyStore("Standard1"); RowMutation rm; DecoratedKey dk = Util.dk("key1"); // add data rm = new RowMutation("Keyspace1", dk.key); rm.add( new QueryPath("Standard1", null, "Column1".getBytes()), "asdf".getBytes(), new TimestampClock(0)); rm.apply(); store.forceBlockingFlush(); // remove rm = new RowMutation("Keyspace1", dk.key); rm.delete(new QueryPath("Standard1", null, "Column1".getBytes()), new TimestampClock(1)); rm.apply(); ColumnFamily retrieved = store.getColumnFamily( QueryFilter.getNamesFilter(dk, new QueryPath("Standard1"), "Column1".getBytes())); assert retrieved.getColumn("Column1".getBytes()).isMarkedForDelete(); assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE)); assertNull( Util.cloneAndRemoveDeleted( store.getColumnFamily(QueryFilter.getIdentityFilter(dk, new QueryPath("Standard1"))), Integer.MAX_VALUE)); }
// Test compaction of hints column family. It shouldn't remove all columns on compaction. @Test public void testCompactionOfHintsCF() throws Exception { // prepare hints column family Table systemTable = Table.open("system"); ColumnFamilyStore hintStore = systemTable.getColumnFamilyStore(SystemTable.HINTS_CF); hintStore.clearUnsafe(); hintStore.metadata.gcGraceSeconds(36000); // 10 hours hintStore.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getCanonicalName()); hintStore.disableAutoCompaction(); // insert 1 hint RowMutation rm = new RowMutation(TABLE4, ByteBufferUtil.bytes(1)); rm.add( new QueryPath(STANDARD1_CF, null, ByteBufferUtil.bytes(String.valueOf(COLUMN1))), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); RowMutation.hintFor(rm, UUID.randomUUID()).apply(); // flush data to disk hintStore.forceBlockingFlush(); assertEquals(1, hintStore.getSSTables().size()); // submit compaction FBUtilities.waitOnFuture(HintedHandOffManager.instance.compact()); while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1); // single row should not be removed because of gc_grace_seconds // is 10 hours and there are no any tombstones in sstable assertEquals(1, hintStore.getSSTables().size()); }
private void testRowMutationWrite() throws IOException { RowMutation standardRowRm = new RowMutation(statics.KS, statics.StandardRow); RowMutation superRowRm = new RowMutation(statics.KS, statics.SuperRow); RowMutation standardRm = new RowMutation(statics.KS, statics.Key, statics.StandardCf); RowMutation superRm = new RowMutation(statics.KS, statics.Key, statics.SuperCf); Map<UUID, ColumnFamily> mods = new HashMap<UUID, ColumnFamily>(); mods.put(statics.StandardCf.metadata().cfId, statics.StandardCf); mods.put(statics.SuperCf.metadata().cfId, statics.SuperCf); RowMutation mixedRm = new RowMutation(statics.KS, statics.Key, mods); DataOutputStream out = getOutput("db.RowMutation.bin"); RowMutation.serializer.serialize(standardRowRm, out, getVersion()); RowMutation.serializer.serialize(superRowRm, out, getVersion()); RowMutation.serializer.serialize(standardRm, out, getVersion()); RowMutation.serializer.serialize(superRm, out, getVersion()); RowMutation.serializer.serialize(mixedRm, out, getVersion()); standardRowRm.createMessage().serialize(out, getVersion()); superRowRm.createMessage().serialize(out, getVersion()); standardRm.createMessage().serialize(out, getVersion()); superRm.createMessage().serialize(out, getVersion()); mixedRm.createMessage().serialize(out, getVersion()); out.close(); // test serializedSize testSerializedSize(standardRowRm, RowMutation.serializer); testSerializedSize(superRowRm, RowMutation.serializer); testSerializedSize(standardRm, RowMutation.serializer); testSerializedSize(superRm, RowMutation.serializer); testSerializedSize(mixedRm, RowMutation.serializer); }
@Test public void testGetSliceFromSuperBasic() throws Throwable { // tests slicing against data from one row spread across two sstables final Table table = Table.open("Keyspace1"); final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Super1"); final String ROW = "row2"; RowMutation rm = new RowMutation("Keyspace1", ROW); ColumnFamily cf = ColumnFamily.create("Keyspace1", "Super1"); SuperColumn sc = new SuperColumn("sc1".getBytes(), new LongType()); sc.addColumn(new Column(getBytes(1), "val1".getBytes(), 1L)); cf.addColumn(sc); rm.add(cf); rm.apply(); Runnable verify = new WrappedRunnable() { public void runMayThrow() throws Exception { ColumnFamily cf = cfStore.getColumnFamily( ROW, new QueryPath("Super1"), ArrayUtils.EMPTY_BYTE_ARRAY, ArrayUtils.EMPTY_BYTE_ARRAY, false, 10); assertColumns(cf, "sc1"); assertEquals( new String(cf.getColumn("sc1".getBytes()).getSubColumn(getBytes(1)).value()), "val1"); } }; reTest(table.getColumnFamilyStore("Standard1"), verify); }
@Test public void testCompactions() throws IOException, ExecutionException, InterruptedException { CompactionManager.instance.disableAutoCompaction(); // this test does enough rows to force multiple block indexes to be used Table table = Table.open(TABLE1); ColumnFamilyStore store = table.getColumnFamilyStore("Standard1"); final int ROWS_PER_SSTABLE = 10; Set<DecoratedKey> inserted = new HashSet<DecoratedKey>(); for (int j = 0; j < (DatabaseDescriptor.getIndexInterval() * 3) / ROWS_PER_SSTABLE; j++) { for (int i = 0; i < ROWS_PER_SSTABLE; i++) { DecoratedKey key = Util.dk(String.valueOf(i % 2)); RowMutation rm = new RowMutation(TABLE1, key.key); rm.add( new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i / 2))), ByteBufferUtil.EMPTY_BYTE_BUFFER, j * ROWS_PER_SSTABLE + i); rm.apply(); inserted.add(key); } store.forceBlockingFlush(); assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(store).size()); } while (true) { Future<Integer> ft = CompactionManager.instance.submitMinorIfNeeded(store); if (ft.get() == 0) break; } if (store.getSSTables().size() > 1) { CompactionManager.instance.performMajor(store); } assertEquals(inserted.size(), Util.getRangeSlice(store).size()); }
@Test public void testOne() throws IOException, ExecutionException, InterruptedException { Table table1 = Table.open("Keyspace1"); Table table2 = Table.open("Keyspace2"); RowMutation rm; DecoratedKey dk = Util.dk("keymulti"); ColumnFamily cf; rm = new RowMutation("Keyspace1", dk.key); cf = ColumnFamily.create("Keyspace1", "Standard1"); cf.addColumn(column("col1", "val1", 1L)); rm.add(cf); rm.apply(); rm = new RowMutation("Keyspace2", dk.key); cf = ColumnFamily.create("Keyspace2", "Standard3"); cf.addColumn(column("col2", "val2", 1L)); rm.add(cf); rm.apply(); table1.getColumnFamilyStore("Standard1").clearUnsafe(); table2.getColumnFamilyStore("Standard3").clearUnsafe(); CommitLog.instance.resetUnsafe(); // disassociate segments from live CL CommitLog.instance.recover(); assertColumns(Util.getColumnFamily(table1, dk, "Standard1"), "col1"); assertColumns(Util.getColumnFamily(table2, dk, "Standard3"), "col2"); }
@Test public void testGetRowSliceByRange() throws Throwable { String key = TEST_KEY + "slicerow"; Table table = Table.open("Keyspace1"); ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1"); RowMutation rm = new RowMutation("Keyspace1", key); ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1"); // First write "a", "b", "c" cf.addColumn(column("a", "val1", 1L)); cf.addColumn(column("b", "val2", 1L)); cf.addColumn(column("c", "val3", 1L)); rm.add(cf); rm.apply(); cf = cfStore.getColumnFamily( key, new QueryPath("Standard1"), "b".getBytes(), "c".getBytes(), false, 100); assertEquals(2, cf.getColumnCount()); cf = cfStore.getColumnFamily( key, new QueryPath("Standard1"), "b".getBytes(), "b".getBytes(), false, 100); assertEquals(1, cf.getColumnCount()); cf = cfStore.getColumnFamily( key, new QueryPath("Standard1"), "b".getBytes(), "c".getBytes(), false, 1); assertEquals(1, cf.getColumnCount()); cf = cfStore.getColumnFamily( key, new QueryPath("Standard1"), "c".getBytes(), "b".getBytes(), false, 1); assertNull(cf); }
@Test public void testRecoverCounter() throws IOException, ExecutionException, InterruptedException { Table table1 = Table.open("Keyspace1"); RowMutation rm; DecoratedKey dk = Util.dk("key"); ColumnFamily cf; for (int i = 0; i < 10; ++i) { rm = new RowMutation("Keyspace1", dk.key); cf = ColumnFamily.create("Keyspace1", "Counter1"); cf.addColumn(new CounterColumn(ByteBufferUtil.bytes("col"), 1L, 1L)); rm.add(cf); rm.apply(); } table1.getColumnFamilyStore("Counter1").clearUnsafe(); CommitLog.instance.resetUnsafe(); // disassociate segments from live CL CommitLog.instance.recover(); cf = Util.getColumnFamily(table1, dk, "Counter1"); assert cf.getColumnCount() == 1; Column c = cf.getColumn(ByteBufferUtil.bytes("col")); assert c != null; assert ((CounterColumn) c).total() == 10L; }
@Test public void testRemoveColumnFamily() throws IOException, ExecutionException, InterruptedException { Table table = Table.open("Keyspace1"); ColumnFamilyStore store = table.getColumnFamilyStore("Standard1"); RowMutation rm; // add data rm = new RowMutation("Keyspace1", "key1"); rm.add(new QueryPath("Standard1", null, "Column1".getBytes()), "asdf".getBytes(), 0); rm.apply(); // remove rm = new RowMutation("Keyspace1", "key1"); rm.delete(new QueryPath("Standard1"), 1); rm.apply(); ColumnFamily retrieved = store.getColumnFamily( new IdentityQueryFilter( "key1", new QueryPath("Standard1", null, "Column1".getBytes()))); assert retrieved.isMarkedForDelete(); assertNull(retrieved.getColumn("Column1".getBytes())); assertNull(ColumnFamilyStore.removeDeleted(retrieved, Integer.MAX_VALUE)); }
@Test(timeout = 5000) public void testTruncateHints() throws Exception { Keyspace systemKeyspace = Keyspace.open("system"); ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS_CF); hintStore.clearUnsafe(); // insert 1 hint RowMutation rm = new RowMutation(KEYSPACE4, ByteBufferUtil.bytes(1)); rm.add( STANDARD1_CF, ByteBufferUtil.bytes(String.valueOf(COLUMN1)), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); HintedHandOffManager.instance .hintFor( rm, System.currentTimeMillis(), HintedHandOffManager.calculateHintTTL(rm), UUID.randomUUID()) .apply(); assert getNoOfHints() == 1; HintedHandOffManager.instance.truncateAllHints(); while (getNoOfHints() > 0) { Thread.sleep(100); } assert getNoOfHints() == 0; }
@Test public void testGetRowSingleColumn() throws Throwable { final Table table = Table.open("Keyspace1"); final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1"); RowMutation rm = new RowMutation("Keyspace1", TEST_KEY); ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1"); cf.addColumn(column("col1", "val1", 1L)); cf.addColumn(column("col2", "val2", 1L)); cf.addColumn(column("col3", "val3", 1L)); rm.add(cf); rm.apply(); Runnable verify = new WrappedRunnable() { public void runMayThrow() throws Exception { ColumnFamily cf; cf = cfStore.getColumnFamily( new NamesQueryFilter(TEST_KEY, new QueryPath("Standard1"), "col1".getBytes())); assertColumns(cf, "col1"); cf = cfStore.getColumnFamily( new NamesQueryFilter(TEST_KEY, new QueryPath("Standard1"), "col3".getBytes())); assertColumns(cf, "col3"); } }; reTest(table.getColumnFamilyStore("Standard1"), verify); }
public RowMutation dropFromSchema(long timestamp) { RowMutation rm = new RowMutation(Table.SYSTEM_TABLE, SystemTable.getSchemaKSKey(name)); rm.delete(new QueryPath(SystemTable.SCHEMA_KEYSPACES_CF), timestamp); rm.delete(new QueryPath(SystemTable.SCHEMA_COLUMNFAMILIES_CF), timestamp); rm.delete(new QueryPath(SystemTable.SCHEMA_COLUMNS_CF), timestamp); return rm; }
public void batch_insert(String table, BatchMutation batch_mutation, int consistency_level) throws InvalidRequestException, UnavailableException { if (logger.isDebugEnabled()) logger.debug("batch_insert"); RowMutation rm = RowMutation.getRowMutation(table, batch_mutation); Set<String> cfNames = rm.columnFamilyNames(); ThriftValidation.validateKeyCommand( rm.key(), rm.table(), cfNames.toArray(new String[cfNames.size()])); doInsert(consistency_level, rm); }
private static void insertRowWithKey(int key) { long timestamp = System.currentTimeMillis(); DecoratedKey decoratedKey = Util.dk(String.format("%03d", key)); RowMutation rm = new RowMutation(KEYSPACE1, decoratedKey.key); rm.add( "Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1000); rm.apply(); }
public RowMutation toSchema(long timestamp) { RowMutation rm = new RowMutation(Table.SYSTEM_TABLE, SystemTable.getSchemaKSKey(name)); ColumnFamily cf = rm.addOrGet(SystemTable.SCHEMA_KEYSPACES_CF); cf.addColumn(Column.create(name, timestamp, "name")); cf.addColumn(Column.create(durableWrites, timestamp, "durable_writes")); cf.addColumn(Column.create(strategyClass.getName(), timestamp, "strategy_class")); cf.addColumn(Column.create(json(strategyOptions), timestamp, "strategy_options")); for (CFMetaData cfm : cfMetaData.values()) cfm.toSchema(rm, timestamp); return rm; }
private void testDontPurgeAccidentaly(String k, String cfname) throws IOException, ExecutionException, InterruptedException { // This test catches the regression of CASSANDRA-2786 Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname); // disable compaction while flushing cfs.clearUnsafe(); cfs.disableAutoCompaction(); // Add test row DecoratedKey key = Util.dk(k); RowMutation rm = new RowMutation(KEYSPACE1, key.key); rm.add( cfname, CompositeType.build(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); rm.apply(); cfs.forceBlockingFlush(); Collection<SSTableReader> sstablesBefore = cfs.getSSTables(); QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis()); assert !(cfs.getColumnFamily(filter).getColumnCount() == 0); // Remove key rm = new RowMutation(KEYSPACE1, key.key); rm.delete(cfname, 2); rm.apply(); ColumnFamily cf = cfs.getColumnFamily(filter); assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf; // Sleep one second so that the removal is indeed purgeable even with gcgrace == 0 Thread.sleep(1000); cfs.forceBlockingFlush(); Collection<SSTableReader> sstablesAfter = cfs.getSSTables(); Collection<SSTableReader> toCompact = new ArrayList<SSTableReader>(); for (SSTableReader sstable : sstablesAfter) if (!sstablesBefore.contains(sstable)) toCompact.add(sstable); Util.compact(cfs, toCompact); cf = cfs.getColumnFamily(filter); assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf; }
public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1"); store.clearUnsafe(); store.metadata.gcGraceSeconds(1); store.setCompactionStrategyClass(strategyClassName); // disable compaction while flushing store.disableAutoCompaction(); long timestamp = System.currentTimeMillis(); for (int i = 0; i < 10; i++) { DecoratedKey key = Util.dk(Integer.toString(i)); RowMutation rm = new RowMutation(KEYSPACE1, key.key); for (int j = 0; j < 10; j++) rm.add( "Standard1", ByteBufferUtil.bytes(Integer.toString(j)), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, j > 0 ? 3 : 0); // let first column never expire, since deleting all columns does not produce // sstable rm.apply(); } store.forceBlockingFlush(); assertEquals(1, store.getSSTables().size()); long originalSize = store.getSSTables().iterator().next().uncompressedLength(); // wait enough to force single compaction TimeUnit.SECONDS.sleep(5); // enable compaction, submit background and wait for it to complete store.enableAutoCompaction(); FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store)); while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1); // and sstable with ttl should be compacted assertEquals(1, store.getSSTables().size()); long size = store.getSSTables().iterator().next().uncompressedLength(); assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize); // make sure max timestamp of compacted sstables is recorded properly after compaction. assertMaxTimestamp(store, timestamp); return store; }
@Test public void testGetSliceFromAdvanced() throws Throwable { // tests slicing against data from one row spread across two sstables final Table table = Table.open("Keyspace1"); final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1"); final String ROW = "row2"; RowMutation rm = new RowMutation("Keyspace1", ROW); ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1"); cf.addColumn(column("col1", "val1", 1L)); cf.addColumn(column("col2", "val2", 1L)); cf.addColumn(column("col3", "val3", 1L)); cf.addColumn(column("col4", "val4", 1L)); cf.addColumn(column("col5", "val5", 1L)); cf.addColumn(column("col6", "val6", 1L)); rm.add(cf); rm.apply(); cfStore.forceBlockingFlush(); rm = new RowMutation("Keyspace1", ROW); cf = ColumnFamily.create("Keyspace1", "Standard1"); cf.addColumn(column("col1", "valx", 2L)); cf.addColumn(column("col2", "valx", 2L)); cf.addColumn(column("col3", "valx", 2L)); rm.add(cf); rm.apply(); Runnable verify = new WrappedRunnable() { public void runMayThrow() throws Exception { ColumnFamily cf; cf = cfStore.getColumnFamily( ROW, new QueryPath("Standard1"), "col2".getBytes(), ArrayUtils.EMPTY_BYTE_ARRAY, false, 3); assertColumns(cf, "col2", "col3", "col4"); assertEquals(new String(cf.getColumn("col2".getBytes()).value()), "valx"); assertEquals(new String(cf.getColumn("col3".getBytes()).value()), "valx"); assertEquals(new String(cf.getColumn("col4".getBytes()).value()), "val4"); } }; reTest(table.getColumnFamilyStore("Standard1"), verify); }
public void remove( String table, String key, ColumnPathOrParent column_path_or_parent, long timestamp, int consistency_level) throws InvalidRequestException, UnavailableException { if (logger.isDebugEnabled()) logger.debug("remove"); ThriftValidation.validateColumnPathOrParent(table, column_path_or_parent); RowMutation rm = new RowMutation(table, key.trim()); rm.delete(new QueryPath(column_path_or_parent), timestamp); doInsert(consistency_level, rm); }
protected void fillCF(ColumnFamilyStore cfs, int rowsPerSSTable) throws ExecutionException, InterruptedException, IOException { CompactionManager.instance.disableAutoCompaction(); for (int i = 0; i < rowsPerSSTable; i++) { String key = String.valueOf(i); // create a row and update the birthdate value, test that the index query fetches the new // version RowMutation rm; rm = new RowMutation(KEYSPACE1, ByteBufferUtil.bytes(key)); rm.add(cfs.name, COLUMN, VALUE, System.currentTimeMillis()); rm.applyUnsafe(); } cfs.forceBlockingFlush(); }
@Test public void testGetColumn() throws IOException, ColumnFamilyNotDefinedException { Table table = Table.open("Keyspace1"); RowMutation rm; // add data rm = new RowMutation("Keyspace1", "key1"); rm.add(new QueryPath("Standard1", null, "Column1".getBytes()), "abcd".getBytes(), 0); rm.apply(); ReadCommand command = new SliceByNamesReadCommand( "Keyspace1", "key1", new QueryPath("Standard1"), Arrays.asList("Column1".getBytes())); Row row = command.getRow(table); IColumn col = row.cf.getColumn("Column1".getBytes()); assert Arrays.equals(col.value(), "abcd".getBytes()); }
@Test public void overlappingRangeTest() throws Exception { CompactionManager.instance.disableAutoCompaction(); Table table = Table.open(KSNAME); ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME); // Inserting data String key = "k2"; RowMutation rm; ColumnFamily cf; rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); for (int i = 0; i < 20; i++) add(rm, i, 0); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); cf = rm.addOrGet(CFNAME); delete(cf, 5, 15, 1); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); cf = rm.addOrGet(CFNAME); delete(cf, 5, 10, 1); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); cf = rm.addOrGet(CFNAME); delete(cf, 5, 8, 2); rm.apply(); cfs.forceBlockingFlush(); QueryPath path = new QueryPath(CFNAME); cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(key), path)); for (int i = 0; i < 5; i++) assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live"; for (int i = 16; i < 20; i++) assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live"; for (int i = 5; i <= 15; i++) assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live"; // Compact everything and re-test CompactionManager.instance.performMaximal(cfs); cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(key), path)); for (int i = 0; i < 5; i++) assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live"; for (int i = 16; i < 20; i++) assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live"; for (int i = 5; i <= 15; i++) assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live"; }
@Test public void testGetSliceNoMatch() throws Throwable { Table table = Table.open("Keyspace1"); RowMutation rm = new RowMutation("Keyspace1", "row1000"); ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard2"); cf.addColumn(column("col1", "val1", 1)); rm.add(cf); rm.apply(); validateGetSliceNoMatch(table); table.getColumnFamilyStore("Standard2").forceBlockingFlush(); validateGetSliceNoMatch(table); Collection<SSTableReader> ssTables = table.getColumnFamilyStore("Standard2").getSSTables(); assertEquals(1, ssTables.size()); ssTables.iterator().next().forceBloomFilterFailures(); validateGetSliceNoMatch(table); }
@Test public void testSuperColumnTombstones() throws IOException, ExecutionException, InterruptedException { Keyspace keyspace = Keyspace.open(KEYSPACE1); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1"); cfs.disableAutoCompaction(); DecoratedKey key = Util.dk("tskey"); ByteBuffer scName = ByteBufferUtil.bytes("TestSuperColumn"); // a subcolumn RowMutation rm = new RowMutation(KEYSPACE1, key.key); rm.add( "Super1", CompositeType.build(scName, ByteBufferUtil.bytes(0)), ByteBufferUtil.EMPTY_BYTE_BUFFER, FBUtilities.timestampMicros()); rm.apply(); cfs.forceBlockingFlush(); // shadow the subcolumn with a supercolumn tombstone rm = new RowMutation(KEYSPACE1, key.key); rm.deleteRange( "Super1", SuperColumns.startOf(scName), SuperColumns.endOf(scName), FBUtilities.timestampMicros()); rm.apply(); cfs.forceBlockingFlush(); CompactionManager.instance.performMaximal(cfs); assertEquals(1, cfs.getSSTables().size()); // check that the shadowed column is gone SSTableReader sstable = cfs.getSSTables().iterator().next(); Range keyRange = new Range<RowPosition>(key, sstable.partitioner.getMinimumToken().maxKeyBound()); SSTableScanner scanner = sstable.getScanner(DataRange.forKeyRange(keyRange)); OnDiskAtomIterator iter = scanner.next(); assertEquals(key, iter.getKey()); assert iter.next() instanceof RangeTombstone; assert !iter.hasNext(); }
/* * This excercise in particular the code of #4142 */ @Test public void testValidationMultipleSSTablePerLevel() throws Exception { String ksname = "Keyspace1"; String cfname = "StandardLeveled"; Table table = Table.open(ksname); ColumnFamilyStore store = table.getColumnFamilyStore(cfname); ByteBuffer value = ByteBuffer.wrap(new byte[100 * 1024]); // 100 KB value, make it easy to have multiple files // Enough data to have a level 1 and 2 int rows = 20; int columns = 10; // Adds enough data to trigger multiple sstable per level for (int r = 0; r < rows; r++) { DecoratedKey key = Util.dk(String.valueOf(r)); RowMutation rm = new RowMutation(ksname, key.key); for (int c = 0; c < columns; c++) { rm.add(new QueryPath(cfname, null, ByteBufferUtil.bytes("column" + c)), value, 0); } rm.apply(); store.forceFlush(); } LeveledCompactionStrategy strat = (LeveledCompactionStrategy) store.getCompactionStrategy(); while (strat.getLevelSize(0) > 0) { store.forceMajorCompaction(); Thread.sleep(200); } // Checking we're not completely bad at math assert strat.getLevelSize(1) > 0; assert strat.getLevelSize(2) > 0; AntiEntropyService.CFPair p = new AntiEntropyService.CFPair(ksname, cfname); Range<Token> range = new Range<Token>(Util.token(""), Util.token("")); AntiEntropyService.TreeRequest req = new AntiEntropyService.TreeRequest("1", FBUtilities.getLocalAddress(), range, p); AntiEntropyService.Validator validator = new AntiEntropyService.Validator(req); CompactionManager.instance.submitValidation(store, validator).get(); }
public void insert( String table, String key, ColumnPath column_path, byte[] value, long timestamp, int consistency_level) throws InvalidRequestException, UnavailableException { if (logger.isDebugEnabled()) logger.debug("insert"); ThriftValidation.validateKey(key); ThriftValidation.validateColumnPath(table, column_path); RowMutation rm = new RowMutation(table, key.trim()); try { rm.add(new QueryPath(column_path), value, timestamp); } catch (MarshalException e) { throw new InvalidRequestException(e.getMessage()); } doInsert(consistency_level, rm); }
@Test public void testGetRowNoColumns() throws Throwable { final Table table = Table.open("Keyspace2"); final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard3"); RowMutation rm = new RowMutation("Keyspace2", TEST_KEY); ColumnFamily cf = ColumnFamily.create("Keyspace2", "Standard3"); cf.addColumn(column("col1", "val1", 1L)); rm.add(cf); rm.apply(); Runnable verify = new WrappedRunnable() { public void runMayThrow() throws Exception { ColumnFamily cf; cf = cfStore.getColumnFamily( new NamesQueryFilter( TEST_KEY, new QueryPath("Standard3"), new TreeSet<byte[]>())); assertColumns(cf); cf = cfStore.getColumnFamily( new SliceQueryFilter( TEST_KEY, new QueryPath("Standard3"), ArrayUtils.EMPTY_BYTE_ARRAY, ArrayUtils.EMPTY_BYTE_ARRAY, false, 0)); assertColumns(cf); cf = cfStore.getColumnFamily( new NamesQueryFilter(TEST_KEY, new QueryPath("Standard3"), "col99".getBytes())); assertColumns(cf); } }; reTest(table.getColumnFamilyStore("Standard3"), verify); }
@Test public void simpleQueryWithRangeTombstoneTest() throws Exception { Table table = Table.open(KSNAME); ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME); // Inserting data String key = "k1"; RowMutation rm; ColumnFamily cf; rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); for (int i = 0; i < 40; i += 2) add(rm, i, 0); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); cf = rm.addOrGet(CFNAME); delete(cf, 10, 22, 1); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); for (int i = 1; i < 40; i += 2) add(rm, i, 2); rm.apply(); cfs.forceBlockingFlush(); rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key)); cf = rm.addOrGet(CFNAME); delete(cf, 19, 27, 3); rm.apply(); // We don't flush to test with both a range tomsbtone in memtable and in sstable QueryPath path = new QueryPath(CFNAME); // Queries by name int[] live = new int[] {4, 9, 11, 17, 28}; int[] dead = new int[] {12, 19, 21, 24, 27}; SortedSet<ByteBuffer> columns = new TreeSet<ByteBuffer>(cfs.getComparator()); for (int i : live) columns.add(b(i)); for (int i : dead) columns.add(b(i)); cf = cfs.getColumnFamily(QueryFilter.getNamesFilter(dk(key), path, columns)); for (int i : live) assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live"; for (int i : dead) assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live"; // Queries by slices cf = cfs.getColumnFamily( QueryFilter.getSliceFilter(dk(key), path, b(7), b(30), false, Integer.MAX_VALUE)); for (int i : new int[] {7, 8, 9, 11, 13, 15, 17, 28, 29, 30}) assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live"; for (int i : new int[] {10, 12, 14, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27}) assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live"; }
@Test public void testUserDefinedCompaction() throws Exception { Keyspace keyspace = Keyspace.open(KEYSPACE1); final String cfname = "Standard3"; // use clean(no sstable) CF ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname); // disable compaction while flushing cfs.disableAutoCompaction(); final int ROWS_PER_SSTABLE = 10; for (int i = 0; i < ROWS_PER_SSTABLE; i++) { DecoratedKey key = Util.dk(String.valueOf(i)); RowMutation rm = new RowMutation(KEYSPACE1, key.key); rm.add( cfname, ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); rm.apply(); } cfs.forceBlockingFlush(); Collection<SSTableReader> sstables = cfs.getSSTables(); assert sstables.size() == 1; SSTableReader sstable = sstables.iterator().next(); int prevGeneration = sstable.descriptor.generation; String file = new File(sstable.descriptor.filenameFor(Component.DATA)).getName(); // submit user defined compaction on flushed sstable CompactionManager.instance.forceUserDefinedCompaction(file); // wait until user defined compaction finishes do { Thread.sleep(100); } while (CompactionManager.instance.getPendingTasks() > 0 || CompactionManager.instance.getActiveCompactions() > 0); // CF should have only one sstable with generation number advanced sstables = cfs.getSSTables(); assert sstables.size() == 1; assert sstables.iterator().next().descriptor.generation == prevGeneration + 1; }