@Test
  public void testOne() throws IOException, ExecutionException, InterruptedException {
    Table table1 = Table.open("Keyspace1");
    Table table2 = Table.open("Keyspace2");

    RowMutation rm;
    DecoratedKey dk = Util.dk("keymulti");
    ColumnFamily cf;

    rm = new RowMutation("Keyspace1", dk.key);
    cf = ColumnFamily.create("Keyspace1", "Standard1");
    cf.addColumn(column("col1", "val1", 1L));
    rm.add(cf);
    rm.apply();

    rm = new RowMutation("Keyspace2", dk.key);
    cf = ColumnFamily.create("Keyspace2", "Standard3");
    cf.addColumn(column("col2", "val2", 1L));
    rm.add(cf);
    rm.apply();

    table1.getColumnFamilyStore("Standard1").clearUnsafe();
    table2.getColumnFamilyStore("Standard3").clearUnsafe();

    CommitLog.instance.resetUnsafe(); // disassociate segments from live CL
    CommitLog.instance.recover();

    assertColumns(Util.getColumnFamily(table1, dk, "Standard1"), "col1");
    assertColumns(Util.getColumnFamily(table2, dk, "Standard3"), "col2");
  }
Beispiel #2
0
  @Test
  public void testEchoedRow() throws IOException, ExecutionException, InterruptedException {
    // This test check that EchoedRow doesn't skipp rows: see CASSANDRA-2653

    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2");

    // disable compaction while flushing
    cfs.disableAutoCompaction();

    // Insert 4 keys in two sstables. We need the sstables to have 2 rows
    // at least to trigger what was causing CASSANDRA-2653
    for (int i = 1; i < 5; i++) {
      DecoratedKey key = Util.dk(String.valueOf(i));
      RowMutation rm = new RowMutation(KEYSPACE1, key.key);
      rm.add(
          "Standard2",
          ByteBufferUtil.bytes(String.valueOf(i)),
          ByteBufferUtil.EMPTY_BYTE_BUFFER,
          i);
      rm.apply();

      if (i % 2 == 0) cfs.forceBlockingFlush();
    }
    Collection<SSTableReader> toCompact = cfs.getSSTables();
    assert toCompact.size() == 2;

    // Reinserting the same keys. We will compact only the previous sstable, but we need those new
    // ones
    // to make sure we use EchoedRow, otherwise it won't be used because purge can be done.
    for (int i = 1; i < 5; i++) {
      DecoratedKey key = Util.dk(String.valueOf(i));
      RowMutation rm = new RowMutation(KEYSPACE1, key.key);
      rm.add(
          "Standard2",
          ByteBufferUtil.bytes(String.valueOf(i)),
          ByteBufferUtil.EMPTY_BYTE_BUFFER,
          i);
      rm.apply();
    }
    cfs.forceBlockingFlush();
    SSTableReader tmpSSTable = null;
    for (SSTableReader sstable : cfs.getSSTables())
      if (!toCompact.contains(sstable)) tmpSSTable = sstable;
    assert tmpSSTable != null;

    // Force compaction on first sstables. Since each row is in only one sstable, we will be using
    // EchoedRow.
    Util.compact(cfs, toCompact);
    assertEquals(2, cfs.getSSTables().size());

    // Now, we remove the sstable that was just created to force the use of EchoedRow (so that it
    // doesn't hide the problem)
    cfs.markObsolete(Collections.singleton(tmpSSTable), OperationType.UNKNOWN);
    assertEquals(1, cfs.getSSTables().size());

    // Now assert we do have the 4 keys
    assertEquals(4, Util.getRangeSlice(cfs).size());
  }
  @Test
  public void testGetSliceFromAdvanced() throws Throwable {
    // tests slicing against data from one row spread across two sstables
    final Table table = Table.open("Keyspace1");
    final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
    final String ROW = "row2";

    RowMutation rm = new RowMutation("Keyspace1", ROW);
    ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
    cf.addColumn(column("col1", "val1", 1L));
    cf.addColumn(column("col2", "val2", 1L));
    cf.addColumn(column("col3", "val3", 1L));
    cf.addColumn(column("col4", "val4", 1L));
    cf.addColumn(column("col5", "val5", 1L));
    cf.addColumn(column("col6", "val6", 1L));
    rm.add(cf);
    rm.apply();
    cfStore.forceBlockingFlush();

    rm = new RowMutation("Keyspace1", ROW);
    cf = ColumnFamily.create("Keyspace1", "Standard1");
    cf.addColumn(column("col1", "valx", 2L));
    cf.addColumn(column("col2", "valx", 2L));
    cf.addColumn(column("col3", "valx", 2L));
    rm.add(cf);
    rm.apply();

    Runnable verify =
        new WrappedRunnable() {
          public void runMayThrow() throws Exception {
            ColumnFamily cf;

            cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Standard1"),
                    "col2".getBytes(),
                    ArrayUtils.EMPTY_BYTE_ARRAY,
                    false,
                    3);
            assertColumns(cf, "col2", "col3", "col4");
            assertEquals(new String(cf.getColumn("col2".getBytes()).value()), "valx");
            assertEquals(new String(cf.getColumn("col3".getBytes()).value()), "valx");
            assertEquals(new String(cf.getColumn("col4".getBytes()).value()), "val4");
          }
        };

    reTest(table.getColumnFamilyStore("Standard1"), verify);
  }
  @Test
  public void testRemoveColumn() throws IOException, ExecutionException, InterruptedException {
    Table table = Table.open("Keyspace1");
    ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
    RowMutation rm;
    DecoratedKey dk = Util.dk("key1");

    // add data
    rm = new RowMutation("Keyspace1", dk.key);
    rm.add(
        new QueryPath("Standard1", null, "Column1".getBytes()),
        "asdf".getBytes(),
        new TimestampClock(0));
    rm.apply();
    store.forceBlockingFlush();

    // remove
    rm = new RowMutation("Keyspace1", dk.key);
    rm.delete(new QueryPath("Standard1", null, "Column1".getBytes()), new TimestampClock(1));
    rm.apply();

    ColumnFamily retrieved =
        store.getColumnFamily(
            QueryFilter.getNamesFilter(dk, new QueryPath("Standard1"), "Column1".getBytes()));
    assert retrieved.getColumn("Column1".getBytes()).isMarkedForDelete();
    assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE));
    assertNull(
        Util.cloneAndRemoveDeleted(
            store.getColumnFamily(QueryFilter.getIdentityFilter(dk, new QueryPath("Standard1"))),
            Integer.MAX_VALUE));
  }
  @Test
  public void testCompactions() throws IOException, ExecutionException, InterruptedException {
    CompactionManager.instance.disableAutoCompaction();

    // this test does enough rows to force multiple block indexes to be used
    Table table = Table.open(TABLE1);
    ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");

    final int ROWS_PER_SSTABLE = 10;
    Set<DecoratedKey> inserted = new HashSet<DecoratedKey>();
    for (int j = 0; j < (DatabaseDescriptor.getIndexInterval() * 3) / ROWS_PER_SSTABLE; j++) {
      for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
        DecoratedKey key = Util.dk(String.valueOf(i % 2));
        RowMutation rm = new RowMutation(TABLE1, key.key);
        rm.add(
            new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i / 2))),
            ByteBufferUtil.EMPTY_BYTE_BUFFER,
            j * ROWS_PER_SSTABLE + i);
        rm.apply();
        inserted.add(key);
      }
      store.forceBlockingFlush();
      assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(store).size());
    }
    while (true) {
      Future<Integer> ft = CompactionManager.instance.submitMinorIfNeeded(store);
      if (ft.get() == 0) break;
    }
    if (store.getSSTables().size() > 1) {
      CompactionManager.instance.performMajor(store);
    }
    assertEquals(inserted.size(), Util.getRangeSlice(store).size());
  }
  @Test
  public void testRemoveColumnFamily()
      throws IOException, ExecutionException, InterruptedException {
    Table table = Table.open("Keyspace1");
    ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
    RowMutation rm;

    // add data
    rm = new RowMutation("Keyspace1", "key1");
    rm.add(new QueryPath("Standard1", null, "Column1".getBytes()), "asdf".getBytes(), 0);
    rm.apply();

    // remove
    rm = new RowMutation("Keyspace1", "key1");
    rm.delete(new QueryPath("Standard1"), 1);
    rm.apply();

    ColumnFamily retrieved =
        store.getColumnFamily(
            new IdentityQueryFilter(
                "key1", new QueryPath("Standard1", null, "Column1".getBytes())));
    assert retrieved.isMarkedForDelete();
    assertNull(retrieved.getColumn("Column1".getBytes()));
    assertNull(ColumnFamilyStore.removeDeleted(retrieved, Integer.MAX_VALUE));
  }
  @Test
  public void testGetSliceFromSuperBasic() throws Throwable {
    // tests slicing against data from one row spread across two sstables
    final Table table = Table.open("Keyspace1");
    final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Super1");
    final String ROW = "row2";

    RowMutation rm = new RowMutation("Keyspace1", ROW);
    ColumnFamily cf = ColumnFamily.create("Keyspace1", "Super1");
    SuperColumn sc = new SuperColumn("sc1".getBytes(), new LongType());
    sc.addColumn(new Column(getBytes(1), "val1".getBytes(), 1L));
    cf.addColumn(sc);
    rm.add(cf);
    rm.apply();

    Runnable verify =
        new WrappedRunnable() {
          public void runMayThrow() throws Exception {
            ColumnFamily cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Super1"),
                    ArrayUtils.EMPTY_BYTE_ARRAY,
                    ArrayUtils.EMPTY_BYTE_ARRAY,
                    false,
                    10);
            assertColumns(cf, "sc1");
            assertEquals(
                new String(cf.getColumn("sc1".getBytes()).getSubColumn(getBytes(1)).value()),
                "val1");
          }
        };

    reTest(table.getColumnFamilyStore("Standard1"), verify);
  }
  @Test
  public void testGetRowSliceByRange() throws Throwable {
    String key = TEST_KEY + "slicerow";
    Table table = Table.open("Keyspace1");
    ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
    RowMutation rm = new RowMutation("Keyspace1", key);
    ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
    // First write "a", "b", "c"
    cf.addColumn(column("a", "val1", 1L));
    cf.addColumn(column("b", "val2", 1L));
    cf.addColumn(column("c", "val3", 1L));
    rm.add(cf);
    rm.apply();

    cf =
        cfStore.getColumnFamily(
            key, new QueryPath("Standard1"), "b".getBytes(), "c".getBytes(), false, 100);
    assertEquals(2, cf.getColumnCount());

    cf =
        cfStore.getColumnFamily(
            key, new QueryPath("Standard1"), "b".getBytes(), "b".getBytes(), false, 100);
    assertEquals(1, cf.getColumnCount());

    cf =
        cfStore.getColumnFamily(
            key, new QueryPath("Standard1"), "b".getBytes(), "c".getBytes(), false, 1);
    assertEquals(1, cf.getColumnCount());

    cf =
        cfStore.getColumnFamily(
            key, new QueryPath("Standard1"), "c".getBytes(), "b".getBytes(), false, 1);
    assertNull(cf);
  }
  @Test
  public void testGetSliceFromLarge() throws Throwable {
    // tests slicing against 1000 columns in an sstable
    Table table = Table.open("Keyspace1");
    ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
    String key = "row3";
    RowMutation rm = new RowMutation("Keyspace1", key);
    ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
    for (int i = 1000; i < 2000; i++) cf.addColumn(column("col" + i, ("v" + i), 1L));
    rm.add(cf);
    rm.apply();
    cfStore.forceBlockingFlush();

    validateSliceLarge(cfStore);
    // compact so we have a big row with more than the minimum index count
    if (cfStore.getSSTables().size() > 1) {
      CompactionManager.instance.submitMajor(cfStore).get();
    }
    SSTableReader sstable = cfStore.getSSTables().iterator().next();
    DecoratedKey decKey = sstable.getPartitioner().decorateKey(key);
    SSTable.PositionSize info = sstable.getPosition(decKey);
    BufferedRandomAccessFile file = new BufferedRandomAccessFile(sstable.getFilename(), "r");
    file.seek(info.position);
    assert file.readUTF().equals(key);
    file.readInt();
    IndexHelper.skipBloomFilter(file);
    ArrayList<IndexHelper.IndexInfo> indexes = IndexHelper.deserializeIndex(file);
    assert indexes.size() > 2;
    validateSliceLarge(cfStore);
  }
  @Test
  public void testRecoverCounter() throws IOException, ExecutionException, InterruptedException {
    Table table1 = Table.open("Keyspace1");

    RowMutation rm;
    DecoratedKey dk = Util.dk("key");
    ColumnFamily cf;

    for (int i = 0; i < 10; ++i) {
      rm = new RowMutation("Keyspace1", dk.key);
      cf = ColumnFamily.create("Keyspace1", "Counter1");
      cf.addColumn(new CounterColumn(ByteBufferUtil.bytes("col"), 1L, 1L));
      rm.add(cf);
      rm.apply();
    }

    table1.getColumnFamilyStore("Counter1").clearUnsafe();

    CommitLog.instance.resetUnsafe(); // disassociate segments from live CL
    CommitLog.instance.recover();

    cf = Util.getColumnFamily(table1, dk, "Counter1");

    assert cf.getColumnCount() == 1;
    Column c = cf.getColumn(ByteBufferUtil.bytes("col"));

    assert c != null;
    assert ((CounterColumn) c).total() == 10L;
  }
  // Test compaction of hints column family. It shouldn't remove all columns on compaction.
  @Test
  public void testCompactionOfHintsCF() throws Exception {
    // prepare hints column family
    Table systemTable = Table.open("system");
    ColumnFamilyStore hintStore = systemTable.getColumnFamilyStore(SystemTable.HINTS_CF);
    hintStore.clearUnsafe();
    hintStore.metadata.gcGraceSeconds(36000); // 10 hours
    hintStore.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getCanonicalName());
    hintStore.disableAutoCompaction();

    // insert 1 hint
    RowMutation rm = new RowMutation(TABLE4, ByteBufferUtil.bytes(1));
    rm.add(
        new QueryPath(STANDARD1_CF, null, ByteBufferUtil.bytes(String.valueOf(COLUMN1))),
        ByteBufferUtil.EMPTY_BYTE_BUFFER,
        System.currentTimeMillis());

    RowMutation.hintFor(rm, UUID.randomUUID()).apply();

    // flush data to disk
    hintStore.forceBlockingFlush();
    assertEquals(1, hintStore.getSSTables().size());

    // submit compaction
    FBUtilities.waitOnFuture(HintedHandOffManager.instance.compact());
    while (CompactionManager.instance.getPendingTasks() > 0
        || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);

    // single row should not be removed because of gc_grace_seconds
    // is 10 hours and there are no any tombstones in sstable
    assertEquals(1, hintStore.getSSTables().size());
  }
  @Test(timeout = 5000)
  public void testTruncateHints() throws Exception {
    Keyspace systemKeyspace = Keyspace.open("system");
    ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS_CF);
    hintStore.clearUnsafe();

    // insert 1 hint
    RowMutation rm = new RowMutation(KEYSPACE4, ByteBufferUtil.bytes(1));
    rm.add(
        STANDARD1_CF,
        ByteBufferUtil.bytes(String.valueOf(COLUMN1)),
        ByteBufferUtil.EMPTY_BYTE_BUFFER,
        System.currentTimeMillis());

    HintedHandOffManager.instance
        .hintFor(
            rm,
            System.currentTimeMillis(),
            HintedHandOffManager.calculateHintTTL(rm),
            UUID.randomUUID())
        .apply();

    assert getNoOfHints() == 1;

    HintedHandOffManager.instance.truncateAllHints();

    while (getNoOfHints() > 0) {
      Thread.sleep(100);
    }

    assert getNoOfHints() == 0;
  }
  @Test
  public void testGetRowSingleColumn() throws Throwable {
    final Table table = Table.open("Keyspace1");
    final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");

    RowMutation rm = new RowMutation("Keyspace1", TEST_KEY);
    ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
    cf.addColumn(column("col1", "val1", 1L));
    cf.addColumn(column("col2", "val2", 1L));
    cf.addColumn(column("col3", "val3", 1L));
    rm.add(cf);
    rm.apply();

    Runnable verify =
        new WrappedRunnable() {
          public void runMayThrow() throws Exception {
            ColumnFamily cf;

            cf =
                cfStore.getColumnFamily(
                    new NamesQueryFilter(TEST_KEY, new QueryPath("Standard1"), "col1".getBytes()));
            assertColumns(cf, "col1");

            cf =
                cfStore.getColumnFamily(
                    new NamesQueryFilter(TEST_KEY, new QueryPath("Standard1"), "col3".getBytes()));
            assertColumns(cf, "col3");
          }
        };
    reTest(table.getColumnFamilyStore("Standard1"), verify);
  }
Beispiel #14
0
 private static void insertRowWithKey(int key) {
   long timestamp = System.currentTimeMillis();
   DecoratedKey decoratedKey = Util.dk(String.format("%03d", key));
   RowMutation rm = new RowMutation(KEYSPACE1, decoratedKey.key);
   rm.add(
       "Standard1",
       ByteBufferUtil.bytes("col"),
       ByteBufferUtil.EMPTY_BYTE_BUFFER,
       timestamp,
       1000);
   rm.apply();
 }
Beispiel #15
0
 public static void addMutation(
     RowMutation rm,
     String columnFamilyName,
     String superColumnName,
     long columnName,
     String value,
     long timestamp) {
   rm.add(
       new QueryPath(
           columnFamilyName, ByteBufferUtil.bytes(superColumnName), getBytes(columnName)),
       ByteBufferUtil.bytes(value),
       timestamp);
 }
Beispiel #16
0
  private void testDontPurgeAccidentaly(String k, String cfname)
      throws IOException, ExecutionException, InterruptedException {
    // This test catches the regression of CASSANDRA-2786
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);

    // disable compaction while flushing
    cfs.clearUnsafe();
    cfs.disableAutoCompaction();

    // Add test row
    DecoratedKey key = Util.dk(k);
    RowMutation rm = new RowMutation(KEYSPACE1, key.key);
    rm.add(
        cfname,
        CompositeType.build(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")),
        ByteBufferUtil.EMPTY_BYTE_BUFFER,
        0);
    rm.apply();

    cfs.forceBlockingFlush();

    Collection<SSTableReader> sstablesBefore = cfs.getSSTables();

    QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis());
    assert !(cfs.getColumnFamily(filter).getColumnCount() == 0);

    // Remove key
    rm = new RowMutation(KEYSPACE1, key.key);
    rm.delete(cfname, 2);
    rm.apply();

    ColumnFamily cf = cfs.getColumnFamily(filter);
    assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;

    // Sleep one second so that the removal is indeed purgeable even with gcgrace == 0
    Thread.sleep(1000);

    cfs.forceBlockingFlush();

    Collection<SSTableReader> sstablesAfter = cfs.getSSTables();
    Collection<SSTableReader> toCompact = new ArrayList<SSTableReader>();
    for (SSTableReader sstable : sstablesAfter)
      if (!sstablesBefore.contains(sstable)) toCompact.add(sstable);

    Util.compact(cfs, toCompact);

    cf = cfs.getColumnFamily(filter);
    assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;
  }
Beispiel #17
0
  public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception {
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
    store.clearUnsafe();
    store.metadata.gcGraceSeconds(1);
    store.setCompactionStrategyClass(strategyClassName);

    // disable compaction while flushing
    store.disableAutoCompaction();

    long timestamp = System.currentTimeMillis();
    for (int i = 0; i < 10; i++) {
      DecoratedKey key = Util.dk(Integer.toString(i));
      RowMutation rm = new RowMutation(KEYSPACE1, key.key);
      for (int j = 0; j < 10; j++)
        rm.add(
            "Standard1",
            ByteBufferUtil.bytes(Integer.toString(j)),
            ByteBufferUtil.EMPTY_BYTE_BUFFER,
            timestamp,
            j > 0
                ? 3
                : 0); // let first column never expire, since deleting all columns does not produce
                      // sstable
      rm.apply();
    }
    store.forceBlockingFlush();
    assertEquals(1, store.getSSTables().size());
    long originalSize = store.getSSTables().iterator().next().uncompressedLength();

    // wait enough to force single compaction
    TimeUnit.SECONDS.sleep(5);

    // enable compaction, submit background and wait for it to complete
    store.enableAutoCompaction();
    FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
    while (CompactionManager.instance.getPendingTasks() > 0
        || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);

    // and sstable with ttl should be compacted
    assertEquals(1, store.getSSTables().size());
    long size = store.getSSTables().iterator().next().uncompressedLength();
    assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize);

    // make sure max timestamp of compacted sstables is recorded properly after compaction.
    assertMaxTimestamp(store, timestamp);

    return store;
  }
Beispiel #18
0
  protected void fillCF(ColumnFamilyStore cfs, int rowsPerSSTable)
      throws ExecutionException, InterruptedException, IOException {
    CompactionManager.instance.disableAutoCompaction();

    for (int i = 0; i < rowsPerSSTable; i++) {
      String key = String.valueOf(i);
      // create a row and update the birthdate value, test that the index query fetches the new
      // version
      RowMutation rm;
      rm = new RowMutation(KEYSPACE1, ByteBufferUtil.bytes(key));
      rm.add(cfs.name, COLUMN, VALUE, System.currentTimeMillis());
      rm.applyUnsafe();
    }

    cfs.forceBlockingFlush();
  }
  @Test
  public void testGetColumn() throws IOException, ColumnFamilyNotDefinedException {
    Table table = Table.open("Keyspace1");
    RowMutation rm;

    // add data
    rm = new RowMutation("Keyspace1", "key1");
    rm.add(new QueryPath("Standard1", null, "Column1".getBytes()), "abcd".getBytes(), 0);
    rm.apply();

    ReadCommand command =
        new SliceByNamesReadCommand(
            "Keyspace1", "key1", new QueryPath("Standard1"), Arrays.asList("Column1".getBytes()));
    Row row = command.getRow(table);
    IColumn col = row.cf.getColumn("Column1".getBytes());
    assert Arrays.equals(col.value(), "abcd".getBytes());
  }
  @Test
  public void testGetSliceNoMatch() throws Throwable {
    Table table = Table.open("Keyspace1");
    RowMutation rm = new RowMutation("Keyspace1", "row1000");
    ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard2");
    cf.addColumn(column("col1", "val1", 1));
    rm.add(cf);
    rm.apply();

    validateGetSliceNoMatch(table);
    table.getColumnFamilyStore("Standard2").forceBlockingFlush();
    validateGetSliceNoMatch(table);

    Collection<SSTableReader> ssTables = table.getColumnFamilyStore("Standard2").getSSTables();
    assertEquals(1, ssTables.size());
    ssTables.iterator().next().forceBloomFilterFailures();
    validateGetSliceNoMatch(table);
  }
Beispiel #21
0
  @Test
  public void testSuperColumnTombstones()
      throws IOException, ExecutionException, InterruptedException {
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1");
    cfs.disableAutoCompaction();

    DecoratedKey key = Util.dk("tskey");
    ByteBuffer scName = ByteBufferUtil.bytes("TestSuperColumn");

    // a subcolumn
    RowMutation rm = new RowMutation(KEYSPACE1, key.key);
    rm.add(
        "Super1",
        CompositeType.build(scName, ByteBufferUtil.bytes(0)),
        ByteBufferUtil.EMPTY_BYTE_BUFFER,
        FBUtilities.timestampMicros());
    rm.apply();
    cfs.forceBlockingFlush();

    // shadow the subcolumn with a supercolumn tombstone
    rm = new RowMutation(KEYSPACE1, key.key);
    rm.deleteRange(
        "Super1",
        SuperColumns.startOf(scName),
        SuperColumns.endOf(scName),
        FBUtilities.timestampMicros());
    rm.apply();
    cfs.forceBlockingFlush();

    CompactionManager.instance.performMaximal(cfs);
    assertEquals(1, cfs.getSSTables().size());

    // check that the shadowed column is gone
    SSTableReader sstable = cfs.getSSTables().iterator().next();
    Range keyRange =
        new Range<RowPosition>(key, sstable.partitioner.getMinimumToken().maxKeyBound());
    SSTableScanner scanner = sstable.getScanner(DataRange.forKeyRange(keyRange));
    OnDiskAtomIterator iter = scanner.next();
    assertEquals(key, iter.getKey());
    assert iter.next() instanceof RangeTombstone;
    assert !iter.hasNext();
  }
Beispiel #22
0
  /*
   * This excercise in particular the code of #4142
   */
  @Test
  public void testValidationMultipleSSTablePerLevel() throws Exception {
    String ksname = "Keyspace1";
    String cfname = "StandardLeveled";
    Table table = Table.open(ksname);
    ColumnFamilyStore store = table.getColumnFamilyStore(cfname);

    ByteBuffer value =
        ByteBuffer.wrap(new byte[100 * 1024]); // 100 KB value, make it easy to have multiple files

    // Enough data to have a level 1 and 2
    int rows = 20;
    int columns = 10;

    // Adds enough data to trigger multiple sstable per level
    for (int r = 0; r < rows; r++) {
      DecoratedKey key = Util.dk(String.valueOf(r));
      RowMutation rm = new RowMutation(ksname, key.key);
      for (int c = 0; c < columns; c++) {
        rm.add(new QueryPath(cfname, null, ByteBufferUtil.bytes("column" + c)), value, 0);
      }
      rm.apply();
      store.forceFlush();
    }

    LeveledCompactionStrategy strat = (LeveledCompactionStrategy) store.getCompactionStrategy();

    while (strat.getLevelSize(0) > 0) {
      store.forceMajorCompaction();
      Thread.sleep(200);
    }
    // Checking we're not completely bad at math
    assert strat.getLevelSize(1) > 0;
    assert strat.getLevelSize(2) > 0;

    AntiEntropyService.CFPair p = new AntiEntropyService.CFPair(ksname, cfname);
    Range<Token> range = new Range<Token>(Util.token(""), Util.token(""));
    AntiEntropyService.TreeRequest req =
        new AntiEntropyService.TreeRequest("1", FBUtilities.getLocalAddress(), range, p);
    AntiEntropyService.Validator validator = new AntiEntropyService.Validator(req);
    CompactionManager.instance.submitValidation(store, validator).get();
  }
Beispiel #23
0
  public void insert(
      String table,
      String key,
      ColumnPath column_path,
      byte[] value,
      long timestamp,
      int consistency_level)
      throws InvalidRequestException, UnavailableException {
    if (logger.isDebugEnabled()) logger.debug("insert");
    ThriftValidation.validateKey(key);
    ThriftValidation.validateColumnPath(table, column_path);

    RowMutation rm = new RowMutation(table, key.trim());
    try {
      rm.add(new QueryPath(column_path), value, timestamp);
    } catch (MarshalException e) {
      throw new InvalidRequestException(e.getMessage());
    }
    doInsert(consistency_level, rm);
  }
  @Test
  public void testGetRowNoColumns() throws Throwable {
    final Table table = Table.open("Keyspace2");
    final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard3");

    RowMutation rm = new RowMutation("Keyspace2", TEST_KEY);
    ColumnFamily cf = ColumnFamily.create("Keyspace2", "Standard3");
    cf.addColumn(column("col1", "val1", 1L));
    rm.add(cf);
    rm.apply();

    Runnable verify =
        new WrappedRunnable() {
          public void runMayThrow() throws Exception {
            ColumnFamily cf;

            cf =
                cfStore.getColumnFamily(
                    new NamesQueryFilter(
                        TEST_KEY, new QueryPath("Standard3"), new TreeSet<byte[]>()));
            assertColumns(cf);

            cf =
                cfStore.getColumnFamily(
                    new SliceQueryFilter(
                        TEST_KEY,
                        new QueryPath("Standard3"),
                        ArrayUtils.EMPTY_BYTE_ARRAY,
                        ArrayUtils.EMPTY_BYTE_ARRAY,
                        false,
                        0));
            assertColumns(cf);

            cf =
                cfStore.getColumnFamily(
                    new NamesQueryFilter(TEST_KEY, new QueryPath("Standard3"), "col99".getBytes()));
            assertColumns(cf);
          }
        };
    reTest(table.getColumnFamilyStore("Standard3"), verify);
  }
Beispiel #25
0
  @Test
  public void testUserDefinedCompaction() throws Exception {
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    final String cfname = "Standard3"; // use clean(no sstable) CF
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);

    // disable compaction while flushing
    cfs.disableAutoCompaction();

    final int ROWS_PER_SSTABLE = 10;
    for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
      DecoratedKey key = Util.dk(String.valueOf(i));
      RowMutation rm = new RowMutation(KEYSPACE1, key.key);
      rm.add(
          cfname,
          ByteBufferUtil.bytes("col"),
          ByteBufferUtil.EMPTY_BYTE_BUFFER,
          System.currentTimeMillis());
      rm.apply();
    }
    cfs.forceBlockingFlush();
    Collection<SSTableReader> sstables = cfs.getSSTables();

    assert sstables.size() == 1;
    SSTableReader sstable = sstables.iterator().next();

    int prevGeneration = sstable.descriptor.generation;
    String file = new File(sstable.descriptor.filenameFor(Component.DATA)).getName();
    // submit user defined compaction on flushed sstable
    CompactionManager.instance.forceUserDefinedCompaction(file);
    // wait until user defined compaction finishes
    do {
      Thread.sleep(100);
    } while (CompactionManager.instance.getPendingTasks() > 0
        || CompactionManager.instance.getActiveCompactions() > 0);
    // CF should have only one sstable with generation number advanced
    sstables = cfs.getSSTables();
    assert sstables.size() == 1;
    assert sstables.iterator().next().descriptor.generation == prevGeneration + 1;
  }
  @Test
  public void testGetSliceWithCutoff() throws Throwable {
    // tests slicing against data from one row in a memtable and then flushed to an sstable
    final Table table = Table.open("Keyspace1");
    final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
    final String ROW = "row4";
    final NumberFormat fmt = new DecimalFormat("000");

    RowMutation rm = new RowMutation("Keyspace1", ROW);
    ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
    // at this rate, we're getting 78-79 cos/block, assuming the blocks are set to be about 4k.
    // so if we go to 300, we'll get at least 4 blocks, which is plenty for testing.
    for (int i = 0; i < 300; i++)
      cf.addColumn(column("col" + fmt.format(i), "omg!thisisthevalue!" + i, 1L));
    rm.add(cf);
    rm.apply();

    Runnable verify =
        new WrappedRunnable() {
          public void runMayThrow() throws Exception {
            ColumnFamily cf;

            // blocks are partitioned like this: 000-097, 098-193, 194-289, 290-299, assuming a 4k
            // column index size.
            assert DatabaseDescriptor.getColumnIndexSize() == 4096
                : "Unexpected column index size, block boundaries won't be where tests expect them.";

            // test forward, spanning a segment.
            cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Standard1"),
                    "col096".getBytes(),
                    "col099".getBytes(),
                    false,
                    4);
            assertColumns(cf, "col096", "col097", "col098", "col099");

            // test reversed, spanning a segment.
            cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Standard1"),
                    "col099".getBytes(),
                    "col096".getBytes(),
                    true,
                    4);
            assertColumns(cf, "col096", "col097", "col098", "col099");

            // test forward, within a segment.
            cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Standard1"),
                    "col100".getBytes(),
                    "col103".getBytes(),
                    false,
                    4);
            assertColumns(cf, "col100", "col101", "col102", "col103");

            // test reversed, within a segment.
            cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Standard1"),
                    "col103".getBytes(),
                    "col100".getBytes(),
                    true,
                    4);
            assertColumns(cf, "col100", "col101", "col102", "col103");

            // test forward from beginning, spanning a segment.
            String[] strCols = new String[100]; // col000-col099
            for (int i = 0; i < 100; i++) strCols[i] = "col" + fmt.format(i);
            cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Standard1"),
                    "".getBytes(),
                    "col099".getBytes(),
                    false,
                    100);
            assertColumns(cf, strCols);

            // test reversed, from end, spanning a segment.
            cf =
                cfStore.getColumnFamily(
                    ROW, new QueryPath("Standard1"), "".getBytes(), "col288".getBytes(), true, 12);
            assertColumns(
                cf, "col288", "col289", "col290", "col291", "col292", "col293", "col294", "col295",
                "col296", "col297", "col298", "col299");
          }
        };

    reTest(table.getColumnFamilyStore("Standard1"), verify);
  }
 private static void add(RowMutation rm, int value, long timestamp) {
   rm.add(new QueryPath(CFNAME, null, b(value)), b(value), timestamp);
 }
  @Test
  public void testGetSliceFromBasic() throws Throwable {
    // tests slicing against data from one row in a memtable and then flushed to an sstable
    final Table table = Table.open("Keyspace1");
    final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
    final String ROW = "row1";

    RowMutation rm = new RowMutation("Keyspace1", ROW);
    ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
    cf.addColumn(column("col1", "val1", 1L));
    cf.addColumn(column("col3", "val3", 1L));
    cf.addColumn(column("col4", "val4", 1L));
    cf.addColumn(column("col5", "val5", 1L));
    cf.addColumn(column("col7", "val7", 1L));
    cf.addColumn(column("col9", "val9", 1L));
    rm.add(cf);
    rm.apply();

    rm = new RowMutation("Keyspace1", ROW);
    rm.delete(new QueryPath("Standard1", null, "col4".getBytes()), 2L);
    rm.apply();

    Runnable verify =
        new WrappedRunnable() {
          public void runMayThrow() throws Exception {
            ColumnFamily cf;

            cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Standard1"),
                    "col5".getBytes(),
                    ArrayUtils.EMPTY_BYTE_ARRAY,
                    false,
                    2);
            assertColumns(cf, "col5", "col7");

            cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Standard1"),
                    "col4".getBytes(),
                    ArrayUtils.EMPTY_BYTE_ARRAY,
                    false,
                    2);
            assertColumns(cf, "col4", "col5", "col7");
            assertColumns(ColumnFamilyStore.removeDeleted(cf, Integer.MAX_VALUE), "col5", "col7");

            cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Standard1"),
                    "col5".getBytes(),
                    ArrayUtils.EMPTY_BYTE_ARRAY,
                    true,
                    2);
            assertColumns(cf, "col3", "col4", "col5");

            cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Standard1"),
                    "col6".getBytes(),
                    ArrayUtils.EMPTY_BYTE_ARRAY,
                    true,
                    2);
            assertColumns(cf, "col3", "col4", "col5");

            cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Standard1"),
                    ArrayUtils.EMPTY_BYTE_ARRAY,
                    ArrayUtils.EMPTY_BYTE_ARRAY,
                    true,
                    2);
            assertColumns(cf, "col7", "col9");

            cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Standard1"),
                    "col95".getBytes(),
                    ArrayUtils.EMPTY_BYTE_ARRAY,
                    false,
                    2);
            assertColumns(cf);

            cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Standard1"),
                    "col0".getBytes(),
                    ArrayUtils.EMPTY_BYTE_ARRAY,
                    true,
                    2);
            assertColumns(cf);
          }
        };

    reTest(table.getColumnFamilyStore("Standard1"), verify);
  }