Example #1
0
  public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception {
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1);
    store.clearUnsafe();
    store.metadata.gcGraceSeconds(1);
    store.setCompactionStrategyClass(strategyClassName);

    // disable compaction while flushing
    store.disableAutoCompaction();

    long timestamp = populate(KEYSPACE1, CF_STANDARD1, 0, 9, 3); // ttl=3s

    store.forceBlockingFlush();
    assertEquals(1, store.getSSTables().size());
    long originalSize = store.getSSTables().iterator().next().uncompressedLength();

    // wait enough to force single compaction
    TimeUnit.SECONDS.sleep(5);

    // enable compaction, submit background and wait for it to complete
    store.enableAutoCompaction();
    FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
    while (CompactionManager.instance.getPendingTasks() > 0
        || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);

    // and sstable with ttl should be compacted
    assertEquals(1, store.getSSTables().size());
    long size = store.getSSTables().iterator().next().uncompressedLength();
    assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize);

    // make sure max timestamp of compacted sstables is recorded properly after compaction.
    assertMaxTimestamp(store, timestamp);

    return store;
  }
Example #2
0
  public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception {
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
    store.clearUnsafe();
    store.metadata.gcGraceSeconds(1);
    store.setCompactionStrategyClass(strategyClassName);

    // disable compaction while flushing
    store.disableAutoCompaction();

    long timestamp = System.currentTimeMillis();
    for (int i = 0; i < 10; i++) {
      DecoratedKey key = Util.dk(Integer.toString(i));
      RowMutation rm = new RowMutation(KEYSPACE1, key.key);
      for (int j = 0; j < 10; j++)
        rm.add(
            "Standard1",
            ByteBufferUtil.bytes(Integer.toString(j)),
            ByteBufferUtil.EMPTY_BYTE_BUFFER,
            timestamp,
            j > 0
                ? 3
                : 0); // let first column never expire, since deleting all columns does not produce
                      // sstable
      rm.apply();
    }
    store.forceBlockingFlush();
    assertEquals(1, store.getSSTables().size());
    long originalSize = store.getSSTables().iterator().next().uncompressedLength();

    // wait enough to force single compaction
    TimeUnit.SECONDS.sleep(5);

    // enable compaction, submit background and wait for it to complete
    store.enableAutoCompaction();
    FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
    while (CompactionManager.instance.getPendingTasks() > 0
        || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);

    // and sstable with ttl should be compacted
    assertEquals(1, store.getSSTables().size());
    long size = store.getSSTables().iterator().next().uncompressedLength();
    assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize);

    // make sure max timestamp of compacted sstables is recorded properly after compaction.
    assertMaxTimestamp(store, timestamp);

    return store;
  }
  @Test
  public void testNoExpire() throws ExecutionException, InterruptedException {
    ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
    cfs.disableAutoCompaction();
    cfs.metadata.gcGraceSeconds(0);
    long timestamp = System.currentTimeMillis();
    RowMutation rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col7"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);

    rm.apply();
    cfs.forceBlockingFlush();

    rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.apply();
    cfs.forceBlockingFlush();
    rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col3"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.apply();
    cfs.forceBlockingFlush();
    DecoratedKey noTTLKey = Util.dk("nottl");
    rm = new RowMutation("Keyspace1", noTTLKey.key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col311"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp);
    rm.apply();
    cfs.forceBlockingFlush();
    Thread.sleep(2000); // wait for ttl to expire
    assertEquals(4, cfs.getSSTables().size());
    cfs.enableAutoCompaction(true);
    assertEquals(1, cfs.getSSTables().size());
    SSTableReader sstable = cfs.getSSTables().iterator().next();
    SSTableScanner scanner = sstable.getScanner(DataRange.allData(sstable.partitioner));
    assertTrue(scanner.hasNext());
    while (scanner.hasNext()) {
      OnDiskAtomIterator iter = scanner.next();
      assertEquals(noTTLKey, iter.getKey());
    }
  }
  @Test
  public void testSimpleExpire() throws ExecutionException, InterruptedException {
    ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
    cfs.disableAutoCompaction();
    cfs.metadata.gcGraceSeconds(0);
    long timestamp = System.currentTimeMillis();
    RowMutation rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col7"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);

    rm.apply();
    cfs.forceBlockingFlush();

    rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.apply();
    cfs.forceBlockingFlush();
    rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col3"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.apply();
    cfs.forceBlockingFlush();
    rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1",
        ByteBufferUtil.bytes("col311"),
        ByteBufferUtil.EMPTY_BYTE_BUFFER,
        timestamp,
        1);
    rm.apply();

    cfs.forceBlockingFlush();
    Thread.sleep(2000); // wait for ttl to expire
    assertEquals(4, cfs.getSSTables().size());
    cfs.enableAutoCompaction(true);
    assertEquals(0, cfs.getSSTables().size());
  }
Example #5
0
  @Test
  public void testUncheckedTombstoneSizeTieredCompaction() throws Exception {
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1);
    store.clearUnsafe();
    store.metadata.gcGraceSeconds(1);
    store.metadata.compactionStrategyOptions.put("tombstone_compaction_interval", "1");
    store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "false");
    store.reload();
    store.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getName());

    // disable compaction while flushing
    store.disableAutoCompaction();

    // Populate sstable1 with with keys [0..9]
    populate(KEYSPACE1, CF_STANDARD1, 0, 9, 3); // ttl=3s
    store.forceBlockingFlush();

    // Populate sstable2 with with keys [10..19] (keys do not overlap with SSTable1)
    long timestamp2 = populate(KEYSPACE1, CF_STANDARD1, 10, 19, 3); // ttl=3s
    store.forceBlockingFlush();

    assertEquals(2, store.getSSTables().size());

    Iterator<SSTableReader> it = store.getSSTables().iterator();
    long originalSize1 = it.next().uncompressedLength();
    long originalSize2 = it.next().uncompressedLength();

    // wait enough to force single compaction
    TimeUnit.SECONDS.sleep(5);

    // enable compaction, submit background and wait for it to complete
    store.enableAutoCompaction();
    FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
    while (CompactionManager.instance.getPendingTasks() > 0
        || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);

    // even though both sstables were candidate for tombstone compaction
    // it was not executed because they have an overlapping token range
    assertEquals(2, store.getSSTables().size());
    it = store.getSSTables().iterator();
    long newSize1 = it.next().uncompressedLength();
    long newSize2 = it.next().uncompressedLength();
    assertEquals(
        "candidate sstable should not be tombstone-compacted because its key range overlap with other sstable",
        originalSize1,
        newSize1);
    assertEquals(
        "candidate sstable should not be tombstone-compacted because its key range overlap with other sstable",
        originalSize2,
        newSize2);

    // now let's enable the magic property
    store.metadata.compactionStrategyOptions.put("unchecked_tombstone_compaction", "true");
    store.reload();

    // submit background task again and wait for it to complete
    FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
    while (CompactionManager.instance.getPendingTasks() > 0
        || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);

    // we still have 2 sstables, since they were not compacted against each other
    assertEquals(2, store.getSSTables().size());
    it = store.getSSTables().iterator();
    newSize1 = it.next().uncompressedLength();
    newSize2 = it.next().uncompressedLength();
    assertTrue(
        "should be less than " + originalSize1 + ", but was " + newSize1, newSize1 < originalSize1);
    assertTrue(
        "should be less than " + originalSize2 + ", but was " + newSize2, newSize2 < originalSize2);

    // make sure max timestamp of compacted sstables is recorded properly after compaction.
    assertMaxTimestamp(store, timestamp2);
  }