Exemplo n.º 1
0
  @Test
  public void testCleanupWithNewToken()
      throws ExecutionException, InterruptedException, UnknownHostException {
    StorageService.instance.getTokenMetadata().clearUnsafe();

    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF2);

    List<Row> rows;

    // insert data and verify we get it back w/ range query
    fillCF(cfs, LOOPS);

    rows = Util.getRangeSlice(cfs);

    assertEquals(LOOPS, rows.size());
    TokenMetadata tmd = StorageService.instance.getTokenMetadata();

    byte[] tk1 = new byte[1], tk2 = new byte[1];
    tk1[0] = 2;
    tk2[0] = 1;
    tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1"));
    tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2"));
    CompactionManager.instance.performCleanup(cfs);

    rows = Util.getRangeSlice(cfs);
    assertEquals(0, rows.size());
  }
  @Test
  public void testRemoveColumn() throws IOException, ExecutionException, InterruptedException {
    Table table = Table.open("Keyspace1");
    ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");
    RowMutation rm;
    DecoratedKey dk = Util.dk("key1");

    // add data
    rm = new RowMutation("Keyspace1", dk.key);
    rm.add(
        new QueryPath("Standard1", null, "Column1".getBytes()),
        "asdf".getBytes(),
        new TimestampClock(0));
    rm.apply();
    store.forceBlockingFlush();

    // remove
    rm = new RowMutation("Keyspace1", dk.key);
    rm.delete(new QueryPath("Standard1", null, "Column1".getBytes()), new TimestampClock(1));
    rm.apply();

    ColumnFamily retrieved =
        store.getColumnFamily(
            QueryFilter.getNamesFilter(dk, new QueryPath("Standard1"), "Column1".getBytes()));
    assert retrieved.getColumn("Column1".getBytes()).isMarkedForDelete();
    assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE));
    assertNull(
        Util.cloneAndRemoveDeleted(
            store.getColumnFamily(QueryFilter.getIdentityFilter(dk, new QueryPath("Standard1"))),
            Integer.MAX_VALUE));
  }
Exemplo n.º 3
0
 private static void insertRowWithKey(int key) {
   long timestamp = System.currentTimeMillis();
   DecoratedKey decoratedKey = Util.dk(String.format("%03d", key));
   Mutation rm = new Mutation(KEYSPACE1, decoratedKey.getKey());
   rm.add("CF_STANDARD1", Util.cellname("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1000);
   rm.applyUnsafe();
 }
  @Test
  public void testRecoverCounter() throws IOException, ExecutionException, InterruptedException {
    Table table1 = Table.open("Keyspace1");

    RowMutation rm;
    DecoratedKey dk = Util.dk("key");
    ColumnFamily cf;

    for (int i = 0; i < 10; ++i) {
      rm = new RowMutation("Keyspace1", dk.key);
      cf = ColumnFamily.create("Keyspace1", "Counter1");
      cf.addColumn(new CounterColumn(ByteBufferUtil.bytes("col"), 1L, 1L));
      rm.add(cf);
      rm.apply();
    }

    table1.getColumnFamilyStore("Counter1").clearUnsafe();

    CommitLog.instance.resetUnsafe(); // disassociate segments from live CL
    CommitLog.instance.recover();

    cf = Util.getColumnFamily(table1, dk, "Counter1");

    assert cf.getColumnCount() == 1;
    Column c = cf.getColumn(ByteBufferUtil.bytes("col"));

    assert c != null;
    assert ((CounterColumn) c).total() == 10L;
  }
  @Test
  public void testOne() throws IOException, ExecutionException, InterruptedException {
    Table table1 = Table.open("Keyspace1");
    Table table2 = Table.open("Keyspace2");

    RowMutation rm;
    DecoratedKey dk = Util.dk("keymulti");
    ColumnFamily cf;

    rm = new RowMutation("Keyspace1", dk.key);
    cf = ColumnFamily.create("Keyspace1", "Standard1");
    cf.addColumn(column("col1", "val1", 1L));
    rm.add(cf);
    rm.apply();

    rm = new RowMutation("Keyspace2", dk.key);
    cf = ColumnFamily.create("Keyspace2", "Standard3");
    cf.addColumn(column("col2", "val2", 1L));
    rm.add(cf);
    rm.apply();

    table1.getColumnFamilyStore("Standard1").clearUnsafe();
    table2.getColumnFamilyStore("Standard3").clearUnsafe();

    CommitLog.instance.resetUnsafe(); // disassociate segments from live CL
    CommitLog.instance.recover();

    assertColumns(Util.getColumnFamily(table1, dk, "Standard1"), "col1");
    assertColumns(Util.getColumnFamily(table2, dk, "Standard3"), "col2");
  }
  @Test
  public void testRemoveColumnFamilyWithFlush1() {
    Keyspace keyspace = Keyspace.open("Keyspace1");
    ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
    Mutation rm;
    DecoratedKey dk = Util.dk("key1");

    // add data
    rm = new Mutation("Keyspace1", dk.key);
    rm.add("Standard1", Util.cellname("Column1"), ByteBufferUtil.bytes("asdf"), 0);
    rm.add("Standard1", Util.cellname("Column2"), ByteBufferUtil.bytes("asdf"), 0);
    rm.apply();
    store.forceBlockingFlush();

    // remove
    rm = new Mutation("Keyspace1", dk.key);
    rm.delete("Standard1", 1);
    rm.apply();

    ColumnFamily retrieved =
        store.getColumnFamily(
            QueryFilter.getIdentityFilter(dk, "Standard1", System.currentTimeMillis()));
    assert retrieved.isMarkedForDelete();
    assertNull(retrieved.getColumn(Util.cellname("Column1")));
    assertNull(Util.cloneAndRemoveDeleted(retrieved, Integer.MAX_VALUE));
  }
Exemplo n.º 7
0
  @Test
  public void testCleanup()
      throws IOException, ExecutionException, InterruptedException, ConfigurationException {
    StorageService.instance.initServer(0);

    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF2);

    List<Row> rows;

    // insert data and verify we get it back w/ range query
    fillCF(cfs, LOOPS);

    // record max timestamps of the sstables pre-cleanup
    List<Long> expectedMaxTimestamps = getMaxTimestampList(cfs);

    rows = Util.getRangeSlice(cfs);
    assertEquals(LOOPS, rows.size());

    // with one token in the ring, owned by the local node, cleanup should be a no-op
    CompactionManager.instance.performCleanup(cfs, new CounterId.OneShotRenewer());

    // ensure max timestamp of the sstables are retained post-cleanup
    assert expectedMaxTimestamps.equals(getMaxTimestampList(cfs));

    // check data is still there
    rows = Util.getRangeSlice(cfs);
    assertEquals(LOOPS, rows.size());
  }
Exemplo n.º 8
0
  @Test
  public void testCompactions() throws IOException, ExecutionException, InterruptedException {
    CompactionManager.instance.disableAutoCompaction();

    // this test does enough rows to force multiple block indexes to be used
    Table table = Table.open(TABLE1);
    ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");

    final int ROWS_PER_SSTABLE = 10;
    Set<DecoratedKey> inserted = new HashSet<DecoratedKey>();
    for (int j = 0; j < (DatabaseDescriptor.getIndexInterval() * 3) / ROWS_PER_SSTABLE; j++) {
      for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
        DecoratedKey key = Util.dk(String.valueOf(i % 2));
        RowMutation rm = new RowMutation(TABLE1, key.key);
        rm.add(
            new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i / 2))),
            ByteBufferUtil.EMPTY_BYTE_BUFFER,
            j * ROWS_PER_SSTABLE + i);
        rm.apply();
        inserted.add(key);
      }
      store.forceBlockingFlush();
      assertEquals(inserted.toString(), inserted.size(), Util.getRangeSlice(store).size());
    }
    while (true) {
      Future<Integer> ft = CompactionManager.instance.submitMinorIfNeeded(store);
      if (ft.get() == 0) break;
    }
    if (store.getSSTables().size() > 1) {
      CompactionManager.instance.performMajor(store);
    }
    assertEquals(inserted.size(), Util.getRangeSlice(store).size());
  }
  @Test
  public void testCheckForExpiredSSTableBlockers() throws InterruptedException {
    String KEYSPACE1 = "Keyspace1";
    ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
    cfs.truncateBlocking();
    cfs.disableAutoCompaction();
    cfs.metadata.gcGraceSeconds(0);

    RowMutation rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
    rm.add(
        "Standard1",
        ByteBufferUtil.bytes("col1"),
        ByteBufferUtil.EMPTY_BYTE_BUFFER,
        System.currentTimeMillis());
    rm.applyUnsafe();
    cfs.forceBlockingFlush();
    SSTableReader blockingSSTable = cfs.getSSTables().iterator().next();
    for (int i = 0; i < 10; i++) {
      rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
      rm.delete("Standard1", System.currentTimeMillis());
      rm.applyUnsafe();
      cfs.forceBlockingFlush();
    }
    Multimap<SSTableReader, SSTableReader> blockers =
        SSTableExpiredBlockers.checkForExpiredSSTableBlockers(
            cfs.getSSTables(), (int) (System.currentTimeMillis() / 1000) + 100);
    assertEquals(1, blockers.keySet().size());
    assertTrue(blockers.keySet().contains(blockingSSTable));
    assertEquals(10, blockers.get(blockingSSTable).size());
  }
Exemplo n.º 10
0
  @Test
  public void testEchoedRow() throws IOException, ExecutionException, InterruptedException {
    // This test check that EchoedRow doesn't skipp rows: see CASSANDRA-2653

    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2");

    // disable compaction while flushing
    cfs.disableAutoCompaction();

    // Insert 4 keys in two sstables. We need the sstables to have 2 rows
    // at least to trigger what was causing CASSANDRA-2653
    for (int i = 1; i < 5; i++) {
      DecoratedKey key = Util.dk(String.valueOf(i));
      RowMutation rm = new RowMutation(KEYSPACE1, key.key);
      rm.add(
          "Standard2",
          ByteBufferUtil.bytes(String.valueOf(i)),
          ByteBufferUtil.EMPTY_BYTE_BUFFER,
          i);
      rm.apply();

      if (i % 2 == 0) cfs.forceBlockingFlush();
    }
    Collection<SSTableReader> toCompact = cfs.getSSTables();
    assert toCompact.size() == 2;

    // Reinserting the same keys. We will compact only the previous sstable, but we need those new
    // ones
    // to make sure we use EchoedRow, otherwise it won't be used because purge can be done.
    for (int i = 1; i < 5; i++) {
      DecoratedKey key = Util.dk(String.valueOf(i));
      RowMutation rm = new RowMutation(KEYSPACE1, key.key);
      rm.add(
          "Standard2",
          ByteBufferUtil.bytes(String.valueOf(i)),
          ByteBufferUtil.EMPTY_BYTE_BUFFER,
          i);
      rm.apply();
    }
    cfs.forceBlockingFlush();
    SSTableReader tmpSSTable = null;
    for (SSTableReader sstable : cfs.getSSTables())
      if (!toCompact.contains(sstable)) tmpSSTable = sstable;
    assert tmpSSTable != null;

    // Force compaction on first sstables. Since each row is in only one sstable, we will be using
    // EchoedRow.
    Util.compact(cfs, toCompact);
    assertEquals(2, cfs.getSSTables().size());

    // Now, we remove the sstable that was just created to force the use of EchoedRow (so that it
    // doesn't hide the problem)
    cfs.markObsolete(Collections.singleton(tmpSSTable), OperationType.UNKNOWN);
    assertEquals(1, cfs.getSSTables().size());

    // Now assert we do have the 4 keys
    assertEquals(4, Util.getRangeSlice(cfs).size());
  }
Exemplo n.º 11
0
  private static class Statics {
    private final String KS = "Keyspace1";
    private final ByteBuffer Key = ByteBufferUtil.bytes("Key01");
    private final SortedSet<ByteBuffer> NamedCols =
        new TreeSet<ByteBuffer>(BytesType.instance) {
          {
            add(ByteBufferUtil.bytes("AAA"));
            add(ByteBufferUtil.bytes("BBB"));
            add(ByteBufferUtil.bytes("CCC"));
          }
        };
    private final ByteBuffer SC = ByteBufferUtil.bytes("SCName");
    private final SortedSet<ByteBuffer> NamedSCCols =
        new TreeSet<ByteBuffer>(BytesType.instance) {
          {
            add(CompositeType.build(SC, ByteBufferUtil.bytes("AAA")));
            add(CompositeType.build(SC, ByteBufferUtil.bytes("BBB")));
            add(CompositeType.build(SC, ByteBufferUtil.bytes("CCC")));
          }
        };
    private final String StandardCF = "Standard1";
    private final String SuperCF = "Super1";

    private final long readTs = 1369935512292L;

    private final ColumnFamily StandardCf =
        TreeMapBackedSortedColumns.factory.create(KS, StandardCF);
    private final ColumnFamily SuperCf = TreeMapBackedSortedColumns.factory.create(KS, SuperCF);

    private final Row StandardRow = new Row(Util.dk("key0"), StandardCf);
    private final Row SuperRow = new Row(Util.dk("key1"), SuperCf);
    private final Row NullRow = new Row(Util.dk("key2"), null);

    private Statics() {
      StandardCf.addColumn(new Column(bb("aaaa")));
      StandardCf.addColumn(new Column(bb("bbbb"), bb("bbbbb-value")));
      StandardCf.addColumn(new Column(bb("cccc"), bb("ccccc-value"), 1000L));
      StandardCf.addColumn(new DeletedColumn(bb("dddd"), 500, 1000));
      StandardCf.addColumn(new DeletedColumn(bb("eeee"), bb("eeee-value"), 1001));
      StandardCf.addColumn(new ExpiringColumn(bb("ffff"), bb("ffff-value"), 2000, 1000));
      StandardCf.addColumn(new ExpiringColumn(bb("gggg"), bb("gggg-value"), 2001, 1000, 2002));

      SuperCf.addColumn(new Column(CompositeType.build(SC, bb("aaaa"))));
      SuperCf.addColumn(new Column(CompositeType.build(SC, bb("bbbb")), bb("bbbbb-value")));
      SuperCf.addColumn(new Column(CompositeType.build(SC, bb("cccc")), bb("ccccc-value"), 1000L));
      SuperCf.addColumn(new DeletedColumn(CompositeType.build(SC, bb("dddd")), 500, 1000));
      SuperCf.addColumn(
          new DeletedColumn(CompositeType.build(SC, bb("eeee")), bb("eeee-value"), 1001));
      SuperCf.addColumn(
          new ExpiringColumn(CompositeType.build(SC, bb("ffff")), bb("ffff-value"), 2000, 1000));
      SuperCf.addColumn(
          new ExpiringColumn(
              CompositeType.build(SC, bb("gggg")), bb("gggg-value"), 2001, 1000, 2002));
    }
  }
Exemplo n.º 12
0
  @Test
  public void testCleanupWithIndexes()
      throws IOException, ExecutionException, InterruptedException {
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF1);

    List<Row> rows;

    // insert data and verify we get it back w/ range query
    fillCF(cfs, LOOPS);
    rows = Util.getRangeSlice(cfs);
    assertEquals(LOOPS, rows.size());

    SecondaryIndex index = cfs.indexManager.getIndexForColumn(COLUMN);
    long start = System.nanoTime();
    while (!index.isIndexBuilt(COLUMN) && System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10))
      Thread.sleep(10);

    // verify we get it back w/ index query too
    IndexExpression expr = new IndexExpression(COLUMN, IndexExpression.Operator.EQ, VALUE);
    List<IndexExpression> clause = Arrays.asList(expr);
    IDiskAtomFilter filter = new IdentityQueryFilter();
    IPartitioner p = StorageService.getPartitioner();
    Range<RowPosition> range = Util.range("", "");
    rows = keyspace.getColumnFamilyStore(CF1).search(range, clause, filter, Integer.MAX_VALUE);
    assertEquals(LOOPS, rows.size());

    // we don't allow cleanup when the local host has no range to avoid wipping up all data when a
    // node has not join the ring.
    // So to make sure cleanup erase everything here, we give the localhost the tiniest possible
    // range.
    TokenMetadata tmd = StorageService.instance.getTokenMetadata();
    byte[] tk1 = new byte[1], tk2 = new byte[1];
    tk1[0] = 2;
    tk2[0] = 1;
    tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1"));
    tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2"));

    CompactionManager.instance.performCleanup(cfs, new CounterId.OneShotRenewer());

    // row data should be gone
    rows = Util.getRangeSlice(cfs);
    assertEquals(0, rows.size());

    // not only should it be gone but there should be no data on disk, not even tombstones
    assert cfs.getSSTables().isEmpty();

    // 2ary indexes should result in no results, too (although tombstones won't be gone until
    // compacted)
    rows = cfs.search(range, clause, filter, Integer.MAX_VALUE);
    assertEquals(0, rows.size());
  }
Exemplo n.º 13
0
  private void testDontPurgeAccidentaly(String k, String cfname)
      throws IOException, ExecutionException, InterruptedException {
    // This test catches the regression of CASSANDRA-2786
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);

    // disable compaction while flushing
    cfs.clearUnsafe();
    cfs.disableAutoCompaction();

    // Add test row
    DecoratedKey key = Util.dk(k);
    RowMutation rm = new RowMutation(KEYSPACE1, key.key);
    rm.add(
        cfname,
        CompositeType.build(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")),
        ByteBufferUtil.EMPTY_BYTE_BUFFER,
        0);
    rm.apply();

    cfs.forceBlockingFlush();

    Collection<SSTableReader> sstablesBefore = cfs.getSSTables();

    QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis());
    assert !(cfs.getColumnFamily(filter).getColumnCount() == 0);

    // Remove key
    rm = new RowMutation(KEYSPACE1, key.key);
    rm.delete(cfname, 2);
    rm.apply();

    ColumnFamily cf = cfs.getColumnFamily(filter);
    assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;

    // Sleep one second so that the removal is indeed purgeable even with gcgrace == 0
    Thread.sleep(1000);

    cfs.forceBlockingFlush();

    Collection<SSTableReader> sstablesAfter = cfs.getSSTables();
    Collection<SSTableReader> toCompact = new ArrayList<SSTableReader>();
    for (SSTableReader sstable : sstablesAfter)
      if (!sstablesBefore.contains(sstable)) toCompact.add(sstable);

    Util.compact(cfs, toCompact);

    cf = cfs.getColumnFamily(filter);
    assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;
  }
  @Test(timeout = 5000)
  public void testTruncateHints() throws Exception {
    Keyspace systemKeyspace = Keyspace.open("system");
    ColumnFamilyStore hintStore = systemKeyspace.getColumnFamilyStore(SystemKeyspace.HINTS);
    hintStore.clearUnsafe();

    // insert 1 hint
    Mutation rm = new Mutation(KEYSPACE4, ByteBufferUtil.bytes(1));
    rm.add(
        STANDARD1_CF,
        Util.cellname(COLUMN1),
        ByteBufferUtil.EMPTY_BYTE_BUFFER,
        System.currentTimeMillis());

    HintedHandOffManager.instance
        .hintFor(
            rm,
            System.currentTimeMillis(),
            HintedHandOffManager.calculateHintTTL(rm),
            UUID.randomUUID())
        .applyUnsafe();

    assert getNoOfHints() == 1;

    HintedHandOffManager.instance.truncateAllHints();

    while (getNoOfHints() > 0) {
      Thread.sleep(100);
    }

    assert getNoOfHints() == 0;
  }
  /**
   * Tests that the system.peers table is not updated after a node has been removed. (See
   * CASSANDRA-6053)
   */
  @Test
  public void testStateChangeOnRemovedNode() throws UnknownHostException {
    StorageService ss = StorageService.instance;
    VersionedValue.VersionedValueFactory valueFactory =
        new VersionedValue.VersionedValueFactory(partitioner);

    // create a ring of 2 nodes
    ArrayList<Token> endpointTokens = new ArrayList<>();
    List<InetAddress> hosts = new ArrayList<>();
    Util.createInitialRing(
        ss, partitioner, endpointTokens, new ArrayList<Token>(), hosts, new ArrayList<UUID>(), 2);

    InetAddress toRemove = hosts.get(1);
    SystemKeyspace.updatePeerInfo(toRemove, "data_center", "dc42");
    SystemKeyspace.updatePeerInfo(toRemove, "rack", "rack42");
    assertEquals("rack42", SystemKeyspace.loadDcRackInfo().get(toRemove).get("rack"));

    // mark the node as removed
    Gossiper.instance.injectApplicationState(
        toRemove,
        ApplicationState.STATUS,
        valueFactory.left(
            Collections.singleton(endpointTokens.get(1)), Gossiper.computeExpireTime()));
    assertTrue(
        Gossiper.instance.isDeadState(Gossiper.instance.getEndpointStateForEndpoint(hosts.get(1))));

    // state changes made after the endpoint has left should be ignored
    ss.onChange(hosts.get(1), ApplicationState.RACK, valueFactory.rack("rack9999"));
    assertEquals("rack42", SystemKeyspace.loadDcRackInfo().get(toRemove).get("rack"));
  }
Exemplo n.º 16
0
 private long populate(String ks, String cf, int startRowKey, int endRowKey, int ttl) {
   long timestamp = System.currentTimeMillis();
   for (int i = startRowKey; i <= endRowKey; i++) {
     DecoratedKey key = Util.dk(Integer.toString(i));
     Mutation rm = new Mutation(ks, key.getKey());
     for (int j = 0; j < 10; j++)
       rm.add(
           cf,
           Util.cellname(Integer.toString(j)),
           ByteBufferUtil.EMPTY_BYTE_BUFFER,
           timestamp,
           j > 0
               ? ttl
               : 0); // let first column never expire, since deleting all columns does not produce
                     // sstable
     rm.applyUnsafe();
   }
   return timestamp;
 }
  @Test
  public void testNoExpire() throws ExecutionException, InterruptedException {
    ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
    cfs.disableAutoCompaction();
    cfs.metadata.gcGraceSeconds(0);
    long timestamp = System.currentTimeMillis();
    RowMutation rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col7"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);

    rm.apply();
    cfs.forceBlockingFlush();

    rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.apply();
    cfs.forceBlockingFlush();
    rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col3"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.apply();
    cfs.forceBlockingFlush();
    DecoratedKey noTTLKey = Util.dk("nottl");
    rm = new RowMutation("Keyspace1", noTTLKey.key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col311"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp);
    rm.apply();
    cfs.forceBlockingFlush();
    Thread.sleep(2000); // wait for ttl to expire
    assertEquals(4, cfs.getSSTables().size());
    cfs.enableAutoCompaction(true);
    assertEquals(1, cfs.getSSTables().size());
    SSTableReader sstable = cfs.getSSTables().iterator().next();
    SSTableScanner scanner = sstable.getScanner(DataRange.allData(sstable.partitioner));
    assertTrue(scanner.hasNext());
    while (scanner.hasNext()) {
      OnDiskAtomIterator iter = scanner.next();
      assertEquals(noTTLKey, iter.getKey());
    }
  }
Exemplo n.º 18
0
  /*
   * This excercise in particular the code of #4142
   */
  @Test
  public void testValidationMultipleSSTablePerLevel() throws Exception {
    String ksname = "Keyspace1";
    String cfname = "StandardLeveled";
    Table table = Table.open(ksname);
    ColumnFamilyStore store = table.getColumnFamilyStore(cfname);

    ByteBuffer value =
        ByteBuffer.wrap(new byte[100 * 1024]); // 100 KB value, make it easy to have multiple files

    // Enough data to have a level 1 and 2
    int rows = 20;
    int columns = 10;

    // Adds enough data to trigger multiple sstable per level
    for (int r = 0; r < rows; r++) {
      DecoratedKey key = Util.dk(String.valueOf(r));
      RowMutation rm = new RowMutation(ksname, key.key);
      for (int c = 0; c < columns; c++) {
        rm.add(new QueryPath(cfname, null, ByteBufferUtil.bytes("column" + c)), value, 0);
      }
      rm.apply();
      store.forceFlush();
    }

    LeveledCompactionStrategy strat = (LeveledCompactionStrategy) store.getCompactionStrategy();

    while (strat.getLevelSize(0) > 0) {
      store.forceMajorCompaction();
      Thread.sleep(200);
    }
    // Checking we're not completely bad at math
    assert strat.getLevelSize(1) > 0;
    assert strat.getLevelSize(2) > 0;

    AntiEntropyService.CFPair p = new AntiEntropyService.CFPair(ksname, cfname);
    Range<Token> range = new Range<Token>(Util.token(""), Util.token(""));
    AntiEntropyService.TreeRequest req =
        new AntiEntropyService.TreeRequest("1", FBUtilities.getLocalAddress(), range, p);
    AntiEntropyService.Validator validator = new AntiEntropyService.Validator(req);
    CompactionManager.instance.submitValidation(store, validator).get();
  }
Exemplo n.º 19
0
  @Test
  public void testSuperColumnTombstones() throws ExecutionException, InterruptedException {
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1");
    cfs.disableAutoCompaction();

    DecoratedKey key = Util.dk("tskey");
    ByteBuffer scName = ByteBufferUtil.bytes("TestSuperColumn");

    // a subcolumn
    Mutation rm = new Mutation(KEYSPACE1, key.getKey());
    rm.add(
        "Super1",
        Util.cellname(scName, ByteBufferUtil.bytes(0)),
        ByteBufferUtil.EMPTY_BYTE_BUFFER,
        FBUtilities.timestampMicros());
    rm.applyUnsafe();
    cfs.forceBlockingFlush();

    // shadow the subcolumn with a supercolumn tombstone
    rm = new Mutation(KEYSPACE1, key.getKey());
    rm.deleteRange(
        "Super1",
        SuperColumns.startOf(scName),
        SuperColumns.endOf(scName),
        FBUtilities.timestampMicros());
    rm.applyUnsafe();
    cfs.forceBlockingFlush();

    CompactionManager.instance.performMaximal(cfs);
    assertEquals(1, cfs.getSSTables().size());

    // check that the shadowed column is gone
    SSTableReader sstable = cfs.getSSTables().iterator().next();
    Range keyRange =
        new Range<RowPosition>(key, sstable.partitioner.getMinimumToken().maxKeyBound());
    SSTableScanner scanner = sstable.getScanner(DataRange.forKeyRange(keyRange));
    OnDiskAtomIterator iter = scanner.next();
    assertEquals(key, iter.getKey());
    assertTrue(iter.next() instanceof RangeTombstone);
    assertFalse(iter.hasNext());
  }
  @Test
  public void testStateJumpToLeft() throws UnknownHostException {
    StorageService ss = StorageService.instance;
    TokenMetadata tmd = ss.getTokenMetadata();
    tmd.clearUnsafe();
    IPartitioner partitioner = RandomPartitioner.instance;
    VersionedValue.VersionedValueFactory valueFactory =
        new VersionedValue.VersionedValueFactory(partitioner);

    ArrayList<Token> endpointTokens = new ArrayList<Token>();
    ArrayList<Token> keyTokens = new ArrayList<Token>();
    List<InetAddress> hosts = new ArrayList<InetAddress>();
    List<UUID> hostIds = new ArrayList<UUID>();

    // create a ring of 6 nodes
    Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 7);

    // node hosts.get(2) goes jumps to left
    ss.onChange(
        hosts.get(2),
        ApplicationState.STATUS,
        valueFactory.left(
            Collections.singleton(endpointTokens.get(2)), Gossiper.computeExpireTime()));

    assertFalse(tmd.isMember(hosts.get(2)));

    // node hosts.get(4) goes to bootstrap
    Gossiper.instance.injectApplicationState(
        hosts.get(3),
        ApplicationState.TOKENS,
        valueFactory.tokens(Collections.singleton(keyTokens.get(1))));
    ss.onChange(
        hosts.get(3),
        ApplicationState.STATUS,
        valueFactory.bootstrapping(Collections.<Token>singleton(keyTokens.get(1))));

    assertFalse(tmd.isMember(hosts.get(3)));
    assertEquals(1, tmd.getBootstrapTokens().size());
    assertEquals(hosts.get(3), tmd.getBootstrapTokens().get(keyTokens.get(1)));

    // and then directly to 'left'
    Gossiper.instance.injectApplicationState(
        hosts.get(2),
        ApplicationState.TOKENS,
        valueFactory.tokens(Collections.singleton(keyTokens.get(1))));
    ss.onChange(
        hosts.get(2),
        ApplicationState.STATUS,
        valueFactory.left(Collections.singleton(keyTokens.get(1)), Gossiper.computeExpireTime()));

    assertTrue(tmd.getBootstrapTokens().size() == 0);
    assertFalse(tmd.isMember(hosts.get(2)));
    assertFalse(tmd.isLeaving(hosts.get(2)));
  }
Exemplo n.º 21
0
 private static void insertRowWithKey(int key) {
   long timestamp = System.currentTimeMillis();
   DecoratedKey decoratedKey = Util.dk(String.format("%03d", key));
   RowMutation rm = new RowMutation(KEYSPACE1, decoratedKey.key);
   rm.add(
       "Standard1",
       ByteBufferUtil.bytes("col"),
       ByteBufferUtil.EMPTY_BYTE_BUFFER,
       timestamp,
       1000);
   rm.apply();
 }
  @Test
  public void testSimpleExpire() throws ExecutionException, InterruptedException {
    ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
    cfs.disableAutoCompaction();
    cfs.metadata.gcGraceSeconds(0);
    long timestamp = System.currentTimeMillis();
    RowMutation rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col7"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);

    rm.apply();
    cfs.forceBlockingFlush();

    rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.apply();
    cfs.forceBlockingFlush();
    rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col3"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.apply();
    cfs.forceBlockingFlush();
    rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1",
        ByteBufferUtil.bytes("col311"),
        ByteBufferUtil.EMPTY_BYTE_BUFFER,
        timestamp,
        1);
    rm.apply();

    cfs.forceBlockingFlush();
    Thread.sleep(2000); // wait for ttl to expire
    assertEquals(4, cfs.getSSTables().size());
    cfs.enableAutoCompaction(true);
    assertEquals(0, cfs.getSSTables().size());
  }
Exemplo n.º 23
0
  @Test
  public void testUserDefinedCompaction() throws Exception {
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    final String cfname = "Standard3"; // use clean(no sstable) CF
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);

    // disable compaction while flushing
    cfs.disableAutoCompaction();

    final int ROWS_PER_SSTABLE = 10;
    for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
      DecoratedKey key = Util.dk(String.valueOf(i));
      Mutation rm = new Mutation(KEYSPACE1, key.getKey());
      rm.add(
          cfname,
          Util.cellname("col"),
          ByteBufferUtil.EMPTY_BYTE_BUFFER,
          System.currentTimeMillis());
      rm.applyUnsafe();
    }
    cfs.forceBlockingFlush();
    Collection<SSTableReader> sstables = cfs.getSSTables();

    assertEquals(1, sstables.size());
    SSTableReader sstable = sstables.iterator().next();

    int prevGeneration = sstable.descriptor.generation;
    String file = new File(sstable.descriptor.filenameFor(Component.DATA)).getAbsolutePath();
    // submit user defined compaction on flushed sstable
    CompactionManager.instance.forceUserDefinedCompaction(file);
    // wait until user defined compaction finishes
    do {
      Thread.sleep(100);
    } while (CompactionManager.instance.getPendingTasks() > 0
        || CompactionManager.instance.getActiveCompactions() > 0);
    // CF should have only one sstable with generation number advanced
    sstables = cfs.getSSTables();
    assertEquals(1, sstables.size());
    assertEquals(prevGeneration + 1, sstables.iterator().next().descriptor.generation);
  }
Exemplo n.º 24
0
  @Before
  public void setup() throws IOException, ConfigurationException {
    tmd.clearUnsafe();

    // create a ring of 5 nodes
    Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 6);

    MessagingService.instance().listen(FBUtilities.getBroadcastAddress());
    Gossiper.instance.start(1);
    removalhost = hosts.get(5);
    hosts.remove(removalhost);
    removalId = hostIds.get(5);
    hostIds.remove(removalId);
  }
  /** test get_count() to work correctly with 'count' settings around page size. (CASSANDRA-4833) */
  @Test
  public void test_get_count() throws Exception {
    Schema.instance.clear(); // Schema are now written on disk and will be reloaded
    new EmbeddedCassandraService().start();
    ThriftSessionManager.instance.setCurrentSocket(new InetSocketAddress(9160));

    DecoratedKey key = Util.dk("testkey");
    for (int i = 0; i < 3050; i++) {
      RowMutation rm = new RowMutation("Keyspace1", key.key);
      rm.add(
          new QueryPath("Standard1", null, ByteBufferUtil.bytes(String.valueOf(i))),
          ByteBufferUtil.EMPTY_BYTE_BUFFER,
          System.currentTimeMillis());
      rm.apply();
    }

    CassandraServer server = new CassandraServer();
    server.set_keyspace("Keyspace1");

    // same as page size
    int count =
        server.get_count(
            key.key, new ColumnParent("Standard1"), predicateWithCount(1024), ConsistencyLevel.ONE);
    assert count == 1024 : "expected 1024 but was " + count;

    // 1 above page size
    count =
        server.get_count(
            key.key, new ColumnParent("Standard1"), predicateWithCount(1025), ConsistencyLevel.ONE);
    assert count == 1025 : "expected 1025 but was " + count;

    // above number of columns
    count =
        server.get_count(
            key.key, new ColumnParent("Standard1"), predicateWithCount(4000), ConsistencyLevel.ONE);
    assert count == 3050 : "expected 3050 but was " + count;

    // same as number of columns
    count =
        server.get_count(
            key.key, new ColumnParent("Standard1"), predicateWithCount(3050), ConsistencyLevel.ONE);
    assert count == 3050 : "expected 3050 but was " + count;

    // 1 above number of columns
    count =
        server.get_count(
            key.key, new ColumnParent("Standard1"), predicateWithCount(3051), ConsistencyLevel.ONE);
    assert count == 3050 : "expected 3050 but was " + count;
  }
Exemplo n.º 26
0
  public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception {
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
    store.clearUnsafe();
    store.metadata.gcGraceSeconds(1);
    store.setCompactionStrategyClass(strategyClassName);

    // disable compaction while flushing
    store.disableAutoCompaction();

    long timestamp = System.currentTimeMillis();
    for (int i = 0; i < 10; i++) {
      DecoratedKey key = Util.dk(Integer.toString(i));
      RowMutation rm = new RowMutation(KEYSPACE1, key.key);
      for (int j = 0; j < 10; j++)
        rm.add(
            "Standard1",
            ByteBufferUtil.bytes(Integer.toString(j)),
            ByteBufferUtil.EMPTY_BYTE_BUFFER,
            timestamp,
            j > 0
                ? 3
                : 0); // let first column never expire, since deleting all columns does not produce
                      // sstable
      rm.apply();
    }
    store.forceBlockingFlush();
    assertEquals(1, store.getSSTables().size());
    long originalSize = store.getSSTables().iterator().next().uncompressedLength();

    // wait enough to force single compaction
    TimeUnit.SECONDS.sleep(5);

    // enable compaction, submit background and wait for it to complete
    store.enableAutoCompaction();
    FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
    while (CompactionManager.instance.getPendingTasks() > 0
        || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);

    // and sstable with ttl should be compacted
    assertEquals(1, store.getSSTables().size());
    long size = store.getSSTables().iterator().next().uncompressedLength();
    assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize);

    // make sure max timestamp of compacted sstables is recorded properly after compaction.
    assertMaxTimestamp(store, timestamp);

    return store;
  }
Exemplo n.º 27
0
  protected void fillCF(ColumnFamilyStore cfs, int rowsPerSSTable) {
    CompactionManager.instance.disableAutoCompaction();

    for (int i = 0; i < rowsPerSSTable; i++) {
      String key = String.valueOf(i);
      // create a row and update the birthdate value, test that the index query fetches the new
      // version
      Mutation rm;
      rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes(key));
      rm.add(cfs.name, Util.cellname(COLUMN), VALUE, System.currentTimeMillis());
      rm.applyUnsafe();
    }

    cfs.forceBlockingFlush();
  }
  @Test
  public void testAggressiveFullyExpired() {
    String KEYSPACE1 = "Keyspace1";
    ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
    cfs.disableAutoCompaction();
    cfs.metadata.gcGraceSeconds(0);

    DecoratedKey ttlKey = Util.dk("ttl");
    RowMutation rm = new RowMutation("Keyspace1", ttlKey.key);
    rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 1, 1);
    rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 3, 1);
    rm.applyUnsafe();
    cfs.forceBlockingFlush();

    rm = new RowMutation(KEYSPACE1, ttlKey.key);
    rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 2, 1);
    rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 5, 1);
    rm.applyUnsafe();
    cfs.forceBlockingFlush();

    rm = new RowMutation(KEYSPACE1, ttlKey.key);
    rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 4, 1);
    rm.add("Standard1", ByteBufferUtil.bytes("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 7, 1);
    rm.applyUnsafe();
    cfs.forceBlockingFlush();

    rm = new RowMutation(KEYSPACE1, ttlKey.key);
    rm.add("Standard1", ByteBufferUtil.bytes("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 6, 3);
    rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 8, 1);
    rm.applyUnsafe();
    cfs.forceBlockingFlush();

    Set<SSTableReader> sstables = Sets.newHashSet(cfs.getSSTables());
    int now = (int) (System.currentTimeMillis() / 1000);
    int gcBefore = now + 2;
    Set<SSTableReader> expired =
        CompactionController.getFullyExpiredSSTables(
            cfs, sstables, Collections.EMPTY_SET, gcBefore);
    assertEquals(2, expired.size());

    cfs.clearUnsafe();
  }
Exemplo n.º 29
0
  protected void testCompaction(int sstableCount, int rowsPerSSTable, int colsPerRow)
      throws Exception {
    CompactionManager.instance.disableAutoCompaction();

    Table table = Table.open(TABLE1);
    ColumnFamilyStore store = table.getColumnFamilyStore("Standard1");

    ArrayList<SSTableReader> sstables = new ArrayList<SSTableReader>();
    for (int k = 0; k < sstableCount; k++) {
      SortedMap<String, ColumnFamily> rows = new TreeMap<String, ColumnFamily>();
      for (int j = 0; j < rowsPerSSTable; j++) {
        String key = String.valueOf(j);
        IColumn[] cols = new IColumn[colsPerRow];
        for (int i = 0; i < colsPerRow; i++) {
          // last sstable has highest timestamps
          cols[i] = Util.column(String.valueOf(i), String.valueOf(i), k);
        }
        rows.put(key, SSTableUtils.createCF(Long.MIN_VALUE, Integer.MIN_VALUE, cols));
      }
      SSTableReader sstable = SSTableUtils.writeSSTable(rows);
      sstables.add(sstable);
      store.addSSTable(sstable);
    }

    // give garbage collection a bit of time to catch up
    Thread.sleep(1000);

    long start = System.currentTimeMillis();
    CompactionManager.instance.doCompaction(
        store,
        sstables,
        (int) (System.currentTimeMillis() / 1000)
            - DatabaseDescriptor.getCFMetaData(TABLE1, "Standard1").gcGraceSeconds);
    System.out.println(
        String.format(
            "%s: sstables=%d rowsper=%d colsper=%d: %d ms",
            this.getClass().getName(),
            sstableCount,
            rowsPerSSTable,
            colsPerRow,
            System.currentTimeMillis() - start));
  }
  @Test
  public void testRemovingStatusForNonMember() throws UnknownHostException {
    // create a ring of 1 node
    StorageService ss = StorageService.instance;
    VersionedValue.VersionedValueFactory valueFactory =
        new VersionedValue.VersionedValueFactory(partitioner);
    Util.createInitialRing(
        ss,
        partitioner,
        new ArrayList<Token>(),
        new ArrayList<Token>(),
        new ArrayList<InetAddress>(),
        new ArrayList<UUID>(),
        1);

    // make a REMOVING state change on a non-member endpoint; without the CASSANDRA-6564 fix, this
    // would result in an ArrayIndexOutOfBoundsException
    ss.onChange(
        InetAddress.getByName("192.168.1.42"),
        ApplicationState.STATUS,
        valueFactory.removingNonlocal(UUID.randomUUID()));
  }