コード例 #1
1
    public void prepare(ColumnFamilyStore cfs) {
      if (tree.partitioner() instanceof RandomPartitioner) {
        // You can't beat an even tree distribution for md5
        tree.init();
      } else {
        List<DecoratedKey> keys = new ArrayList<DecoratedKey>();
        for (DecoratedKey sample : cfs.keySamples(request.range)) {
          assert request.range.contains(sample.token)
              : "Token " + sample.token + " is not within range " + request.range;
          keys.add(sample);
        }

        if (keys.isEmpty()) {
          // use an even tree distribution
          tree.init();
        } else {
          int numkeys = keys.size();
          Random random = new Random();
          // sample the column family using random keys from the index
          while (true) {
            DecoratedKey dk = keys.get(random.nextInt(numkeys));
            if (!tree.split(dk.token)) break;
          }
        }
      }
      logger.debug("Prepared AEService tree of size " + tree.size() + " for " + request);
      ranges = tree.invalids();
    }
コード例 #2
0
  public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception {
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARD1);
    store.clearUnsafe();
    store.metadata.gcGraceSeconds(1);
    store.setCompactionStrategyClass(strategyClassName);

    // disable compaction while flushing
    store.disableAutoCompaction();

    long timestamp = populate(KEYSPACE1, CF_STANDARD1, 0, 9, 3); // ttl=3s

    store.forceBlockingFlush();
    assertEquals(1, store.getSSTables().size());
    long originalSize = store.getSSTables().iterator().next().uncompressedLength();

    // wait enough to force single compaction
    TimeUnit.SECONDS.sleep(5);

    // enable compaction, submit background and wait for it to complete
    store.enableAutoCompaction();
    FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
    while (CompactionManager.instance.getPendingTasks() > 0
        || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);

    // and sstable with ttl should be compacted
    assertEquals(1, store.getSSTables().size());
    long size = store.getSSTables().iterator().next().uncompressedLength();
    assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize);

    // make sure max timestamp of compacted sstables is recorded properly after compaction.
    assertMaxTimestamp(store, timestamp);

    return store;
  }
コード例 #3
0
  @Test
  public void testCompactionLog() throws Exception {
    SystemKeyspace.discardCompactionsInProgress();

    String cf = "Standard4";
    ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(cf);
    SchemaLoader.insertData(KEYSPACE1, cf, 0, 1);
    cfs.forceBlockingFlush();

    Collection<SSTableReader> sstables = cfs.getSSTables();
    assertFalse(sstables.isEmpty());
    Set<Integer> generations =
        Sets.newHashSet(
            Iterables.transform(
                sstables,
                new Function<SSTableReader, Integer>() {
                  public Integer apply(SSTableReader sstable) {
                    return sstable.descriptor.generation;
                  }
                }));
    UUID taskId = SystemKeyspace.startCompaction(cfs, sstables);
    Map<Pair<String, String>, Map<Integer, UUID>> compactionLogs =
        SystemKeyspace.getUnfinishedCompactions();
    Set<Integer> unfinishedCompactions = compactionLogs.get(Pair.create(KEYSPACE1, cf)).keySet();
    assertTrue(unfinishedCompactions.containsAll(generations));

    SystemKeyspace.finishCompaction(taskId);
    compactionLogs = SystemKeyspace.getUnfinishedCompactions();
    assertFalse(compactionLogs.containsKey(Pair.create(KEYSPACE1, cf)));
  }
コード例 #4
0
 @Test
 public void testSingleSSTableCompactionWithLeveledCompaction() throws Exception {
   ColumnFamilyStore store =
       testSingleSSTableCompaction(LeveledCompactionStrategy.class.getCanonicalName());
   LeveledCompactionStrategy strategy = (LeveledCompactionStrategy) store.getCompactionStrategy();
   // tombstone removal compaction should not promote level
   assertEquals(1, strategy.getLevelSize(0));
 }
コード例 #5
0
ファイル: CompactionsTest.java プロジェクト: placrosse/ACaZoo
  @Test
  public void testEchoedRow() throws IOException, ExecutionException, InterruptedException {
    // This test check that EchoedRow doesn't skipp rows: see CASSANDRA-2653

    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard2");

    // disable compaction while flushing
    cfs.disableAutoCompaction();

    // Insert 4 keys in two sstables. We need the sstables to have 2 rows
    // at least to trigger what was causing CASSANDRA-2653
    for (int i = 1; i < 5; i++) {
      DecoratedKey key = Util.dk(String.valueOf(i));
      RowMutation rm = new RowMutation(KEYSPACE1, key.key);
      rm.add(
          "Standard2",
          ByteBufferUtil.bytes(String.valueOf(i)),
          ByteBufferUtil.EMPTY_BYTE_BUFFER,
          i);
      rm.apply();

      if (i % 2 == 0) cfs.forceBlockingFlush();
    }
    Collection<SSTableReader> toCompact = cfs.getSSTables();
    assert toCompact.size() == 2;

    // Reinserting the same keys. We will compact only the previous sstable, but we need those new
    // ones
    // to make sure we use EchoedRow, otherwise it won't be used because purge can be done.
    for (int i = 1; i < 5; i++) {
      DecoratedKey key = Util.dk(String.valueOf(i));
      RowMutation rm = new RowMutation(KEYSPACE1, key.key);
      rm.add(
          "Standard2",
          ByteBufferUtil.bytes(String.valueOf(i)),
          ByteBufferUtil.EMPTY_BYTE_BUFFER,
          i);
      rm.apply();
    }
    cfs.forceBlockingFlush();
    SSTableReader tmpSSTable = null;
    for (SSTableReader sstable : cfs.getSSTables())
      if (!toCompact.contains(sstable)) tmpSSTable = sstable;
    assert tmpSSTable != null;

    // Force compaction on first sstables. Since each row is in only one sstable, we will be using
    // EchoedRow.
    Util.compact(cfs, toCompact);
    assertEquals(2, cfs.getSSTables().size());

    // Now, we remove the sstable that was just created to force the use of EchoedRow (so that it
    // doesn't hide the problem)
    cfs.markObsolete(Collections.singleton(tmpSSTable), OperationType.UNKNOWN);
    assertEquals(1, cfs.getSSTables().size());

    // Now assert we do have the 4 keys
    assertEquals(4, Util.getRangeSlice(cfs).size());
  }
コード例 #6
0
ファイル: Util.java プロジェクト: hobinyoon/mutants-cassandra
  /**
   * Writes out a bunch of mutations for a single column family.
   *
   * @param mutations A group of Mutations for the same keyspace and column family.
   * @return The ColumnFamilyStore that was used.
   */
  public static ColumnFamilyStore writeColumnFamily(List<Mutation> mutations) {
    IMutation first = mutations.get(0);
    String keyspaceName = first.getKeyspaceName();
    UUID cfid = first.getColumnFamilyIds().iterator().next();

    for (Mutation rm : mutations) rm.applyUnsafe();

    ColumnFamilyStore store = Keyspace.open(keyspaceName).getColumnFamilyStore(cfid);
    store.forceBlockingFlush();
    return store;
  }
コード例 #7
0
ファイル: Util.java プロジェクト: QStephen/Cassandra-Research
  /**
   * Writes out a bunch of rows for a single column family.
   *
   * @param rows A group of RowMutations for the same table and column family.
   * @return The ColumnFamilyStore that was used.
   */
  public static ColumnFamilyStore writeColumnFamily(List<IMutation> rms)
      throws IOException, ExecutionException, InterruptedException {
    IMutation first = rms.get(0);
    String tablename = first.getTable();
    UUID cfid = first.getColumnFamilyIds().iterator().next();

    for (IMutation rm : rms) rm.apply();

    ColumnFamilyStore store = Table.open(tablename).getColumnFamilyStore(cfid);
    store.forceBlockingFlush();
    return store;
  }
コード例 #8
0
 public static SSTableReader sstable(
     int generation, int size, boolean keepRef, ColumnFamilyStore cfs) {
   Descriptor descriptor =
       new Descriptor(
           cfs.getDirectories().getDirectoryForNewSSTables(),
           cfs.keyspace.getName(),
           cfs.getColumnFamilyName(),
           generation);
   Set<Component> components =
       ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
   for (Component component : components) {
     File file = new File(descriptor.filenameFor(component));
     try {
       file.createNewFile();
     } catch (IOException e) {
     }
   }
   if (size > 0) {
     try {
       File file = new File(descriptor.filenameFor(Component.DATA));
       try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
         raf.setLength(size);
       }
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
   }
   SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.emptyList());
   StatsMetadata metadata =
       (StatsMetadata)
           new MetadataCollector(cfs.metadata.comparator)
               .finalizeMetadata(
                   cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
               .get(MetadataType.STATS);
   SSTableReader reader =
       SSTableReader.internalOpen(
           descriptor,
           components,
           cfs.metadata,
           segmentedFile.sharedCopy(),
           segmentedFile.sharedCopy(),
           indexSummary.sharedCopy(),
           new AlwaysPresentFilter(),
           1L,
           metadata,
           SSTableReader.OpenReason.NORMAL,
           header);
   reader.first = reader.last = readerBounds(generation);
   if (!keepRef) reader.selfRef().release();
   return reader;
 }
コード例 #9
0
ファイル: CompactionsTest.java プロジェクト: placrosse/ACaZoo
  private void testDontPurgeAccidentaly(String k, String cfname)
      throws IOException, ExecutionException, InterruptedException {
    // This test catches the regression of CASSANDRA-2786
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);

    // disable compaction while flushing
    cfs.clearUnsafe();
    cfs.disableAutoCompaction();

    // Add test row
    DecoratedKey key = Util.dk(k);
    RowMutation rm = new RowMutation(KEYSPACE1, key.key);
    rm.add(
        cfname,
        CompositeType.build(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")),
        ByteBufferUtil.EMPTY_BYTE_BUFFER,
        0);
    rm.apply();

    cfs.forceBlockingFlush();

    Collection<SSTableReader> sstablesBefore = cfs.getSSTables();

    QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis());
    assert !(cfs.getColumnFamily(filter).getColumnCount() == 0);

    // Remove key
    rm = new RowMutation(KEYSPACE1, key.key);
    rm.delete(cfname, 2);
    rm.apply();

    ColumnFamily cf = cfs.getColumnFamily(filter);
    assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;

    // Sleep one second so that the removal is indeed purgeable even with gcgrace == 0
    Thread.sleep(1000);

    cfs.forceBlockingFlush();

    Collection<SSTableReader> sstablesAfter = cfs.getSSTables();
    Collection<SSTableReader> toCompact = new ArrayList<SSTableReader>();
    for (SSTableReader sstable : sstablesAfter)
      if (!sstablesBefore.contains(sstable)) toCompact.add(sstable);

    Util.compact(cfs, toCompact);

    cf = cfs.getColumnFamily(filter);
    assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;
  }
コード例 #10
0
ファイル: CompactionsTest.java プロジェクト: placrosse/ACaZoo
  public ColumnFamilyStore testSingleSSTableCompaction(String strategyClassName) throws Exception {
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
    store.clearUnsafe();
    store.metadata.gcGraceSeconds(1);
    store.setCompactionStrategyClass(strategyClassName);

    // disable compaction while flushing
    store.disableAutoCompaction();

    long timestamp = System.currentTimeMillis();
    for (int i = 0; i < 10; i++) {
      DecoratedKey key = Util.dk(Integer.toString(i));
      RowMutation rm = new RowMutation(KEYSPACE1, key.key);
      for (int j = 0; j < 10; j++)
        rm.add(
            "Standard1",
            ByteBufferUtil.bytes(Integer.toString(j)),
            ByteBufferUtil.EMPTY_BYTE_BUFFER,
            timestamp,
            j > 0
                ? 3
                : 0); // let first column never expire, since deleting all columns does not produce
                      // sstable
      rm.apply();
    }
    store.forceBlockingFlush();
    assertEquals(1, store.getSSTables().size());
    long originalSize = store.getSSTables().iterator().next().uncompressedLength();

    // wait enough to force single compaction
    TimeUnit.SECONDS.sleep(5);

    // enable compaction, submit background and wait for it to complete
    store.enableAutoCompaction();
    FBUtilities.waitOnFutures(CompactionManager.instance.submitBackground(store));
    while (CompactionManager.instance.getPendingTasks() > 0
        || CompactionManager.instance.getActiveCompactions() > 0) TimeUnit.SECONDS.sleep(1);

    // and sstable with ttl should be compacted
    assertEquals(1, store.getSSTables().size());
    long size = store.getSSTables().iterator().next().uncompressedLength();
    assertTrue("should be less than " + originalSize + ", but was " + size, size < originalSize);

    // make sure max timestamp of compacted sstables is recorded properly after compaction.
    assertMaxTimestamp(store, timestamp);

    return store;
  }
コード例 #11
0
  @Test
  public void testCheckForExpiredSSTableBlockers() throws InterruptedException {
    String KEYSPACE1 = "Keyspace1";
    ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
    cfs.truncateBlocking();
    cfs.disableAutoCompaction();
    cfs.metadata.gcGraceSeconds(0);

    RowMutation rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
    rm.add(
        "Standard1",
        ByteBufferUtil.bytes("col1"),
        ByteBufferUtil.EMPTY_BYTE_BUFFER,
        System.currentTimeMillis());
    rm.applyUnsafe();
    cfs.forceBlockingFlush();
    SSTableReader blockingSSTable = cfs.getSSTables().iterator().next();
    for (int i = 0; i < 10; i++) {
      rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
      rm.delete("Standard1", System.currentTimeMillis());
      rm.applyUnsafe();
      cfs.forceBlockingFlush();
    }
    Multimap<SSTableReader, SSTableReader> blockers =
        SSTableExpiredBlockers.checkForExpiredSSTableBlockers(
            cfs.getSSTables(), (int) (System.currentTimeMillis() / 1000) + 100);
    assertEquals(1, blockers.keySet().size());
    assertTrue(blockers.keySet().contains(blockingSSTable));
    assertEquals(10, blockers.get(blockingSSTable).size());
  }
 public SizeTieredCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options) {
   super(cfs, options);
   this.estimatedRemainingTasks = 0;
   String optionValue = options.get(MIN_SSTABLE_SIZE_KEY);
   minSSTableSize = optionValue == null ? DEFAULT_MIN_SSTABLE_SIZE : Long.parseLong(optionValue);
   optionValue = options.get(BUCKET_LOW_KEY);
   bucketLow = optionValue == null ? DEFAULT_BUCKET_LOW : Double.parseDouble(optionValue);
   optionValue = options.get(BUCKET_HIGH_KEY);
   bucketHigh = optionValue == null ? DEFAULT_BUCKET_HIGH : Double.parseDouble(optionValue);
   if (bucketHigh <= bucketLow) {
     logger.warn(
         "Bucket low/high marks for {} incorrect, using defaults.", cfs.getColumnFamilyName());
     bucketLow = DEFAULT_BUCKET_LOW;
     bucketHigh = DEFAULT_BUCKET_HIGH;
   }
   cfs.setCompactionThresholds(
       cfs.metadata.getMinCompactionThreshold(), cfs.metadata.getMaxCompactionThreshold());
 }
コード例 #13
0
  @Test
  public void testNoExpire() throws ExecutionException, InterruptedException {
    ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
    cfs.disableAutoCompaction();
    cfs.metadata.gcGraceSeconds(0);
    long timestamp = System.currentTimeMillis();
    RowMutation rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col7"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);

    rm.apply();
    cfs.forceBlockingFlush();

    rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.apply();
    cfs.forceBlockingFlush();
    rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col3"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.apply();
    cfs.forceBlockingFlush();
    DecoratedKey noTTLKey = Util.dk("nottl");
    rm = new RowMutation("Keyspace1", noTTLKey.key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col311"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp);
    rm.apply();
    cfs.forceBlockingFlush();
    Thread.sleep(2000); // wait for ttl to expire
    assertEquals(4, cfs.getSSTables().size());
    cfs.enableAutoCompaction(true);
    assertEquals(1, cfs.getSSTables().size());
    SSTableReader sstable = cfs.getSSTables().iterator().next();
    SSTableScanner scanner = sstable.getScanner(DataRange.allData(sstable.partitioner));
    assertTrue(scanner.hasNext());
    while (scanner.hasNext()) {
      OnDiskAtomIterator iter = scanner.next();
      assertEquals(noTTLKey, iter.getKey());
    }
  }
コード例 #14
0
ファイル: Util.java プロジェクト: QStephen/Cassandra-Research
 public static List<Row> getRangeSlice(ColumnFamilyStore cfs, ByteBuffer superColumn)
     throws IOException, ExecutionException, InterruptedException {
   Token min = StorageService.getPartitioner().getMinimumToken();
   return cfs.getRangeSlice(
       superColumn,
       new Bounds<Token>(min, min).toRowBounds(),
       10000,
       new IdentityQueryFilter(),
       null);
 }
コード例 #15
0
  /*
   * This excercise in particular the code of #4142
   */
  @Test
  public void testValidationMultipleSSTablePerLevel() throws Exception {
    String ksname = "Keyspace1";
    String cfname = "StandardLeveled";
    Table table = Table.open(ksname);
    ColumnFamilyStore store = table.getColumnFamilyStore(cfname);

    ByteBuffer value =
        ByteBuffer.wrap(new byte[100 * 1024]); // 100 KB value, make it easy to have multiple files

    // Enough data to have a level 1 and 2
    int rows = 20;
    int columns = 10;

    // Adds enough data to trigger multiple sstable per level
    for (int r = 0; r < rows; r++) {
      DecoratedKey key = Util.dk(String.valueOf(r));
      RowMutation rm = new RowMutation(ksname, key.key);
      for (int c = 0; c < columns; c++) {
        rm.add(new QueryPath(cfname, null, ByteBufferUtil.bytes("column" + c)), value, 0);
      }
      rm.apply();
      store.forceFlush();
    }

    LeveledCompactionStrategy strat = (LeveledCompactionStrategy) store.getCompactionStrategy();

    while (strat.getLevelSize(0) > 0) {
      store.forceMajorCompaction();
      Thread.sleep(200);
    }
    // Checking we're not completely bad at math
    assert strat.getLevelSize(1) > 0;
    assert strat.getLevelSize(2) > 0;

    AntiEntropyService.CFPair p = new AntiEntropyService.CFPair(ksname, cfname);
    Range<Token> range = new Range<Token>(Util.token(""), Util.token(""));
    AntiEntropyService.TreeRequest req =
        new AntiEntropyService.TreeRequest("1", FBUtilities.getLocalAddress(), range, p);
    AntiEntropyService.Validator validator = new AntiEntropyService.Validator(req);
    CompactionManager.instance.submitValidation(store, validator).get();
  }
コード例 #16
0
 /**
  * Starts sending/receiving our list of differences to/from the remote endpoint: creates a
  * callback that will be called out of band once the streams complete.
  */
 void performStreamingRepair() throws IOException {
   logger.info(
       "Performing streaming repair of "
           + differences.size()
           + " ranges with "
           + remote
           + " for "
           + range);
   ColumnFamilyStore cfstore = Table.open(tablename).getColumnFamilyStore(cfname);
   try {
     Collection<SSTableReader> sstables = cfstore.getSSTables();
     Callback callback = new Callback();
     // send ranges to the remote node
     StreamOutSession outsession = StreamOutSession.create(tablename, remote, callback);
     StreamOut.transferSSTables(outsession, sstables, differences, OperationType.AES);
     // request ranges from the remote node
     StreamIn.requestRanges(remote, tablename, differences, callback, OperationType.AES);
   } catch (Exception e) {
     throw new IOException("Streaming repair failed.", e);
   }
 }
コード例 #17
0
  @Test
  public void testUserDefinedCompaction() throws Exception {
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    final String cfname = "Standard3"; // use clean(no sstable) CF
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);

    // disable compaction while flushing
    cfs.disableAutoCompaction();

    final int ROWS_PER_SSTABLE = 10;
    for (int i = 0; i < ROWS_PER_SSTABLE; i++) {
      DecoratedKey key = Util.dk(String.valueOf(i));
      Mutation rm = new Mutation(KEYSPACE1, key.getKey());
      rm.add(
          cfname,
          Util.cellname("col"),
          ByteBufferUtil.EMPTY_BYTE_BUFFER,
          System.currentTimeMillis());
      rm.applyUnsafe();
    }
    cfs.forceBlockingFlush();
    Collection<SSTableReader> sstables = cfs.getSSTables();

    assertEquals(1, sstables.size());
    SSTableReader sstable = sstables.iterator().next();

    int prevGeneration = sstable.descriptor.generation;
    String file = new File(sstable.descriptor.filenameFor(Component.DATA)).getAbsolutePath();
    // submit user defined compaction on flushed sstable
    CompactionManager.instance.forceUserDefinedCompaction(file);
    // wait until user defined compaction finishes
    do {
      Thread.sleep(100);
    } while (CompactionManager.instance.getPendingTasks() > 0
        || CompactionManager.instance.getActiveCompactions() > 0);
    // CF should have only one sstable with generation number advanced
    sstables = cfs.getSSTables();
    assertEquals(1, sstables.size());
    assertEquals(prevGeneration + 1, sstables.iterator().next().descriptor.generation);
  }
コード例 #18
0
ファイル: Util.java プロジェクト: hobinyoon/mutants-cassandra
  public static List<Row> getRangeSlice(ColumnFamilyStore cfs, ByteBuffer superColumn) {
    IDiskAtomFilter filter =
        superColumn == null
            ? new IdentityQueryFilter()
            : new SliceQueryFilter(
                SuperColumns.startOf(superColumn),
                SuperColumns.endOf(superColumn),
                false,
                Integer.MAX_VALUE);

    Token min = StorageService.getPartitioner().getMinimumToken();
    return cfs.getRangeSlice(Bounds.makeRowBounds(min, min), null, filter, 10000);
  }
コード例 #19
0
  public void applyModels() throws IOException {
    ColumnFamilyStore cfs = Table.open(tableName).getColumnFamilyStore(cfName);

    // reinitialize the table.
    KSMetaData existing = DatabaseDescriptor.getTableDefinition(tableName);
    CFMetaData cfm = existing.cfMetaData().get(cfName);
    KSMetaData ksm = makeNewKeyspaceDefinition(existing);
    CFMetaData.purge(cfm);
    DatabaseDescriptor.setTableDefinition(ksm, newVersion);

    if (!clientMode) {
      cfs.snapshot(Table.getTimestampedSnapshotName(null));

      CompactionManager.instance.getCompactionLock().lock();
      cfs.flushLock.lock();
      try {
        Table.open(ksm.name).dropCf(cfm.cfId);
      } finally {
        cfs.flushLock.unlock();
        CompactionManager.instance.getCompactionLock().unlock();
      }
    }
  }
コード例 #20
0
 public CommitLogReplayer() {
   this.tablesRecovered = new NonBlockingHashSet<Table>();
   this.futures = new ArrayList<Future<?>>();
   this.buffer = new byte[4096];
   this.invalidMutations = new HashMap<Integer, AtomicInteger>();
   // count the number of replayed mutation. We don't really care about atomicity, but we need it
   // to be a reference.
   this.replayedCount = new AtomicInteger();
   // compute per-CF and global replay positions
   this.cfPositions = new HashMap<Integer, ReplayPosition>();
   for (ColumnFamilyStore cfs : ColumnFamilyStore.all()) {
     // it's important to call RP.gRP per-cf, before aggregating all the positions w/ the
     // Ordering.min call
     // below: gRP will return NONE if there are no flushed sstables, which is important to have in
     // the
     // list (otherwise we'll just start replay from the first flush position that we do have,
     // which is not correct).
     ReplayPosition rp = ReplayPosition.getReplayPosition(cfs.getSSTables());
     cfPositions.put(cfs.metadata.cfId, rp);
   }
   this.globalPosition = Ordering.from(ReplayPosition.comparator).min(cfPositions.values());
   this.checksum = new PureJavaCrc32();
 }
コード例 #21
0
 /** Gets the estimated total amount of data to write during compaction */
 private static long getTotalWriteSize(
     Iterable<SSTableReader> nonExpiredSSTables,
     long estimatedTotalKeys,
     ColumnFamilyStore cfs,
     OperationType compactionType) {
   long estimatedKeysBeforeCompaction = 0;
   for (SSTableReader sstable : nonExpiredSSTables)
     estimatedKeysBeforeCompaction += sstable.estimatedKeys();
   estimatedKeysBeforeCompaction = Math.max(1, estimatedKeysBeforeCompaction);
   double estimatedCompactionRatio = (double) estimatedTotalKeys / estimatedKeysBeforeCompaction;
   return Math.round(
       estimatedCompactionRatio
           * cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType));
 }
コード例 #22
0
  private static SSTableReader sstable(
      File dataFolder, ColumnFamilyStore cfs, int generation, int size) throws IOException {
    Descriptor descriptor =
        new Descriptor(dataFolder, cfs.keyspace.getName(), cfs.getTableName(), generation);
    Set<Component> components =
        ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
    for (Component component : components) {
      File file = new File(descriptor.filenameFor(component));
      if (!file.exists()) assertTrue(file.createNewFile());
      try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
        raf.setLength(size);
      }
    }

    SegmentedFile dFile =
        new BufferedSegmentedFile(
            new ChannelProxy(new File(descriptor.filenameFor(Component.DATA))),
            RandomAccessReader.DEFAULT_BUFFER_SIZE,
            0);
    SegmentedFile iFile =
        new BufferedSegmentedFile(
            new ChannelProxy(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))),
            RandomAccessReader.DEFAULT_BUFFER_SIZE,
            0);

    SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.emptyList());
    StatsMetadata metadata =
        (StatsMetadata)
            new MetadataCollector(cfs.metadata.comparator)
                .finalizeMetadata(
                    cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
                .get(MetadataType.STATS);
    SSTableReader reader =
        SSTableReader.internalOpen(
            descriptor,
            components,
            cfs.metadata,
            dFile,
            iFile,
            MockSchema.indexSummary.sharedCopy(),
            new AlwaysPresentFilter(),
            1L,
            metadata,
            SSTableReader.OpenReason.NORMAL,
            header);
    reader.first = reader.last = MockSchema.readerBounds(generation);
    return reader;
  }
コード例 #23
0
  @Test
  public void testSimpleExpire() throws ExecutionException, InterruptedException {
    ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
    cfs.disableAutoCompaction();
    cfs.metadata.gcGraceSeconds(0);
    long timestamp = System.currentTimeMillis();
    RowMutation rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col7"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);

    rm.apply();
    cfs.forceBlockingFlush();

    rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.apply();
    cfs.forceBlockingFlush();
    rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1", ByteBufferUtil.bytes("col3"), ByteBufferUtil.EMPTY_BYTE_BUFFER, timestamp, 1);
    rm.apply();
    cfs.forceBlockingFlush();
    rm = new RowMutation("Keyspace1", Util.dk("ttl").key);
    rm.add(
        "Standard1",
        ByteBufferUtil.bytes("col311"),
        ByteBufferUtil.EMPTY_BYTE_BUFFER,
        timestamp,
        1);
    rm.apply();

    cfs.forceBlockingFlush();
    Thread.sleep(2000); // wait for ttl to expire
    assertEquals(4, cfs.getSSTables().size());
    cfs.enableAutoCompaction(true);
    assertEquals(0, cfs.getSSTables().size());
  }
コード例 #24
0
  @Test
  public void testAggressiveFullyExpired() {
    String KEYSPACE1 = "Keyspace1";
    ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
    cfs.disableAutoCompaction();
    cfs.metadata.gcGraceSeconds(0);

    DecoratedKey ttlKey = Util.dk("ttl");
    RowMutation rm = new RowMutation("Keyspace1", ttlKey.key);
    rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 1, 1);
    rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 3, 1);
    rm.applyUnsafe();
    cfs.forceBlockingFlush();

    rm = new RowMutation(KEYSPACE1, ttlKey.key);
    rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 2, 1);
    rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 5, 1);
    rm.applyUnsafe();
    cfs.forceBlockingFlush();

    rm = new RowMutation(KEYSPACE1, ttlKey.key);
    rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 4, 1);
    rm.add("Standard1", ByteBufferUtil.bytes("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 7, 1);
    rm.applyUnsafe();
    cfs.forceBlockingFlush();

    rm = new RowMutation(KEYSPACE1, ttlKey.key);
    rm.add("Standard1", ByteBufferUtil.bytes("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 6, 3);
    rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 8, 1);
    rm.applyUnsafe();
    cfs.forceBlockingFlush();

    Set<SSTableReader> sstables = Sets.newHashSet(cfs.getSSTables());
    int now = (int) (System.currentTimeMillis() / 1000);
    int gcBefore = now + 2;
    Set<SSTableReader> expired =
        CompactionController.getFullyExpiredSSTables(
            cfs, sstables, Collections.EMPTY_SET, gcBefore);
    assertEquals(2, expired.size());

    cfs.clearUnsafe();
  }
コード例 #25
0
ファイル: CompactionsTest.java プロジェクト: placrosse/ACaZoo
  @Test
  public void testSuperColumnTombstones()
      throws IOException, ExecutionException, InterruptedException {
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Super1");
    cfs.disableAutoCompaction();

    DecoratedKey key = Util.dk("tskey");
    ByteBuffer scName = ByteBufferUtil.bytes("TestSuperColumn");

    // a subcolumn
    RowMutation rm = new RowMutation(KEYSPACE1, key.key);
    rm.add(
        "Super1",
        CompositeType.build(scName, ByteBufferUtil.bytes(0)),
        ByteBufferUtil.EMPTY_BYTE_BUFFER,
        FBUtilities.timestampMicros());
    rm.apply();
    cfs.forceBlockingFlush();

    // shadow the subcolumn with a supercolumn tombstone
    rm = new RowMutation(KEYSPACE1, key.key);
    rm.deleteRange(
        "Super1",
        SuperColumns.startOf(scName),
        SuperColumns.endOf(scName),
        FBUtilities.timestampMicros());
    rm.apply();
    cfs.forceBlockingFlush();

    CompactionManager.instance.performMaximal(cfs);
    assertEquals(1, cfs.getSSTables().size());

    // check that the shadowed column is gone
    SSTableReader sstable = cfs.getSSTables().iterator().next();
    Range keyRange =
        new Range<RowPosition>(key, sstable.partitioner.getMinimumToken().maxKeyBound());
    SSTableScanner scanner = sstable.getScanner(DataRange.forKeyRange(keyRange));
    OnDiskAtomIterator iter = scanner.next();
    assertEquals(key, iter.getKey());
    assert iter.next() instanceof RangeTombstone;
    assert !iter.hasNext();
  }
コード例 #26
0
ファイル: Util.java プロジェクト: hobinyoon/mutants-cassandra
 public static QueryFilter namesQueryFilter(
     ColumnFamilyStore cfs, DecoratedKey key, CellName... names) {
   SortedSet<CellName> s = new TreeSet<CellName>(cfs.getComparator());
   for (CellName n : names) s.add(n);
   return QueryFilter.getNamesFilter(key, cfs.name, s, System.currentTimeMillis());
 }
コード例 #27
0
ファイル: Util.java プロジェクト: hobinyoon/mutants-cassandra
 public static NamesQueryFilter namesFilter(ColumnFamilyStore cfs, String... names) {
   SortedSet<CellName> s = new TreeSet<CellName>(cfs.getComparator());
   for (String str : names) s.add(cellname(str));
   return new NamesQueryFilter(s);
 }
コード例 #28
0
ファイル: BriskServer.java プロジェクト: thepaul/brisk
  /**
   * Retrieves a local subBlock
   *
   * @param blockId row key
   * @param sblockId SubBlock column name
   * @param offset inside the sblock
   * @return a local sublock
   * @throws TException
   */
  private LocalBlock getLocalSubBlock(
      String subBlockCFName, ByteBuffer blockId, ByteBuffer sblockId, int offset)
      throws TException {
    DecoratedKey<Token<?>> decoratedKey =
        new DecoratedKey<Token<?>>(StorageService.getPartitioner().getToken(blockId), blockId);

    Table table = Table.open(cfsKeyspace);
    ColumnFamilyStore sblockStore = table.getColumnFamilyStore(subBlockCFName);

    Collection<SSTableReader> sstables = sblockStore.getSSTables();

    for (SSTableReader sstable : sstables) {

      long position = sstable.getPosition(decoratedKey, Operator.EQ);

      if (position == -1) continue;

      String filename = sstable.descriptor.filenameFor(Component.DATA);
      RandomAccessFile raf = null;
      int mappedLength = -1;
      MappedByteBuffer mappedData = null;
      MappedFileDataInput file = null;
      try {
        raf = new RandomAccessFile(filename, "r");
        assert position < raf.length();

        mappedLength =
            (raf.length() - position) < Integer.MAX_VALUE
                ? (int) (raf.length() - position)
                : Integer.MAX_VALUE;

        mappedData = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, position, mappedLength);

        file = new MappedFileDataInput(mappedData, filename, 0);

        if (file == null) continue;

        // Verify key was found in data file
        DecoratedKey keyInDisk =
            SSTableReader.decodeKey(
                sstable.partitioner, sstable.descriptor, ByteBufferUtil.readWithShortLength(file));
        assert keyInDisk.equals(decoratedKey)
            : String.format("%s != %s in %s", keyInDisk, decoratedKey, file.getPath());

        long rowSize = SSTableReader.readRowSize(file, sstable.descriptor);

        assert rowSize > 0;
        assert rowSize < mappedLength;

        Filter bf = IndexHelper.defreezeBloomFilter(file, sstable.descriptor.usesOldBloomFilter);

        // verify this column in in this version of the row.
        if (!bf.isPresent(sblockId)) continue;

        List<IndexHelper.IndexInfo> indexList = IndexHelper.deserializeIndex(file);

        // we can stop early if bloom filter says none of the
        // columns actually exist -- but,
        // we can't stop before initializing the cf above, in
        // case there's a relevant tombstone
        ColumnFamilySerializer serializer = ColumnFamily.serializer();
        try {
          ColumnFamily cf =
              serializer.deserializeFromSSTableNoColumns(
                  ColumnFamily.create(sstable.metadata), file);

          if (cf.isMarkedForDelete()) continue;

        } catch (Exception e) {
          e.printStackTrace();

          throw new IOException(
              serializer
                  + " failed to deserialize "
                  + sstable.getColumnFamilyName()
                  + " with "
                  + sstable.metadata
                  + " from "
                  + file,
              e);
        }

        Integer sblockLength = null;

        if (indexList == null) sblockLength = seekToSubColumn(sstable.metadata, file, sblockId);
        else sblockLength = seekToSubColumn(sstable.metadata, file, sblockId, indexList);

        if (sblockLength == null || sblockLength < 0) continue;

        int bytesReadFromStart = mappedLength - (int) file.bytesRemaining();

        if (logger.isDebugEnabled())
          logger.debug("BlockLength = " + sblockLength + " Availible " + file.bytesRemaining());

        assert offset <= sblockLength : String.format("%d > %d", offset, sblockLength);

        long dataOffset = position + bytesReadFromStart;

        if (file.bytesRemaining() == 0 || sblockLength == 0) continue;

        return new LocalBlock(file.getPath(), dataOffset + offset, sblockLength - offset);

      } catch (IOException e) {
        throw new TException(e);
      } finally {
        FileUtils.closeQuietly(raf);
      }
    }

    return null;
  }
コード例 #29
0
  public static void main(String args[]) throws IOException {
    Options options = Options.parseArgs(args);
    try {
      // load keyspace descriptions.
      DatabaseDescriptor.loadSchemas();

      String ksName = null;
      String cfName = null;
      Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
      for (String filename : options.filenames) {
        File file = new File(filename);
        if (!file.exists()) {
          System.out.println("Skipping inexisting file " + file);
          continue;
        }

        Pair<Descriptor, Component> pair =
            SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
        if (pair == null) {
          System.out.println("Skipping non sstable file " + file);
          continue;
        }
        Descriptor desc = pair.left;

        if (ksName == null) ksName = desc.ksname;
        else if (!ksName.equals(desc.ksname))
          throw new IllegalArgumentException("All sstables must be part of the same keyspace");

        if (cfName == null) cfName = desc.cfname;
        else if (!cfName.equals(desc.cfname))
          throw new IllegalArgumentException("All sstables must be part of the same column family");

        Set<Component> components =
            new HashSet<Component>(
                Arrays.asList(
                    new Component[] {
                      Component.DATA,
                      Component.PRIMARY_INDEX,
                      Component.FILTER,
                      Component.COMPRESSION_INFO,
                      Component.STATS
                    }));

        Iterator<Component> iter = components.iterator();
        while (iter.hasNext()) {
          Component component = iter.next();
          if (!(new File(desc.filenameFor(component)).exists())) iter.remove();
        }
        parsedFilenames.put(desc, components);
      }

      if (ksName == null || cfName == null) {
        System.err.println("No valid sstables to split");
        System.exit(1);
      }

      // Do not load sstables since they might be broken
      Table table = Table.openWithoutSSTables(ksName);
      ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);

      String snapshotName = "pre-split-" + System.currentTimeMillis();

      List<SSTableReader> sstables = new ArrayList<SSTableReader>();
      for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet()) {
        try {
          SSTableReader sstable =
              SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs.metadata);
          sstables.add(sstable);

          if (options.snapshot) {
            File snapshotDirectory =
                Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
            sstable.createLinks(snapshotDirectory.getPath());
          }

        } catch (Exception e) {
          System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
          if (options.debug) e.printStackTrace(System.err);
        }
      }
      if (options.snapshot)
        System.out.println(
            String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));

      cfs.getDataTracker().markCompacting(sstables);
      for (SSTableReader sstable : sstables) {
        try {
          new SSTableSplitter(cfs, sstable, options.sizeInMB).split();

          // Remove the sstable
          sstable.markCompacted();
          sstable.releaseReference();
        } catch (Exception e) {
          System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
          if (options.debug) e.printStackTrace(System.err);
        }
      }
      SSTableDeletingTask.waitForDeletions();
      System.exit(0); // We need that to stop non daemonized threads
    } catch (Exception e) {
      System.err.println(e.getMessage());
      if (options.debug) e.printStackTrace(System.err);
      System.exit(1);
    }
  }
コード例 #30
0
 ColumnFamilyMetricNameFactory(ColumnFamilyStore cfs) {
   this.keyspaceName = cfs.keyspace.getName();
   this.columnFamilyName = cfs.name;
   isIndex = cfs.isIndex();
 }