@Test
  public void testGetSliceFromSuperBasic() throws Throwable {
    // tests slicing against data from one row spread across two sstables
    final Table table = Table.open("Keyspace1");
    final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Super1");
    final String ROW = "row2";

    RowMutation rm = new RowMutation("Keyspace1", ROW);
    ColumnFamily cf = ColumnFamily.create("Keyspace1", "Super1");
    SuperColumn sc = new SuperColumn("sc1".getBytes(), new LongType());
    sc.addColumn(new Column(getBytes(1), "val1".getBytes(), 1L));
    cf.addColumn(sc);
    rm.add(cf);
    rm.apply();

    Runnable verify =
        new WrappedRunnable() {
          public void runMayThrow() throws Exception {
            ColumnFamily cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Super1"),
                    ArrayUtils.EMPTY_BYTE_ARRAY,
                    ArrayUtils.EMPTY_BYTE_ARRAY,
                    false,
                    10);
            assertColumns(cf, "sc1");
            assertEquals(
                new String(cf.getColumn("sc1".getBytes()).getSubColumn(getBytes(1)).value()),
                "val1");
          }
        };

    reTest(table.getColumnFamilyStore("Standard1"), verify);
  }
 private int discardLast(ColumnFamily cf, int toDiscard, ColumnFamily newCf) {
   boolean isReversed = isReversed();
   DeletionInfo.InOrderTester tester = cf.deletionInfo().inOrderTester(isReversed);
   return isReversed
       ? discardHead(cf, toDiscard, newCf, cf.reverseIterator(), tester)
       : discardTail(cf, toDiscard, newCf, cf.iterator(), tester);
 }
예제 #3
0
  public Column get_column(String table, String key, ColumnPath column_path, int consistency_level)
      throws InvalidRequestException, NotFoundException {
    if (logger.isDebugEnabled()) logger.debug("get_column");
    ThriftValidation.validateColumnPath(table, column_path);

    QueryPath path = new QueryPath(column_path.column_family, column_path.super_column);
    ColumnFamily cfamily =
        readColumnFamily(
            new SliceByNamesReadCommand(table, key, path, Arrays.asList(column_path.column)),
            consistency_level);
    // TODO can we leverage getSlice here and just check that it returns one column?
    if (cfamily == null) {
      throw new NotFoundException();
    }
    Collection<IColumn> columns = null;
    if (column_path.super_column != null) {
      IColumn column = cfamily.getColumn(column_path.super_column);
      if (column != null) {
        columns = column.getSubColumns();
      }
    } else {
      columns = cfamily.getSortedColumns();
    }
    if (columns == null || columns.size() == 0) {
      throw new NotFoundException();
    }

    assert columns.size() == 1;
    IColumn column = columns.iterator().next();
    if (column.isMarkedForDelete()) {
      throw new NotFoundException();
    }

    return new Column(column.name(), column.value(), column.timestamp());
  }
  private int discardHead(
      ColumnFamily cf,
      int toDiscard,
      ColumnFamily copy,
      Iterator<Column> iter,
      DeletionInfo.InOrderTester tester) {
    ColumnCounter counter = columnCounter();

    List<Column> staticColumns = new ArrayList<>(cfm.staticColumns().size());

    // Discard the first 'toDiscard' live, non-static columns
    while (iter.hasNext()) {
      Column c = iter.next();

      // if it's a static column, don't count it and save it to add to the trimmed results
      ColumnDefinition columnDef = cfm.getColumnDefinitionFromColumnName(c.name());
      if (columnDef != null && columnDef.type == ColumnDefinition.Type.STATIC) {
        staticColumns.add(c);
        continue;
      }

      counter.count(c, tester);

      // once we've discarded the required amount, add the rest
      if (counter.live() > toDiscard) {
        for (Column staticColumn : staticColumns) copy.addColumn(staticColumn);

        copy.addColumn(c);
        while (iter.hasNext()) copy.addColumn(iter.next());
      }
    }
    return Math.min(counter.live(), toDiscard);
  }
예제 #5
0
  public List<SuperColumn> get_slice_super(
      String table,
      String key,
      String column_family,
      byte[] start,
      byte[] finish,
      boolean is_ascending,
      int count,
      int consistency_level)
      throws InvalidRequestException {
    if (logger.isDebugEnabled()) logger.debug("get_slice_super");
    if (!DatabaseDescriptor.getColumnFamilyType(table, column_family).equals("Super"))
      throw new InvalidRequestException("get_slice_super requires a super CF name");
    if (count <= 0) throw new InvalidRequestException("get_slice_super requires positive count");

    ColumnFamily cfamily =
        readColumnFamily(
            new SliceFromReadCommand(
                table, key, new QueryPath(column_family), start, finish, is_ascending, count),
            consistency_level);
    if (cfamily == null) {
      return EMPTY_SUPERCOLUMNS;
    }
    Collection<IColumn> columns = cfamily.getSortedColumns();
    return thriftifySuperColumns(columns, !is_ascending);
  }
  @Test
  public void testGetSliceFromLarge() throws Throwable {
    // tests slicing against 1000 columns in an sstable
    Table table = Table.open("Keyspace1");
    ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
    String key = "row3";
    RowMutation rm = new RowMutation("Keyspace1", key);
    ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
    for (int i = 1000; i < 2000; i++) cf.addColumn(column("col" + i, ("v" + i), 1L));
    rm.add(cf);
    rm.apply();
    cfStore.forceBlockingFlush();

    validateSliceLarge(cfStore);
    // compact so we have a big row with more than the minimum index count
    if (cfStore.getSSTables().size() > 1) {
      CompactionManager.instance.submitMajor(cfStore).get();
    }
    SSTableReader sstable = cfStore.getSSTables().iterator().next();
    DecoratedKey decKey = sstable.getPartitioner().decorateKey(key);
    SSTable.PositionSize info = sstable.getPosition(decKey);
    BufferedRandomAccessFile file = new BufferedRandomAccessFile(sstable.getFilename(), "r");
    file.seek(info.position);
    assert file.readUTF().equals(key);
    file.readInt();
    IndexHelper.skipBloomFilter(file);
    ArrayList<IndexHelper.IndexInfo> indexes = IndexHelper.deserializeIndex(file);
    assert indexes.size() > 2;
    validateSliceLarge(cfStore);
  }
  @VisibleForTesting
  List<Row> discardLast(List<Row> rows, int toDiscard) {
    if (toDiscard == 0 || rows.isEmpty()) return rows;

    int i = rows.size() - 1;
    DecoratedKey lastKey = null;
    ColumnFamily lastCf = null;
    while (toDiscard > 0 && i >= 0) {
      Row last = rows.get(i--);
      lastKey = last.key;
      lastCf = last.cf.cloneMeShallow(isReversed());
      toDiscard -=
          isReversed()
              ? discardFirst(last.cf, toDiscard, lastCf)
              : discardLast(last.cf, toDiscard, lastCf);
    }

    // If there is less live data than to discard, all is discarded
    if (toDiscard > 0) return Collections.<Row>emptyList();

    // i is the index of the last row that we are sure to keep. On top of that,
    // we also keep lastCf is it hasn't been fully emptied by the last iteration above.
    int count = lastCf.getColumnCount();
    int newSize = count == 0 ? i + 1 : i + 2;
    List<Row> newRows = new ArrayList<Row>(newSize);
    newRows.addAll(rows.subList(0, i + 1));
    if (count != 0) newRows.add(new Row(lastKey, lastCf));

    return newRows;
  }
예제 #8
0
  public SuperColumn get_super_column(
      String table, String key, SuperColumnPath super_column_path, int consistency_level)
      throws InvalidRequestException, NotFoundException {
    if (logger.isDebugEnabled()) logger.debug("get_superColumn");
    ThriftValidation.validateSuperColumnPath(table, super_column_path);

    ColumnFamily cfamily =
        readColumnFamily(
            new SliceByNamesReadCommand(
                table,
                key,
                new QueryPath(super_column_path.column_family),
                Arrays.asList(super_column_path.super_column)),
            consistency_level);
    if (cfamily == null) {
      throw new NotFoundException();
    }
    Collection<IColumn> columns = cfamily.getSortedColumns();
    if (columns == null || columns.size() == 0) {
      throw new NotFoundException();
    }

    assert columns.size() == 1;
    IColumn column = columns.iterator().next();
    if (column.getSubColumns().size() == 0) {
      throw new NotFoundException();
    }

    return new SuperColumn(column.name(), thriftifyColumns(column.getSubColumns()));
  }
  @Test
  public void testGetRowSingleColumn() throws Throwable {
    final Table table = Table.open("Keyspace1");
    final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");

    RowMutation rm = new RowMutation("Keyspace1", TEST_KEY);
    ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
    cf.addColumn(column("col1", "val1", 1L));
    cf.addColumn(column("col2", "val2", 1L));
    cf.addColumn(column("col3", "val3", 1L));
    rm.add(cf);
    rm.apply();

    Runnable verify =
        new WrappedRunnable() {
          public void runMayThrow() throws Exception {
            ColumnFamily cf;

            cf =
                cfStore.getColumnFamily(
                    new NamesQueryFilter(TEST_KEY, new QueryPath("Standard1"), "col1".getBytes()));
            assertColumns(cf, "col1");

            cf =
                cfStore.getColumnFamily(
                    new NamesQueryFilter(TEST_KEY, new QueryPath("Standard1"), "col3".getBytes()));
            assertColumns(cf, "col3");
          }
        };
    reTest(table.getColumnFamilyStore("Standard1"), verify);
  }
 private static void delete(ColumnFamily cf, int from, int to, long timestamp) {
   cf.delete(
       new DeletionInfo(
           b(from),
           b(to),
           cf.getComparator(),
           timestamp,
           (int) (System.currentTimeMillis() / 1000)));
 }
  @Test
  public void simpleQueryWithRangeTombstoneTest() throws Exception {
    Table table = Table.open(KSNAME);
    ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME);

    // Inserting data
    String key = "k1";
    RowMutation rm;
    ColumnFamily cf;

    rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
    for (int i = 0; i < 40; i += 2) add(rm, i, 0);
    rm.apply();
    cfs.forceBlockingFlush();

    rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
    cf = rm.addOrGet(CFNAME);
    delete(cf, 10, 22, 1);
    rm.apply();
    cfs.forceBlockingFlush();

    rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
    for (int i = 1; i < 40; i += 2) add(rm, i, 2);
    rm.apply();
    cfs.forceBlockingFlush();

    rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
    cf = rm.addOrGet(CFNAME);
    delete(cf, 19, 27, 3);
    rm.apply();
    // We don't flush to test with both a range tomsbtone in memtable and in sstable

    QueryPath path = new QueryPath(CFNAME);

    // Queries by name
    int[] live = new int[] {4, 9, 11, 17, 28};
    int[] dead = new int[] {12, 19, 21, 24, 27};
    SortedSet<ByteBuffer> columns = new TreeSet<ByteBuffer>(cfs.getComparator());
    for (int i : live) columns.add(b(i));
    for (int i : dead) columns.add(b(i));
    cf = cfs.getColumnFamily(QueryFilter.getNamesFilter(dk(key), path, columns));

    for (int i : live) assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
    for (int i : dead)
      assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live";

    // Queries by slices
    cf =
        cfs.getColumnFamily(
            QueryFilter.getSliceFilter(dk(key), path, b(7), b(30), false, Integer.MAX_VALUE));

    for (int i : new int[] {7, 8, 9, 11, 13, 15, 17, 28, 29, 30})
      assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
    for (int i : new int[] {10, 12, 14, 16, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27})
      assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live";
  }
예제 #12
0
  public RowMutation toSchema(long timestamp) {
    RowMutation rm = new RowMutation(Table.SYSTEM_TABLE, SystemTable.getSchemaKSKey(name));
    ColumnFamily cf = rm.addOrGet(SystemTable.SCHEMA_KEYSPACES_CF);

    cf.addColumn(Column.create(name, timestamp, "name"));
    cf.addColumn(Column.create(durableWrites, timestamp, "durable_writes"));
    cf.addColumn(Column.create(strategyClass.getName(), timestamp, "strategy_class"));
    cf.addColumn(Column.create(json(strategyOptions), timestamp, "strategy_options"));

    for (CFMetaData cfm : cfMetaData.values()) cfm.toSchema(rm, timestamp);

    return rm;
  }
예제 #13
0
  public ColumnFamily updateForKey(
      ByteBuffer key, ColumnNameBuilder builder, UpdateParameters params)
      throws InvalidRequestException {
    CFDefinition cfDef = cfm.getCfDef();
    ColumnFamily cf = UnsortedColumns.factory.create(cfm);

    // Inserting the CQL row marker (see #4361)
    // We always need to insert a marker, because of the following situation:
    //   CREATE TABLE t ( k int PRIMARY KEY, c text );
    //   INSERT INTO t(k, c) VALUES (1, 1)
    //   DELETE c FROM t WHERE k = 1;
    //   SELECT * FROM t;
    // The last query should return one row (but with c == null). Adding
    // the marker with the insert make sure the semantic is correct (while making sure a
    // 'DELETE FROM t WHERE k = 1' does remove the row entirely)
    //
    // We never insert markers for Super CF as this would confuse the thrift side.
    if (cfDef.isComposite && !cfDef.isCompact && !cfm.isSuper()) {
      ByteBuffer name = builder.copy().add(ByteBufferUtil.EMPTY_BYTE_BUFFER).build();
      cf.addColumn(params.makeColumn(name, ByteBufferUtil.EMPTY_BYTE_BUFFER));
    }

    List<Operation> updates = getOperations();

    if (cfDef.isCompact) {
      if (builder.componentCount() == 0)
        throw new InvalidRequestException(
            String.format("Missing PRIMARY KEY part %s", cfDef.columns.values().iterator().next()));

      if (cfDef.value == null) {
        // compact + no compact value implies there is no column outside the PK. So no operation
        // could
        // have passed through validation
        assert updates.isEmpty();
        setToEmptyOperation.execute(key, cf, builder.copy(), params);
      } else {
        // compact means we don't have a row marker, so don't accept to set only the PK. See
        // CASSANDRA-5648.
        if (updates.isEmpty())
          throw new InvalidRequestException(
              String.format("Column %s is mandatory for this COMPACT STORAGE table", cfDef.value));

        for (Operation update : updates) update.execute(key, cf, builder.copy(), params);
      }
    } else {
      for (Operation update : updates) update.execute(key, cf, builder.copy(), params);
    }

    return cf;
  }
예제 #14
0
  private void testDontPurgeAccidentaly(String k, String cfname)
      throws IOException, ExecutionException, InterruptedException {
    // This test catches the regression of CASSANDRA-2786
    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);

    // disable compaction while flushing
    cfs.clearUnsafe();
    cfs.disableAutoCompaction();

    // Add test row
    DecoratedKey key = Util.dk(k);
    RowMutation rm = new RowMutation(KEYSPACE1, key.key);
    rm.add(
        cfname,
        CompositeType.build(ByteBufferUtil.bytes("sc"), ByteBufferUtil.bytes("c")),
        ByteBufferUtil.EMPTY_BYTE_BUFFER,
        0);
    rm.apply();

    cfs.forceBlockingFlush();

    Collection<SSTableReader> sstablesBefore = cfs.getSSTables();

    QueryFilter filter = QueryFilter.getIdentityFilter(key, cfname, System.currentTimeMillis());
    assert !(cfs.getColumnFamily(filter).getColumnCount() == 0);

    // Remove key
    rm = new RowMutation(KEYSPACE1, key.key);
    rm.delete(cfname, 2);
    rm.apply();

    ColumnFamily cf = cfs.getColumnFamily(filter);
    assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;

    // Sleep one second so that the removal is indeed purgeable even with gcgrace == 0
    Thread.sleep(1000);

    cfs.forceBlockingFlush();

    Collection<SSTableReader> sstablesAfter = cfs.getSSTables();
    Collection<SSTableReader> toCompact = new ArrayList<SSTableReader>();
    for (SSTableReader sstable : sstablesAfter)
      if (!sstablesBefore.contains(sstable)) toCompact.add(sstable);

    Util.compact(cfs, toCompact);

    cf = cfs.getColumnFamily(filter);
    assert cf == null || cf.getColumnCount() == 0 : "should be empty: " + cf;
  }
예제 #15
0
 public static void assertColumns(ColumnFamily cf, String... columnNames) {
   Collection<IColumn> columns = cf == null ? new TreeSet<IColumn>() : cf.getSortedColumns();
   List<String> L = new ArrayList<String>();
   for (IColumn column : columns) {
     L.add(new String(column.name()));
   }
   assert Arrays.equals(L.toArray(new String[columns.size()]), columnNames)
       : "Columns ["
           + ((cf == null) ? "" : cf.getComparator().getColumnsString(columns))
           + "]"
           + " is not expected ["
           + StringUtils.join(columnNames, ",")
           + "]";
 }
예제 #16
0
  public Iterator<RangeTombstone> getRangeTombstoneIterator(final ColumnFamily source) {
    final DeletionInfo delInfo = source.deletionInfo();
    if (!delInfo.hasRanges() || slices.length == 0) return Iterators.emptyIterator();

    return new AbstractIterator<RangeTombstone>() {
      private int sliceIdx = 0;
      private Iterator<RangeTombstone> sliceIter = currentRangeIter();

      protected RangeTombstone computeNext() {
        while (true) {
          if (sliceIter.hasNext()) return sliceIter.next();

          if (!nextSlice()) return endOfData();

          sliceIter = currentRangeIter();
        }
      }

      private Iterator<RangeTombstone> currentRangeIter() {
        ColumnSlice slice = slices[reversed ? (slices.length - 1 - sliceIdx) : sliceIdx];
        return reversed
            ? delInfo.rangeIterator(slice.finish, slice.start)
            : delInfo.rangeIterator(slice.start, slice.finish);
      }

      private boolean nextSlice() {
        return ++sliceIdx < slices.length;
      }
    };
  }
예제 #17
0
  /** for resultsets of standard columns */
  private List<Column> getSlice(ReadCommand command, int consistency_level)
      throws InvalidRequestException {
    ColumnFamily cfamily = readColumnFamily(command, consistency_level);
    boolean reverseOrder = false;

    if (command instanceof SliceFromReadCommand)
      reverseOrder = !((SliceFromReadCommand) command).isAscending;

    if (cfamily == null || cfamily.getColumnsMap().size() == 0) {
      return EMPTY_COLUMNS;
    }
    if (cfamily.isSuper()) {
      IColumn column = cfamily.getColumnsMap().values().iterator().next();
      return thriftifyColumns(column.getSubColumns(), reverseOrder);
    }
    return thriftifyColumns(cfamily.getSortedColumns(), reverseOrder);
  }
예제 #18
0
  @Test
  public void testGetRowSliceByRange() throws Throwable {
    String key = TEST_KEY + "slicerow";
    Table table = Table.open("Keyspace1");
    ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
    RowMutation rm = new RowMutation("Keyspace1", key);
    ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
    // First write "a", "b", "c"
    cf.addColumn(column("a", "val1", 1L));
    cf.addColumn(column("b", "val2", 1L));
    cf.addColumn(column("c", "val3", 1L));
    rm.add(cf);
    rm.apply();

    cf =
        cfStore.getColumnFamily(
            key, new QueryPath("Standard1"), "b".getBytes(), "c".getBytes(), false, 100);
    assertEquals(2, cf.getColumnCount());

    cf =
        cfStore.getColumnFamily(
            key, new QueryPath("Standard1"), "b".getBytes(), "b".getBytes(), false, 100);
    assertEquals(1, cf.getColumnCount());

    cf =
        cfStore.getColumnFamily(
            key, new QueryPath("Standard1"), "b".getBytes(), "c".getBytes(), false, 1);
    assertEquals(1, cf.getColumnCount());

    cf =
        cfStore.getColumnFamily(
            key, new QueryPath("Standard1"), "c".getBytes(), "b".getBytes(), false, 1);
    assertNull(cf);
  }
예제 #19
0
  @Test
  public void testGetSliceNoMatch() throws Throwable {
    Table table = Table.open("Keyspace1");
    RowMutation rm = new RowMutation("Keyspace1", "row1000");
    ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard2");
    cf.addColumn(column("col1", "val1", 1));
    rm.add(cf);
    rm.apply();

    validateGetSliceNoMatch(table);
    table.getColumnFamilyStore("Standard2").forceBlockingFlush();
    validateGetSliceNoMatch(table);

    Collection<SSTableReader> ssTables = table.getColumnFamilyStore("Standard2").getSSTables();
    assertEquals(1, ssTables.size());
    ssTables.iterator().next().forceBloomFilterFailures();
    validateGetSliceNoMatch(table);
  }
예제 #20
0
  public boolean isFullyCoveredBy(ColumnFamily cf, long now) {
    // cf is the beginning of a partition. It covers this filter if:
    //   1) either this filter requests the head of the partition and request less
    //      than what cf has to offer (note: we do need to use getLiveCount() for that
    //      as it knows if the filter count cells or CQL3 rows).
    //   2) the start and finish bound of this filter are included in cf.
    if (isHeadFilter() && count <= getLiveCount(cf, now)) return true;

    if (start().isEmpty() || finish().isEmpty() || !cf.hasColumns()) return false;

    Composite low = isReversed() ? finish() : start();
    Composite high = isReversed() ? start() : finish();

    CellName first = cf.iterator(ColumnSlice.ALL_COLUMNS_ARRAY).next().name();
    CellName last = cf.reverseIterator(ColumnSlice.ALL_COLUMNS_ARRAY).next().name();

    return cf.getComparator().compare(first, low) <= 0
        && cf.getComparator().compare(high, last) <= 0;
  }
예제 #21
0
  // TODO move gcBefore into a field
  public void collateColumns(
      final ColumnFamily returnCF,
      List<? extends CloseableIterator<IColumn>> toCollate,
      final int gcBefore) {
    IFilter topLevelFilter = (superFilter == null ? filter : superFilter);
    Comparator<IColumn> fcomp = topLevelFilter.getColumnComparator(returnCF.getComparator());
    // define a 'reduced' iterator that merges columns w/ the same name, which
    // greatly simplifies computing liveColumns in the presence of tombstones.
    MergeIterator.Reducer<IColumn, IColumn> reducer =
        new MergeIterator.Reducer<IColumn, IColumn>() {
          ColumnFamily curCF = returnCF.cloneMeShallow();

          public void reduce(IColumn current) {
            if (curCF.isSuper() && curCF.isEmpty()) {
              // If it is the first super column we add, we must clone it since other super column
              // may modify
              // it otherwise and it could be aliased in a memtable somewhere. We'll also don't have
              // to care about what
              // consumers make of the result (for instance CFS.getColumnFamily() call
              // removeDeleted() on the
              // result which removes column; which shouldn't be done on the original super column).
              assert current instanceof SuperColumn;
              curCF.addColumn(((SuperColumn) current).cloneMe());
            } else {
              curCF.addColumn(current);
            }
          }

          protected IColumn getReduced() {
            IColumn c = curCF.iterator().next();
            if (superFilter != null) {
              // filterSuperColumn only looks at immediate parent (the supercolumn) when determining
              // if a subcolumn
              // is still live, i.e., not shadowed by the parent's tombstone.  so, bump it up
              // temporarily to the tombstone
              // time of the cf, if that is greater.
              long deletedAt = c.getMarkedForDeleteAt();
              if (returnCF.getMarkedForDeleteAt() > deletedAt)
                ((SuperColumn) c).delete(c.getLocalDeletionTime(), returnCF.getMarkedForDeleteAt());

              c = filter.filterSuperColumn((SuperColumn) c, gcBefore);
              ((SuperColumn) c)
                  .delete(
                      c.getLocalDeletionTime(),
                      deletedAt); // reset sc tombstone time to what it should be
            }
            curCF.clear();

            return c;
          }
        };
    Iterator<IColumn> reduced = MergeIterator.get(toCollate, fcomp, reducer);

    topLevelFilter.collectReducedColumns(returnCF, reduced, gcBefore);
  }
예제 #22
0
  public int get_column_count(
      String table, String key, ColumnParent column_parent, int consistency_level)
      throws InvalidRequestException {
    if (logger.isDebugEnabled()) logger.debug("get_column_count");
    // validateColumnParent assumes we require simple columns; g_c_c is the only
    // one of the columnParent-taking apis that can also work at the SC level.
    // so we roll a one-off validator here.
    String cfType = ThriftValidation.validateColumnFamily(table, column_parent.column_family);
    if (cfType.equals("Standard") && column_parent.super_column != null) {
      throw new InvalidRequestException(
          "columnfamily alone is required for standard CF " + column_parent.column_family);
    }

    ColumnFamily cfamily;
    cfamily =
        readColumnFamily(
            new SliceFromReadCommand(
                table,
                key,
                column_parent,
                ArrayUtils.EMPTY_BYTE_ARRAY,
                ArrayUtils.EMPTY_BYTE_ARRAY,
                true,
                Integer.MAX_VALUE),
            consistency_level);
    if (cfamily == null) {
      return 0;
    }
    Collection<IColumn> columns = null;
    if (column_parent.super_column != null) {
      IColumn column = cfamily.getColumn(column_parent.super_column);
      if (column != null) {
        columns = column.getSubColumns();
      }
    } else {
      columns = cfamily.getSortedColumns();
    }
    if (columns == null || columns.size() == 0) {
      return 0;
    }
    return columns.size();
  }
예제 #23
0
  public List<SuperColumn> get_slice_super_by_names(
      String table,
      String key,
      String column_family,
      List<byte[]> super_column_names,
      int consistency_level)
      throws InvalidRequestException {
    if (logger.isDebugEnabled()) logger.debug("get_slice_super_by_names");
    ThriftValidation.validateColumnFamily(table, column_family);

    ColumnFamily cfamily =
        readColumnFamily(
            new SliceByNamesReadCommand(
                table, key, new QueryPath(column_family), super_column_names),
            consistency_level);
    if (cfamily == null) {
      return EMPTY_SUPERCOLUMNS;
    }
    return thriftifySuperColumns(cfamily.getSortedColumns());
  }
예제 #24
0
  @Test
  public void testGetRowNoColumns() throws Throwable {
    final Table table = Table.open("Keyspace2");
    final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard3");

    RowMutation rm = new RowMutation("Keyspace2", TEST_KEY);
    ColumnFamily cf = ColumnFamily.create("Keyspace2", "Standard3");
    cf.addColumn(column("col1", "val1", 1L));
    rm.add(cf);
    rm.apply();

    Runnable verify =
        new WrappedRunnable() {
          public void runMayThrow() throws Exception {
            ColumnFamily cf;

            cf =
                cfStore.getColumnFamily(
                    new NamesQueryFilter(
                        TEST_KEY, new QueryPath("Standard3"), new TreeSet<byte[]>()));
            assertColumns(cf);

            cf =
                cfStore.getColumnFamily(
                    new SliceQueryFilter(
                        TEST_KEY,
                        new QueryPath("Standard3"),
                        ArrayUtils.EMPTY_BYTE_ARRAY,
                        ArrayUtils.EMPTY_BYTE_ARRAY,
                        false,
                        0));
            assertColumns(cf);

            cf =
                cfStore.getColumnFamily(
                    new NamesQueryFilter(TEST_KEY, new QueryPath("Standard3"), "col99".getBytes()));
            assertColumns(cf);
          }
        };
    reTest(table.getColumnFamilyStore("Standard3"), verify);
  }
  @Test
  public void overlappingRangeTest() throws Exception {
    CompactionManager.instance.disableAutoCompaction();
    Table table = Table.open(KSNAME);
    ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME);

    // Inserting data
    String key = "k2";
    RowMutation rm;
    ColumnFamily cf;

    rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
    for (int i = 0; i < 20; i++) add(rm, i, 0);
    rm.apply();
    cfs.forceBlockingFlush();

    rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
    cf = rm.addOrGet(CFNAME);
    delete(cf, 5, 15, 1);
    rm.apply();
    cfs.forceBlockingFlush();

    rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
    cf = rm.addOrGet(CFNAME);
    delete(cf, 5, 10, 1);
    rm.apply();
    cfs.forceBlockingFlush();

    rm = new RowMutation(KSNAME, ByteBufferUtil.bytes(key));
    cf = rm.addOrGet(CFNAME);
    delete(cf, 5, 8, 2);
    rm.apply();
    cfs.forceBlockingFlush();

    QueryPath path = new QueryPath(CFNAME);
    cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(key), path));

    for (int i = 0; i < 5; i++)
      assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
    for (int i = 16; i < 20; i++)
      assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
    for (int i = 5; i <= 15; i++)
      assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live";

    // Compact everything and re-test
    CompactionManager.instance.performMaximal(cfs);
    cf = cfs.getColumnFamily(QueryFilter.getIdentityFilter(dk(key), path));

    for (int i = 0; i < 5; i++)
      assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
    for (int i = 16; i < 20; i++)
      assert isLive(cf, cf.getColumn(b(i))) : "Column " + i + " should be live";
    for (int i = 5; i <= 15; i++)
      assert !isLive(cf, cf.getColumn(b(i))) : "Column " + i + " shouldn't be live";
  }
예제 #26
0
  @Test
  public void testGetSliceFromAdvanced() throws Throwable {
    // tests slicing against data from one row spread across two sstables
    final Table table = Table.open("Keyspace1");
    final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
    final String ROW = "row2";

    RowMutation rm = new RowMutation("Keyspace1", ROW);
    ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
    cf.addColumn(column("col1", "val1", 1L));
    cf.addColumn(column("col2", "val2", 1L));
    cf.addColumn(column("col3", "val3", 1L));
    cf.addColumn(column("col4", "val4", 1L));
    cf.addColumn(column("col5", "val5", 1L));
    cf.addColumn(column("col6", "val6", 1L));
    rm.add(cf);
    rm.apply();
    cfStore.forceBlockingFlush();

    rm = new RowMutation("Keyspace1", ROW);
    cf = ColumnFamily.create("Keyspace1", "Standard1");
    cf.addColumn(column("col1", "valx", 2L));
    cf.addColumn(column("col2", "valx", 2L));
    cf.addColumn(column("col3", "valx", 2L));
    rm.add(cf);
    rm.apply();

    Runnable verify =
        new WrappedRunnable() {
          public void runMayThrow() throws Exception {
            ColumnFamily cf;

            cf =
                cfStore.getColumnFamily(
                    ROW,
                    new QueryPath("Standard1"),
                    "col2".getBytes(),
                    ArrayUtils.EMPTY_BYTE_ARRAY,
                    false,
                    3);
            assertColumns(cf, "col2", "col3", "col4");
            assertEquals(new String(cf.getColumn("col2".getBytes()).value()), "valx");
            assertEquals(new String(cf.getColumn("col3".getBytes()).value()), "valx");
            assertEquals(new String(cf.getColumn("col4".getBytes()).value()), "val4");
          }
        };

    reTest(table.getColumnFamilyStore("Standard1"), verify);
  }
예제 #27
0
  private String getSlicesInfo(ColumnFamily container) {
    StringBuilder sb = new StringBuilder();
    CellNameType type = container.metadata().comparator;
    for (ColumnSlice sl : slices) {
      assert sl != null;

      sb.append('[');
      sb.append(type.getString(sl.start));
      sb.append('-');
      sb.append(type.getString(sl.finish));
      sb.append(']');
    }
    return sb.toString();
  }
예제 #28
0
  public void trim(ColumnFamily cf, int trimTo, long now) {
    // each cell can increment the count by at most one, so if we have fewer cells than trimTo, we
    // can skip trimming
    if (cf.getColumnCount() < trimTo) return;

    ColumnCounter counter = columnCounter(cf.getComparator(), now);

    Collection<Cell> cells = reversed ? cf.getReverseSortedColumns() : cf.getSortedColumns();

    DeletionInfo.InOrderTester tester = cf.deletionInfo().inOrderTester(reversed);

    for (Iterator<Cell> iter = cells.iterator(); iter.hasNext(); ) {
      Cell cell = iter.next();
      counter.count(cell, tester);

      if (counter.live() > trimTo) {
        iter.remove();
        while (iter.hasNext()) {
          iter.next();
          iter.remove();
        }
      }
    }
  }
  private int discardTail(
      ColumnFamily cf,
      int toDiscard,
      ColumnFamily copy,
      Iterator<Column> iter,
      DeletionInfo.InOrderTester tester) {
    // Redoing the counting like that is not extremely efficient.
    // This is called only for reversed slices or in the case of a race between
    // paging and a deletion (pretty unlikely), so this is probably acceptable.
    int liveCount = columnCounter().countAll(cf).live();

    ColumnCounter counter = columnCounter();
    // Discard the last 'toDiscard' live (so stop adding as sound as we're past 'liveCount -
    // toDiscard')
    while (iter.hasNext()) {
      Column c = iter.next();
      counter.count(c, tester);
      if (counter.live() > liveCount - toDiscard) break;

      copy.addColumn(c);
    }
    return Math.min(liveCount, toDiscard);
  }
예제 #30
0
  /**
   * Retrieves a local subBlock
   *
   * @param blockId row key
   * @param sblockId SubBlock column name
   * @param offset inside the sblock
   * @return a local sublock
   * @throws TException
   */
  private LocalBlock getLocalSubBlock(
      String subBlockCFName, ByteBuffer blockId, ByteBuffer sblockId, int offset)
      throws TException {
    DecoratedKey<Token<?>> decoratedKey =
        new DecoratedKey<Token<?>>(StorageService.getPartitioner().getToken(blockId), blockId);

    Table table = Table.open(cfsKeyspace);
    ColumnFamilyStore sblockStore = table.getColumnFamilyStore(subBlockCFName);

    Collection<SSTableReader> sstables = sblockStore.getSSTables();

    for (SSTableReader sstable : sstables) {

      long position = sstable.getPosition(decoratedKey, Operator.EQ);

      if (position == -1) continue;

      String filename = sstable.descriptor.filenameFor(Component.DATA);
      RandomAccessFile raf = null;
      int mappedLength = -1;
      MappedByteBuffer mappedData = null;
      MappedFileDataInput file = null;
      try {
        raf = new RandomAccessFile(filename, "r");
        assert position < raf.length();

        mappedLength =
            (raf.length() - position) < Integer.MAX_VALUE
                ? (int) (raf.length() - position)
                : Integer.MAX_VALUE;

        mappedData = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, position, mappedLength);

        file = new MappedFileDataInput(mappedData, filename, 0);

        if (file == null) continue;

        // Verify key was found in data file
        DecoratedKey keyInDisk =
            SSTableReader.decodeKey(
                sstable.partitioner, sstable.descriptor, ByteBufferUtil.readWithShortLength(file));
        assert keyInDisk.equals(decoratedKey)
            : String.format("%s != %s in %s", keyInDisk, decoratedKey, file.getPath());

        long rowSize = SSTableReader.readRowSize(file, sstable.descriptor);

        assert rowSize > 0;
        assert rowSize < mappedLength;

        Filter bf = IndexHelper.defreezeBloomFilter(file, sstable.descriptor.usesOldBloomFilter);

        // verify this column in in this version of the row.
        if (!bf.isPresent(sblockId)) continue;

        List<IndexHelper.IndexInfo> indexList = IndexHelper.deserializeIndex(file);

        // we can stop early if bloom filter says none of the
        // columns actually exist -- but,
        // we can't stop before initializing the cf above, in
        // case there's a relevant tombstone
        ColumnFamilySerializer serializer = ColumnFamily.serializer();
        try {
          ColumnFamily cf =
              serializer.deserializeFromSSTableNoColumns(
                  ColumnFamily.create(sstable.metadata), file);

          if (cf.isMarkedForDelete()) continue;

        } catch (Exception e) {
          e.printStackTrace();

          throw new IOException(
              serializer
                  + " failed to deserialize "
                  + sstable.getColumnFamilyName()
                  + " with "
                  + sstable.metadata
                  + " from "
                  + file,
              e);
        }

        Integer sblockLength = null;

        if (indexList == null) sblockLength = seekToSubColumn(sstable.metadata, file, sblockId);
        else sblockLength = seekToSubColumn(sstable.metadata, file, sblockId, indexList);

        if (sblockLength == null || sblockLength < 0) continue;

        int bytesReadFromStart = mappedLength - (int) file.bytesRemaining();

        if (logger.isDebugEnabled())
          logger.debug("BlockLength = " + sblockLength + " Availible " + file.bytesRemaining());

        assert offset <= sblockLength : String.format("%d > %d", offset, sblockLength);

        long dataOffset = position + bytesReadFromStart;

        if (file.bytesRemaining() == 0 || sblockLength == 0) continue;

        return new LocalBlock(file.getPath(), dataOffset + offset, sblockLength - offset);

      } catch (IOException e) {
        throw new TException(e);
      } finally {
        FileUtils.closeQuietly(raf);
      }
    }

    return null;
  }