@Test
public void testGetSliceFromAdvanced() throws Throwable {
// tests slicing against data from one row spread across two sstables
final Table table = Table.open("Keyspace1");
final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
final String ROW = "row2";
RowMutation rm = new RowMutation("Keyspace1", ROW);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
cf.addColumn(column("col1", "val1", 1L));
cf.addColumn(column("col2", "val2", 1L));
cf.addColumn(column("col3", "val3", 1L));
cf.addColumn(column("col4", "val4", 1L));
cf.addColumn(column("col5", "val5", 1L));
cf.addColumn(column("col6", "val6", 1L));
rm.add(cf);
rm.apply();
cfStore.forceBlockingFlush();
rm = new RowMutation("Keyspace1", ROW);
cf = ColumnFamily.create("Keyspace1", "Standard1");
cf.addColumn(column("col1", "valx", 2L));
cf.addColumn(column("col2", "valx", 2L));
cf.addColumn(column("col3", "valx", 2L));
rm.add(cf);
rm.apply();
Runnable verify =
new WrappedRunnable() {
public void runMayThrow() throws Exception {
ColumnFamily cf;
cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Standard1"),
"col2".getBytes(),
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
3);
assertColumns(cf, "col2", "col3", "col4");
assertEquals(new String(cf.getColumn("col2".getBytes()).value()), "valx");
assertEquals(new String(cf.getColumn("col3".getBytes()).value()), "valx");
assertEquals(new String(cf.getColumn("col4".getBytes()).value()), "val4");
}
};
reTest(table.getColumnFamilyStore("Standard1"), verify);
}
@Test
public void testGetRowSingleColumn() throws Throwable {
final Table table = Table.open("Keyspace1");
final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
RowMutation rm = new RowMutation("Keyspace1", TEST_KEY);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
cf.addColumn(column("col1", "val1", 1L));
cf.addColumn(column("col2", "val2", 1L));
cf.addColumn(column("col3", "val3", 1L));
rm.add(cf);
rm.apply();
Runnable verify =
new WrappedRunnable() {
public void runMayThrow() throws Exception {
ColumnFamily cf;
cf =
cfStore.getColumnFamily(
new NamesQueryFilter(TEST_KEY, new QueryPath("Standard1"), "col1".getBytes()));
assertColumns(cf, "col1");
cf =
cfStore.getColumnFamily(
new NamesQueryFilter(TEST_KEY, new QueryPath("Standard1"), "col3".getBytes()));
assertColumns(cf, "col3");
}
};
reTest(table.getColumnFamilyStore("Standard1"), verify);
}
@Test
public void testGetSliceFromSuperBasic() throws Throwable {
// tests slicing against data from one row spread across two sstables
final Table table = Table.open("Keyspace1");
final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Super1");
final String ROW = "row2";
RowMutation rm = new RowMutation("Keyspace1", ROW);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Super1");
SuperColumn sc = new SuperColumn("sc1".getBytes(), new LongType());
sc.addColumn(new Column(getBytes(1), "val1".getBytes(), 1L));
cf.addColumn(sc);
rm.add(cf);
rm.apply();
Runnable verify =
new WrappedRunnable() {
public void runMayThrow() throws Exception {
ColumnFamily cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Super1"),
ArrayUtils.EMPTY_BYTE_ARRAY,
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
10);
assertColumns(cf, "sc1");
assertEquals(
new String(cf.getColumn("sc1".getBytes()).getSubColumn(getBytes(1)).value()),
"val1");
}
};
reTest(table.getColumnFamilyStore("Standard1"), verify);
}
@Test
public void testGetSliceFromLarge() throws Throwable {
// tests slicing against 1000 columns in an sstable
Table table = Table.open("Keyspace1");
ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
String key = "row3";
RowMutation rm = new RowMutation("Keyspace1", key);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
for (int i = 1000; i < 2000; i++) cf.addColumn(column("col" + i, ("v" + i), 1L));
rm.add(cf);
rm.apply();
cfStore.forceBlockingFlush();
validateSliceLarge(cfStore);
// compact so we have a big row with more than the minimum index count
if (cfStore.getSSTables().size() > 1) {
CompactionManager.instance.submitMajor(cfStore).get();
}
SSTableReader sstable = cfStore.getSSTables().iterator().next();
DecoratedKey decKey = sstable.getPartitioner().decorateKey(key);
SSTable.PositionSize info = sstable.getPosition(decKey);
BufferedRandomAccessFile file = new BufferedRandomAccessFile(sstable.getFilename(), "r");
file.seek(info.position);
assert file.readUTF().equals(key);
file.readInt();
IndexHelper.skipBloomFilter(file);
ArrayList<IndexHelper.IndexInfo> indexes = IndexHelper.deserializeIndex(file);
assert indexes.size() > 2;
validateSliceLarge(cfStore);
}
@Test
public void testGetRowSliceByRange() throws Throwable {
String key = TEST_KEY + "slicerow";
Table table = Table.open("Keyspace1");
ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
RowMutation rm = new RowMutation("Keyspace1", key);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
// First write "a", "b", "c"
cf.addColumn(column("a", "val1", 1L));
cf.addColumn(column("b", "val2", 1L));
cf.addColumn(column("c", "val3", 1L));
rm.add(cf);
rm.apply();
cf =
cfStore.getColumnFamily(
key, new QueryPath("Standard1"), "b".getBytes(), "c".getBytes(), false, 100);
assertEquals(2, cf.getColumnCount());
cf =
cfStore.getColumnFamily(
key, new QueryPath("Standard1"), "b".getBytes(), "b".getBytes(), false, 100);
assertEquals(1, cf.getColumnCount());
cf =
cfStore.getColumnFamily(
key, new QueryPath("Standard1"), "b".getBytes(), "c".getBytes(), false, 1);
assertEquals(1, cf.getColumnCount());
cf =
cfStore.getColumnFamily(
key, new QueryPath("Standard1"), "c".getBytes(), "b".getBytes(), false, 1);
assertNull(cf);
}
@Test
public void testGetSliceNoMatch() throws Throwable {
Table table = Table.open("Keyspace1");
RowMutation rm = new RowMutation("Keyspace1", "row1000");
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard2");
cf.addColumn(column("col1", "val1", 1));
rm.add(cf);
rm.apply();
validateGetSliceNoMatch(table);
table.getColumnFamilyStore("Standard2").forceBlockingFlush();
validateGetSliceNoMatch(table);
Collection<SSTableReader> ssTables = table.getColumnFamilyStore("Standard2").getSSTables();
assertEquals(1, ssTables.size());
ssTables.iterator().next().forceBloomFilterFailures();
validateGetSliceNoMatch(table);
}
@Test
public void testGetRowNoColumns() throws Throwable {
final Table table = Table.open("Keyspace2");
final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard3");
RowMutation rm = new RowMutation("Keyspace2", TEST_KEY);
ColumnFamily cf = ColumnFamily.create("Keyspace2", "Standard3");
cf.addColumn(column("col1", "val1", 1L));
rm.add(cf);
rm.apply();
Runnable verify =
new WrappedRunnable() {
public void runMayThrow() throws Exception {
ColumnFamily cf;
cf =
cfStore.getColumnFamily(
new NamesQueryFilter(
TEST_KEY, new QueryPath("Standard3"), new TreeSet<byte[]>()));
assertColumns(cf);
cf =
cfStore.getColumnFamily(
new SliceQueryFilter(
TEST_KEY,
new QueryPath("Standard3"),
ArrayUtils.EMPTY_BYTE_ARRAY,
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
0));
assertColumns(cf);
cf =
cfStore.getColumnFamily(
new NamesQueryFilter(TEST_KEY, new QueryPath("Standard3"), "col99".getBytes()));
assertColumns(cf);
}
};
reTest(table.getColumnFamilyStore("Standard3"), verify);
}
/**
* Retrieves a local subBlock
*
* @param blockId row key
* @param sblockId SubBlock column name
* @param offset inside the sblock
* @return a local sublock
* @throws TException
*/
private LocalBlock getLocalSubBlock(
String subBlockCFName, ByteBuffer blockId, ByteBuffer sblockId, int offset)
throws TException {
DecoratedKey<Token<?>> decoratedKey =
new DecoratedKey<Token<?>>(StorageService.getPartitioner().getToken(blockId), blockId);
Table table = Table.open(cfsKeyspace);
ColumnFamilyStore sblockStore = table.getColumnFamilyStore(subBlockCFName);
Collection<SSTableReader> sstables = sblockStore.getSSTables();
for (SSTableReader sstable : sstables) {
long position = sstable.getPosition(decoratedKey, Operator.EQ);
if (position == -1) continue;
String filename = sstable.descriptor.filenameFor(Component.DATA);
RandomAccessFile raf = null;
int mappedLength = -1;
MappedByteBuffer mappedData = null;
MappedFileDataInput file = null;
try {
raf = new RandomAccessFile(filename, "r");
assert position < raf.length();
mappedLength =
(raf.length() - position) < Integer.MAX_VALUE
? (int) (raf.length() - position)
: Integer.MAX_VALUE;
mappedData = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, position, mappedLength);
file = new MappedFileDataInput(mappedData, filename, 0);
if (file == null) continue;
// Verify key was found in data file
DecoratedKey keyInDisk =
SSTableReader.decodeKey(
sstable.partitioner, sstable.descriptor, ByteBufferUtil.readWithShortLength(file));
assert keyInDisk.equals(decoratedKey)
: String.format("%s != %s in %s", keyInDisk, decoratedKey, file.getPath());
long rowSize = SSTableReader.readRowSize(file, sstable.descriptor);
assert rowSize > 0;
assert rowSize < mappedLength;
Filter bf = IndexHelper.defreezeBloomFilter(file, sstable.descriptor.usesOldBloomFilter);
// verify this column in in this version of the row.
if (!bf.isPresent(sblockId)) continue;
List<IndexHelper.IndexInfo> indexList = IndexHelper.deserializeIndex(file);
// we can stop early if bloom filter says none of the
// columns actually exist -- but,
// we can't stop before initializing the cf above, in
// case there's a relevant tombstone
ColumnFamilySerializer serializer = ColumnFamily.serializer();
try {
ColumnFamily cf =
serializer.deserializeFromSSTableNoColumns(
ColumnFamily.create(sstable.metadata), file);
if (cf.isMarkedForDelete()) continue;
} catch (Exception e) {
e.printStackTrace();
throw new IOException(
serializer
+ " failed to deserialize "
+ sstable.getColumnFamilyName()
+ " with "
+ sstable.metadata
+ " from "
+ file,
e);
}
Integer sblockLength = null;
if (indexList == null) sblockLength = seekToSubColumn(sstable.metadata, file, sblockId);
else sblockLength = seekToSubColumn(sstable.metadata, file, sblockId, indexList);
if (sblockLength == null || sblockLength < 0) continue;
int bytesReadFromStart = mappedLength - (int) file.bytesRemaining();
if (logger.isDebugEnabled())
logger.debug("BlockLength = " + sblockLength + " Availible " + file.bytesRemaining());
assert offset <= sblockLength : String.format("%d > %d", offset, sblockLength);
long dataOffset = position + bytesReadFromStart;
if (file.bytesRemaining() == 0 || sblockLength == 0) continue;
return new LocalBlock(file.getPath(), dataOffset + offset, sblockLength - offset);
} catch (IOException e) {
throw new TException(e);
} finally {
FileUtils.closeQuietly(raf);
}
}
return null;
}
@Test
public void testGetSliceFromBasic() throws Throwable {
// tests slicing against data from one row in a memtable and then flushed to an sstable
final Table table = Table.open("Keyspace1");
final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
final String ROW = "row1";
RowMutation rm = new RowMutation("Keyspace1", ROW);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
cf.addColumn(column("col1", "val1", 1L));
cf.addColumn(column("col3", "val3", 1L));
cf.addColumn(column("col4", "val4", 1L));
cf.addColumn(column("col5", "val5", 1L));
cf.addColumn(column("col7", "val7", 1L));
cf.addColumn(column("col9", "val9", 1L));
rm.add(cf);
rm.apply();
rm = new RowMutation("Keyspace1", ROW);
rm.delete(new QueryPath("Standard1", null, "col4".getBytes()), 2L);
rm.apply();
Runnable verify =
new WrappedRunnable() {
public void runMayThrow() throws Exception {
ColumnFamily cf;
cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Standard1"),
"col5".getBytes(),
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
2);
assertColumns(cf, "col5", "col7");
cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Standard1"),
"col4".getBytes(),
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
2);
assertColumns(cf, "col4", "col5", "col7");
assertColumns(ColumnFamilyStore.removeDeleted(cf, Integer.MAX_VALUE), "col5", "col7");
cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Standard1"),
"col5".getBytes(),
ArrayUtils.EMPTY_BYTE_ARRAY,
true,
2);
assertColumns(cf, "col3", "col4", "col5");
cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Standard1"),
"col6".getBytes(),
ArrayUtils.EMPTY_BYTE_ARRAY,
true,
2);
assertColumns(cf, "col3", "col4", "col5");
cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Standard1"),
ArrayUtils.EMPTY_BYTE_ARRAY,
ArrayUtils.EMPTY_BYTE_ARRAY,
true,
2);
assertColumns(cf, "col7", "col9");
cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Standard1"),
"col95".getBytes(),
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
2);
assertColumns(cf);
cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Standard1"),
"col0".getBytes(),
ArrayUtils.EMPTY_BYTE_ARRAY,
true,
2);
assertColumns(cf);
}
};
reTest(table.getColumnFamilyStore("Standard1"), verify);
}
@Test
public void testGetSliceWithCutoff() throws Throwable {
// tests slicing against data from one row in a memtable and then flushed to an sstable
final Table table = Table.open("Keyspace1");
final ColumnFamilyStore cfStore = table.getColumnFamilyStore("Standard1");
final String ROW = "row4";
final NumberFormat fmt = new DecimalFormat("000");
RowMutation rm = new RowMutation("Keyspace1", ROW);
ColumnFamily cf = ColumnFamily.create("Keyspace1", "Standard1");
// at this rate, we're getting 78-79 cos/block, assuming the blocks are set to be about 4k.
// so if we go to 300, we'll get at least 4 blocks, which is plenty for testing.
for (int i = 0; i < 300; i++)
cf.addColumn(column("col" + fmt.format(i), "omg!thisisthevalue!" + i, 1L));
rm.add(cf);
rm.apply();
Runnable verify =
new WrappedRunnable() {
public void runMayThrow() throws Exception {
ColumnFamily cf;
// blocks are partitioned like this: 000-097, 098-193, 194-289, 290-299, assuming a 4k
// column index size.
assert DatabaseDescriptor.getColumnIndexSize() == 4096
: "Unexpected column index size, block boundaries won't be where tests expect them.";
// test forward, spanning a segment.
cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Standard1"),
"col096".getBytes(),
"col099".getBytes(),
false,
4);
assertColumns(cf, "col096", "col097", "col098", "col099");
// test reversed, spanning a segment.
cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Standard1"),
"col099".getBytes(),
"col096".getBytes(),
true,
4);
assertColumns(cf, "col096", "col097", "col098", "col099");
// test forward, within a segment.
cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Standard1"),
"col100".getBytes(),
"col103".getBytes(),
false,
4);
assertColumns(cf, "col100", "col101", "col102", "col103");
// test reversed, within a segment.
cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Standard1"),
"col103".getBytes(),
"col100".getBytes(),
true,
4);
assertColumns(cf, "col100", "col101", "col102", "col103");
// test forward from beginning, spanning a segment.
String[] strCols = new String[100]; // col000-col099
for (int i = 0; i < 100; i++) strCols[i] = "col" + fmt.format(i);
cf =
cfStore.getColumnFamily(
ROW,
new QueryPath("Standard1"),
"".getBytes(),
"col099".getBytes(),
false,
100);
assertColumns(cf, strCols);
// test reversed, from end, spanning a segment.
cf =
cfStore.getColumnFamily(
ROW, new QueryPath("Standard1"), "".getBytes(), "col288".getBytes(), true, 12);
assertColumns(
cf, "col288", "col289", "col290", "col291", "col292", "col293", "col294", "col295",
"col296", "col297", "col298", "col299");
}
};
reTest(table.getColumnFamilyStore("Standard1"), verify);
}
