public void repair(IColumn column) {
Collection<IColumn> columns = column.getSubColumns();
for (IColumn subColumn : columns) {
IColumn columnInternal = columns_.get(subColumn.name());
if (columnInternal == null) columns_.put(subColumn.name(), subColumn);
else columnInternal.repair(subColumn);
}
}
/*
* Deserialize a particular column since the name is in the form of
* superColumn:column.
*/
public IColumn deserialize(DataInputStream dis, String name, IFilter filter) throws IOException {
if (dis.available() == 0) return null;
String[] names = RowMutation.getColumnAndColumnFamily(name);
if (names.length == 1) {
IColumn superColumn = defreezeSuperColumn(dis);
if (name.equals(superColumn.name())) {
if (!superColumn.isMarkedForDelete()) {
/* read the number of columns stored */
int size = dis.readInt();
/* read the size of all columns */
dis.readInt();
IColumn column = null;
for (int i = 0; i < size; ++i) {
column = Column.serializer().deserialize(dis, filter);
if (column != null) {
superColumn.addColumn(column.name(), column);
column = null;
if (filter.isDone()) {
break;
}
}
}
}
return superColumn;
} else {
/* read the number of columns stored */
dis.readInt();
/* read the size of all columns to skip */
int size = dis.readInt();
dis.skip(size);
return null;
}
}
SuperColumn superColumn = defreezeSuperColumn(dis);
if (!superColumn.isMarkedForDelete()) {
int size = dis.readInt();
/* skip the size of the columns */
dis.readInt();
if (size > 0) {
for (int i = 0; i < size; ++i) {
IColumn subColumn = Column.serializer().deserialize(dis, names[1], filter);
if (subColumn != null) {
superColumn.addColumn(subColumn.name(), subColumn);
break;
}
}
}
}
return superColumn;
}
/**
* Given the collection of columns in the Column Family, the name index is generated and written
* into the provided stream
*
* @param columns for whom the name index needs to be generated
* @param dos stream into which the serialized name index needs to be written.
* @throws IOException
*/
private static void doIndexing(
AbstractType comparator, Collection<IColumn> columns, DataOutput dos) throws IOException {
if (columns.isEmpty()) {
dos.writeInt(0);
return;
}
/*
* Maintains a list of ColumnIndexInfo objects for the columns in this
* column family. The key is the column name and the position is the
* relative offset of that column name from the start of the list.
* We do this so that we don't read all the columns into memory.
*/
List<IndexHelper.IndexInfo> indexList = new ArrayList<IndexHelper.IndexInfo>();
int endPosition = 0, startPosition = -1;
int indexSizeInBytes = 0;
IColumn column = null, firstColumn = null;
/* column offsets at the right thresholds into the index map. */
for (Iterator<IColumn> it = columns.iterator(); it.hasNext(); ) {
column = it.next();
if (firstColumn == null) {
firstColumn = column;
startPosition = endPosition;
}
endPosition += column.serializedSize();
/* if we hit the column index size that we have to index after, go ahead and index it. */
if (endPosition - startPosition >= DatabaseDescriptor.getColumnIndexSize()) {
IndexHelper.IndexInfo cIndexInfo =
new IndexHelper.IndexInfo(
firstColumn.name(), column.name(), startPosition, endPosition - startPosition);
indexList.add(cIndexInfo);
indexSizeInBytes += cIndexInfo.serializedSize();
firstColumn = null;
}
}
// the last column may have fallen on an index boundary already. if not, index it explicitly.
if (indexList.isEmpty()
|| comparator.compare(indexList.get(indexList.size() - 1).lastName, column.name()) != 0) {
IndexHelper.IndexInfo cIndexInfo =
new IndexHelper.IndexInfo(
firstColumn.name(), column.name(), startPosition, endPosition - startPosition);
indexList.add(cIndexInfo);
indexSizeInBytes += cIndexInfo.serializedSize();
}
assert indexSizeInBytes > 0;
dos.writeInt(indexSizeInBytes);
for (IndexHelper.IndexInfo cIndexInfo : indexList) {
cIndexInfo.serialize(dos);
}
}
public Column get_column(String table, String key, ColumnPath column_path, int consistency_level)
throws InvalidRequestException, NotFoundException {
if (logger.isDebugEnabled()) logger.debug("get_column");
ThriftValidation.validateColumnPath(table, column_path);
QueryPath path = new QueryPath(column_path.column_family, column_path.super_column);
ColumnFamily cfamily =
readColumnFamily(
new SliceByNamesReadCommand(table, key, path, Arrays.asList(column_path.column)),
consistency_level);
// TODO can we leverage getSlice here and just check that it returns one column?
if (cfamily == null) {
throw new NotFoundException();
}
Collection<IColumn> columns = null;
if (column_path.super_column != null) {
IColumn column = cfamily.getColumn(column_path.super_column);
if (column != null) {
columns = column.getSubColumns();
}
} else {
columns = cfamily.getSortedColumns();
}
if (columns == null || columns.size() == 0) {
throw new NotFoundException();
}
assert columns.size() == 1;
IColumn column = columns.iterator().next();
if (column.isMarkedForDelete()) {
throw new NotFoundException();
}
return new Column(column.name(), column.value(), column.timestamp());
}
public SuperColumn get_super_column(
String table, String key, SuperColumnPath super_column_path, int consistency_level)
throws InvalidRequestException, NotFoundException {
if (logger.isDebugEnabled()) logger.debug("get_superColumn");
ThriftValidation.validateSuperColumnPath(table, super_column_path);
ColumnFamily cfamily =
readColumnFamily(
new SliceByNamesReadCommand(
table,
key,
new QueryPath(super_column_path.column_family),
Arrays.asList(super_column_path.super_column)),
consistency_level);
if (cfamily == null) {
throw new NotFoundException();
}
Collection<IColumn> columns = cfamily.getSortedColumns();
if (columns == null || columns.size() == 0) {
throw new NotFoundException();
}
assert columns.size() == 1;
IColumn column = columns.iterator().next();
if (column.getSubColumns().size() == 0) {
throw new NotFoundException();
}
return new SuperColumn(column.name(), thriftifyColumns(column.getSubColumns()));
}
public String name(String key) {
IColumn column = columns_.get(key);
if (column instanceof SuperColumn)
throw new UnsupportedOperationException("A super column cannot hold other super columns.");
if (column != null) return column.name();
return null;
}
public IColumn diff(IColumn column) {
IColumn columnDiff = new SuperColumn(column.name());
Collection<IColumn> columns = column.getSubColumns();
for (IColumn subColumn : columns) {
IColumn columnInternal = columns_.get(subColumn.name());
if (columnInternal == null) {
columnDiff.addColumn(subColumn.name(), subColumn);
} else {
IColumn subColumnDiff = columnInternal.diff(subColumn);
if (subColumnDiff != null) {
columnDiff.addColumn(subColumn.name(), subColumnDiff);
}
}
}
if (columnDiff.getSubColumns().size() != 0) return columnDiff;
else return null;
}
/*
* Go through each sub column if it exists then as it to resolve itself if the
* column does not exist then create it.
*/
public boolean putColumn(IColumn column) {
if (!(column instanceof SuperColumn))
throw new UnsupportedOperationException("Only Super column objects should be put here");
if (!name_.equals(column.name()))
throw new IllegalArgumentException(
"The name should match the name of the current column or super column");
Collection<IColumn> columns = column.getSubColumns();
for (IColumn subColumn : columns) {
IColumn columnInternal = columns_.get(subColumn.name());
if (columnInternal == null) {
addColumn(subColumn.name(), subColumn);
} else {
columnInternal.putColumn(subColumn);
}
}
return false;
}
/**
* Create a bloom filter that contains the subcolumns and the columns that make up this Column
* Family.
*
* @param columns columns of the ColumnFamily
* @return BloomFilter with the summarized information.
*/
private static BloomFilter createColumnBloomFilter(Collection<IColumn> columns) {
int columnCount = 0;
for (IColumn column : columns) {
columnCount += column.getObjectCount();
}
BloomFilter bf = BloomFilter.getFilter(columnCount, 4);
for (IColumn column : columns) {
bf.add(column.name());
/* If this is SuperColumn type Column Family we need to get the subColumns too. */
if (column instanceof SuperColumn) {
Collection<IColumn> subColumns = column.getSubColumns();
for (IColumn subColumn : subColumns) {
bf.add(subColumn.name());
}
}
}
return bf;
}
private void fillSuperColumn(IColumn superColumn, DataInputStream dis) throws IOException {
if (dis.available() == 0) return;
/* read the number of columns */
int size = dis.readInt();
/* read the size of all columns */
dis.readInt();
for (int i = 0; i < size; ++i) {
IColumn subColumn = Column.serializer().deserialize(dis);
superColumn.addColumn(subColumn.name(), subColumn);
}
}
public static void assertColumns(ColumnFamily cf, String... columnNames) {
Collection<IColumn> columns = cf == null ? new TreeSet<IColumn>() : cf.getSortedColumns();
List<String> L = new ArrayList<String>();
for (IColumn column : columns) {
L.add(new String(column.name()));
}
assert Arrays.equals(L.toArray(new String[columns.size()]), columnNames)
: "Columns ["
+ ((cf == null) ? "" : cf.getComparator().getColumnsString(columns))
+ "]"
+ " is not expected ["
+ StringUtils.join(columnNames, ",")
+ "]";
}
public List<Column> thriftifyColumns(Collection<IColumn> columns, boolean reverseOrder) {
if (columns == null || columns.isEmpty()) {
return EMPTY_COLUMNS;
}
ArrayList<Column> thriftColumns = new ArrayList<Column>(columns.size());
for (IColumn column : columns) {
if (column.isMarkedForDelete()) {
continue;
}
Column thrift_column = new Column(column.name(), column.value(), column.timestamp());
thriftColumns.add(thrift_column);
}
// we have to do the reversing here, since internally we pass results around in ColumnFamily
// objects, which always sort their columns in the "natural" order
if (reverseOrder) Collections.reverse(thriftColumns);
return thriftColumns;
}
private List<SuperColumn> thriftifySuperColumns(
Collection<IColumn> columns, boolean reverseOrder) {
if (columns == null || columns.isEmpty()) {
return EMPTY_SUPERCOLUMNS;
}
ArrayList<SuperColumn> thriftSuperColumns = new ArrayList<SuperColumn>(columns.size());
for (IColumn column : columns) {
List<Column> subcolumns = thriftifyColumns(column.getSubColumns());
if (subcolumns.isEmpty()) {
continue;
}
thriftSuperColumns.add(new SuperColumn(column.name(), subcolumns));
}
if (reverseOrder) Collections.reverse(thriftSuperColumns);
return thriftSuperColumns;
}
@Test
public void testReconcile() throws UnknownHostException {
IColumn left;
IColumn right;
IColumn reconciled;
ByteBuffer context;
// tombstone + tombstone
left = new DeletedColumn(ByteBufferUtil.bytes("x"), 1, 1L);
right = new DeletedColumn(ByteBufferUtil.bytes("x"), 2, 2L);
assert left.reconcile(right).getMarkedForDeleteAt() == right.getMarkedForDeleteAt();
assert right.reconcile(left).getMarkedForDeleteAt() == right.getMarkedForDeleteAt();
// tombstone > live
left = new DeletedColumn(ByteBufferUtil.bytes("x"), 1, 2L);
right = new CounterColumn(ByteBufferUtil.bytes("x"), 0L, 1L);
assert left.reconcile(right) == left;
// tombstone < live last delete
left = new DeletedColumn(ByteBufferUtil.bytes("x"), 1, 1L);
right = new CounterColumn(ByteBufferUtil.bytes("x"), 0L, 4L, 2L);
assert left.reconcile(right) == right;
// tombstone == live last delete
left = new DeletedColumn(ByteBufferUtil.bytes("x"), 1, 2L);
right = new CounterColumn(ByteBufferUtil.bytes("x"), 0L, 4L, 2L);
assert left.reconcile(right) == right;
// tombstone > live last delete
left = new DeletedColumn(ByteBufferUtil.bytes("x"), 1, 4L);
right = new CounterColumn(ByteBufferUtil.bytes("x"), 0L, 9L, 1L);
reconciled = left.reconcile(right);
assert reconciled.name() == right.name();
assert reconciled.value() == right.value();
assert reconciled.timestamp() == right.timestamp();
assert ((CounterColumn) reconciled).timestampOfLastDelete() == left.getMarkedForDeleteAt();
// live < tombstone
left = new CounterColumn(ByteBufferUtil.bytes("x"), 0L, 1L);
right = new DeletedColumn(ByteBufferUtil.bytes("x"), 1, 2L);
assert left.reconcile(right) == right;
// live last delete > tombstone
left = new CounterColumn(ByteBufferUtil.bytes("x"), 0L, 4L, 2L);
right = new DeletedColumn(ByteBufferUtil.bytes("x"), 1, 1L);
assert left.reconcile(right) == left;
// live last delete == tombstone
left = new CounterColumn(ByteBufferUtil.bytes("x"), 0L, 4L, 2L);
right = new DeletedColumn(ByteBufferUtil.bytes("x"), 1, 2L);
assert left.reconcile(right) == left;
// live last delete < tombstone
left = new CounterColumn(ByteBufferUtil.bytes("x"), 0L, 9L, 1L);
right = new DeletedColumn(ByteBufferUtil.bytes("x"), 1, 4L);
reconciled = left.reconcile(right);
assert reconciled.name() == left.name();
assert reconciled.value() == left.value();
assert reconciled.timestamp() == left.timestamp();
assert ((CounterColumn) reconciled).timestampOfLastDelete() == right.getMarkedForDeleteAt();
// live < live last delete
left =
new CounterColumn(
ByteBufferUtil.bytes("x"),
cc.create(NodeId.fromInt(1), 2L, 3L, false),
1L,
Long.MIN_VALUE);
right =
new CounterColumn(
ByteBufferUtil.bytes("x"), cc.create(NodeId.fromInt(1), 1L, 1L, false), 4L, 3L);
assert left.reconcile(right) == right;
// live last delete > live
left =
new CounterColumn(
ByteBufferUtil.bytes("x"), cc.create(NodeId.fromInt(1), 2L, 3L, false), 6L, 5L);
right =
new CounterColumn(
ByteBufferUtil.bytes("x"), cc.create(NodeId.fromInt(1), 1L, 1L, false), 4L, 3L);
assert left.reconcile(right) == left;
// live + live
left =
new CounterColumn(
ByteBufferUtil.bytes("x"),
cc.create(NodeId.fromInt(1), 1L, 1L, false),
4L,
Long.MIN_VALUE);
right =
new CounterColumn(
ByteBufferUtil.bytes("x"),
cc.create(NodeId.fromInt(1), 2L, 3L, false),
1L,
Long.MIN_VALUE);
reconciled = left.reconcile(right);
assert reconciled.name().equals(left.name());
assert ((CounterColumn) reconciled).total() == 3L;
assert reconciled.timestamp() == 4L;
left = reconciled;
right =
new CounterColumn(
ByteBufferUtil.bytes("x"),
cc.create(NodeId.fromInt(2), 1L, 5L, false),
2L,
Long.MIN_VALUE);
reconciled = left.reconcile(right);
assert reconciled.name().equals(left.name());
assert ((CounterColumn) reconciled).total() == 8L;
assert reconciled.timestamp() == 4L;
left = reconciled;
right =
new CounterColumn(
ByteBufferUtil.bytes("x"),
cc.create(NodeId.fromInt(2), 2L, 2L, false),
6L,
Long.MIN_VALUE);
reconciled = left.reconcile(right);
assert reconciled.name().equals(left.name());
assert ((CounterColumn) reconciled).total() == 5L;
assert reconciled.timestamp() == 6L;
context = reconciled.value();
int hd = 2; // header
assert hd + 2 * stepLength == context.remaining();
assert Util.equalsNodeId(NodeId.fromInt(1), context, hd + 0 * stepLength);
assert 2L == context.getLong(hd + 0 * stepLength + idLength);
assert 3L == context.getLong(hd + 0 * stepLength + idLength + clockLength);
assert Util.equalsNodeId(NodeId.fromInt(2), context, hd + 1 * stepLength);
assert 2L == context.getLong(hd + 1 * stepLength + idLength);
assert 2L == context.getLong(hd + 1 * stepLength + idLength + clockLength);
assert ((CounterColumn) reconciled).timestampOfLastDelete() == Long.MIN_VALUE;
}
public int compareTo(IColumn superColumn) {
return (name_.compareTo(superColumn.name()));
}
/**
* !BIGDATA: ************************************************************************ New
* serialized ColumnFamily row with column index. The new method to write column family row
* (column index and columns) to SSTable.
* ************************************************************************
*
* <p>the new format of ColumnFamily row: |------------------------| | bloom filter | (int for
* len, and content) |------------------------| | deletion meta | localDeletionTime(int) and
* markedForDeleteAt(long) |------------------------| | column count | (int)
* |------------------------|<-------|<---- COLUMN BLOCKS START POSITION(CBSP) | column block 0 |
* | BLOCK-INDEX OFFSET FROM CBSP | (compressed) | | |---------- ----------|<---| | | column block
* 1 | | | | (compressed) | | | |------------------------|<---|---|------| | column index size | |
* | | (byte size of column index) |------------------------| | | | | index of block 0 |----|----
* | |---------- ----------| | | | index of block 1 |----- | |------------------------| | | index
* size |---------------- to seek to position of index |------------------------|
*
* @param columnFamily
* @param dos
* @throws IOException
*/
public void bigdataSerializeWithIndexesAtEnd(
ColumnFamily columnFamily, DataOutputBuffer dos, Compression.Algorithm compressAlgo) {
// get the sorted columns from column family row
Collection<IColumn> columns = columnFamily.getSortedColumns();
// create and serialize bloom filter
BigdataColumnIndexer.createAndSerializeBloomFiliter(columns, dos);
/*
* Maintains a list of Column IndexInfo objects for the columns in this
* column family row. The key is the column name and the position is the
* relative offset of that column name from the start of the list.
* We do this so that we don't read all the columns into memory.
*/
List<IndexHelper.IndexInfo> indexList = new ArrayList<IndexHelper.IndexInfo>();
// different column family use different compression algorithm.
CompressionContext context = CompressionContext.getInstance(compressAlgo);
try {
// deletion meta information
dos.writeInt(columnFamily.localDeletionTime.get());
dos.writeLong(columnFamily.markedForDeleteAt.get());
// column count
dos.writeInt(columns.size());
// the current column
IColumn column = null;
// the size of serialized column index, computed up front
int indexSizeInBytes = 0;
// the position of first block, at where the column blocks start.
int firstBlockPos = dos.getLength();
// the first column of current block
IColumn blockFirstColumn = null;
// the start position of current block
// the column index will store the offset from firstBlockPos
int blockStartPos = firstBlockPos;
// uncompressed current block size
int blockSize = 0;
// compressed output stream of current block
DataOutputStream blockOut = null;
for (Iterator<IColumn> it = columns.iterator(); it.hasNext(); ) {
column = it.next();
if ((blockFirstColumn == null) && (blockOut == null)) {
// start a new block
blockFirstColumn = column;
blockStartPos = dos.getLength();
blockSize = 0;
// get a new block output stream
blockOut = getBlockOutputStream(dos, context);
}
// serialize column
columnFamily.getColumnSerializer().serialize(column, blockOut);
// uncompressed block size
blockSize += column.serializedSize();
// if we hit the block size that we have to index after, go ahead and index it.
if (blockSize >= DatabaseDescriptor.getColumnIndexSize()) {
int blockWidth = releaseBlockOutputStream(blockOut, context);
assert blockWidth == blockSize;
blockOut = null;
int blockEndPos = dos.getLength();
IndexHelper.IndexInfo cIndexInfo =
new IndexHelper.IndexInfo(
blockFirstColumn.name(),
column.name(),
blockStartPos - firstBlockPos,
blockWidth,
blockEndPos - blockStartPos);
indexList.add(cIndexInfo);
indexSizeInBytes += cIndexInfo.serializedSize();
// to next block
blockFirstColumn = null;
}
}
if (blockOut != null) {
int blockWidth = releaseBlockOutputStream(blockOut, context);
assert blockWidth == blockSize;
blockOut = null;
int blockEndPos = dos.getLength();
IndexHelper.IndexInfo cIndexInfo =
new IndexHelper.IndexInfo(
blockFirstColumn.name(),
column.name(),
blockStartPos - firstBlockPos,
blockWidth,
blockEndPos - blockStartPos);
indexList.add(cIndexInfo);
indexSizeInBytes += cIndexInfo.serializedSize();
}
// the start position of column index
int indexStartPos = dos.getLength();
// serialize column index
BigdataColumnIndexer.serialize(indexList, indexSizeInBytes, dos);
// write out the size of index.
dos.writeInt(dos.getLength() - indexStartPos);
} catch (IOException e) {
logger.error(e.toString());
throw new RuntimeException(e);
} finally {
context.releaseCompressor();
}
}
/**
* Return whether a given column is deleted by the container having this deletion info.
*
* @param column the column to check.
* @return true if the column is deleted, false otherwise
*/
public boolean isDeleted(IColumn column) {
return isDeleted(column.name(), column.mostRecentLiveChangeAt());
}
private void validateSliceLarge(ColumnFamilyStore cfStore) throws IOException {
String key = "row3";
ColumnFamily cf;
cf =
cfStore.getColumnFamily(
key,
new QueryPath("Standard1"),
"col1000".getBytes(),
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
3);
assertColumns(cf, "col1000", "col1001", "col1002");
assertEquals(new String(cf.getColumn("col1000".getBytes()).value()), "v1000");
assertEquals(new String(cf.getColumn("col1001".getBytes()).value()), "v1001");
assertEquals(new String(cf.getColumn("col1002".getBytes()).value()), "v1002");
cf =
cfStore.getColumnFamily(
key,
new QueryPath("Standard1"),
"col1195".getBytes(),
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
3);
assertColumns(cf, "col1195", "col1196", "col1197");
assertEquals(new String(cf.getColumn("col1195".getBytes()).value()), "v1195");
assertEquals(new String(cf.getColumn("col1196".getBytes()).value()), "v1196");
assertEquals(new String(cf.getColumn("col1197".getBytes()).value()), "v1197");
cf =
cfStore.getColumnFamily(
key,
new QueryPath("Standard1"),
"col1996".getBytes(),
ArrayUtils.EMPTY_BYTE_ARRAY,
true,
1000);
IColumn[] columns = cf.getSortedColumns().toArray(new IColumn[0]);
for (int i = 1000; i < 1996; i++) {
String expectedName = "col" + i;
IColumn column = columns[i - 1000];
assert Arrays.equals(column.name(), expectedName.getBytes())
: cfStore.getComparator().getString(column.name()) + " is not " + expectedName;
assert Arrays.equals(column.value(), ("v" + i).getBytes());
}
cf =
cfStore.getColumnFamily(
key,
new QueryPath("Standard1"),
"col1990".getBytes(),
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
3);
assertColumns(cf, "col1990", "col1991", "col1992");
assertEquals(new String(cf.getColumn("col1990".getBytes()).value()), "v1990");
assertEquals(new String(cf.getColumn("col1991".getBytes()).value()), "v1991");
assertEquals(new String(cf.getColumn("col1992".getBytes()).value()), "v1992");
cf =
cfStore.getColumnFamily(
key,
new QueryPath("Standard1"),
ArrayUtils.EMPTY_BYTE_ARRAY,
ArrayUtils.EMPTY_BYTE_ARRAY,
true,
3);
assertColumns(cf, "col1997", "col1998", "col1999");
assertEquals(new String(cf.getColumn("col1999".getBytes()).value()), "v1999");
assertEquals(new String(cf.getColumn("col1998".getBytes()).value()), "v1998");
assertEquals(new String(cf.getColumn("col1997".getBytes()).value()), "v1997");
cf =
cfStore.getColumnFamily(
key,
new QueryPath("Standard1"),
"col9000".getBytes(),
ArrayUtils.EMPTY_BYTE_ARRAY,
true,
3);
assertColumns(cf, "col1997", "col1998", "col1999");
cf =
cfStore.getColumnFamily(
key,
new QueryPath("Standard1"),
"col9000".getBytes(),
ArrayUtils.EMPTY_BYTE_ARRAY,
false,
3);
assertColumns(cf);
}