public SSTableSliceIterator(
SSTableReader ssTable, String key, byte[] startColumn, byte[] finishColumn, boolean reversed)
throws IOException {
this.reversed = reversed;
/* Morph key into actual key based on the partition type. */
DecoratedKey decoratedKey = ssTable.getPartitioner().decorateKey(key);
FileDataInput fdi =
ssTable.getFileDataInput(
decoratedKey, DatabaseDescriptor.getSlicedReadBufferSizeInKB() * 1024);
this.comparator = ssTable.getColumnComparator();
this.startColumn = startColumn;
this.finishColumn = finishColumn;
if (fdi != null) {
// BIGDATA: move up here
DecoratedKey keyInDisk = ssTable.getPartitioner().convertFromDiskFormat(fdi.readUTF());
assert keyInDisk.equals(decoratedKey)
: String.format("%s != %s in %s", keyInDisk, decoratedKey, fdi.getPath());
dataSize = fdi.readInt(); // row data size
dataStart = fdi.getAbsolutePosition();
rowFormat = ColumnFamily.serializer().deserializeRowFormat(fdi);
realDataStart = fdi.getAbsolutePosition();
// !BIGDATA: branch out to the different code process for new rowformat/compression.
if (ColumnFamily.serializer().isNewRowFormat(rowFormat)) {
// new row format
reader = new BigdataColumnGroupReader(ssTable, decoratedKey, fdi);
} else {
// old row format
reader = new ColumnGroupReader(ssTable, decoratedKey, fdi);
}
}
}
// BIGDATA: new serializeWithIndexes, to support old and new row format
public void serializeWithIndexes(
ColumnFamily columnFamily,
DataOutputBuffer headerDos,
DataOutputBuffer dos,
Compression.Algorithm compressAlgo) {
if ((compressAlgo != null)
&& ((DatabaseDescriptor.getCompressStartRowSize() <= 0)
|| columnFamily.isSizeLargeThan(DatabaseDescriptor.getCompressStartRowSize()))) {
boolean newIndexAtEnd = DatabaseDescriptor.isNewRowFormatIndexAtEnd();
// use the id of compressAlgo
serializeRowFormat(true, newIndexAtEnd, compressAlgo.getId(), headerDos);
if (newIndexAtEnd) bigdataSerializeWithIndexesAtEnd(columnFamily, dos, compressAlgo);
else bigdataSerializeWithIndexes(columnFamily, headerDos, dos, compressAlgo);
} else {
serializeRowFormat(false, false, 0, headerDos);
oldSerializeWithIndexes(columnFamily, dos);
}
}
public void saveRowCache() throws IOException {
Function<String, byte[]> function =
new Function<String, byte[]>() {
public byte[] apply(String key) {
return key.getBytes(Charset.forName("UTF-8"));
}
};
CacheWriter<String, ColumnFamily> writer = new CacheWriter<String, ColumnFamily>();
writer.saveCache(
rowCache, DatabaseDescriptor.getSerializedRowCachePath(ksname, cfname), function);
}
public void saveKeyCache() throws IOException {
Function<Pair<String, DecoratedKey>, byte[]> function =
new Function<Pair<String, DecoratedKey>, byte[]>() {
public byte[] apply(Pair<String, DecoratedKey> key) {
return key.right.key.getBytes(Charset.forName("UTF-8"));
}
};
CacheWriter<Pair<String, DecoratedKey>, SSTable.PositionSize> writer =
new CacheWriter<Pair<String, DecoratedKey>, SSTable.PositionSize>();
writer.saveCache(
keyCache, DatabaseDescriptor.getSerializedKeyCachePath(ksname, cfname), function);
}
/** Resizes the key and row caches based on the current key estimate. */
public synchronized void updateCacheSizes() {
long keys = estimatedKeys();
if (!keyCache.isCapacitySetManually()) {
int keyCacheSize = DatabaseDescriptor.getKeysCachedFor(ksname, cfname, keys);
if (keyCacheSize != keyCache.getCapacity()) {
// update cache size for the new key volume
if (logger.isDebugEnabled())
logger.debug("key cache capacity for " + cfname + " is " + keyCacheSize);
keyCache.updateCapacity(keyCacheSize);
}
}
if (!rowCache.isCapacitySetManually()) {
int rowCacheSize = DatabaseDescriptor.getRowsCachedFor(ksname, cfname, keys);
if (rowCacheSize != rowCache.getCapacity()) {
if (logger.isDebugEnabled())
logger.debug("row cache capacity for " + cfname + " is " + rowCacheSize);
rowCache.updateCapacity(rowCacheSize);
}
}
}
/**
* !BIGDATA: ************************************************************************ New
* serialized ColumnFamily row with column index. The new method to write column family row
* (column index and columns) to SSTable.
* ************************************************************************
*
* <p>the new format of ColumnFamily row: |------------------------| | bloom filter | (int for
* len, and content) |------------------------| | deletion meta | localDeletionTime(int) and
* markedForDeleteAt(long) |------------------------| | column count | (int)
* |------------------------|<-------|<---- COLUMN BLOCKS START POSITION(CBSP) | column block 0 |
* | BLOCK-INDEX OFFSET FROM CBSP | (compressed) | | |---------- ----------|<---| | | column block
* 1 | | | | (compressed) | | | |------------------------|<---|---|------| | column index size | |
* | | (byte size of column index) |------------------------| | | | | index of block 0 |----|----
* | |---------- ----------| | | | index of block 1 |----- | |------------------------| | | index
* size |---------------- to seek to position of index |------------------------|
*
* @param columnFamily
* @param dos
* @throws IOException
*/
public void bigdataSerializeWithIndexesAtEnd(
ColumnFamily columnFamily, DataOutputBuffer dos, Compression.Algorithm compressAlgo) {
// get the sorted columns from column family row
Collection<IColumn> columns = columnFamily.getSortedColumns();
// create and serialize bloom filter
BigdataColumnIndexer.createAndSerializeBloomFiliter(columns, dos);
/*
* Maintains a list of Column IndexInfo objects for the columns in this
* column family row. The key is the column name and the position is the
* relative offset of that column name from the start of the list.
* We do this so that we don't read all the columns into memory.
*/
List<IndexHelper.IndexInfo> indexList = new ArrayList<IndexHelper.IndexInfo>();
// different column family use different compression algorithm.
CompressionContext context = CompressionContext.getInstance(compressAlgo);
try {
// deletion meta information
dos.writeInt(columnFamily.localDeletionTime.get());
dos.writeLong(columnFamily.markedForDeleteAt.get());
// column count
dos.writeInt(columns.size());
// the current column
IColumn column = null;
// the size of serialized column index, computed up front
int indexSizeInBytes = 0;
// the position of first block, at where the column blocks start.
int firstBlockPos = dos.getLength();
// the first column of current block
IColumn blockFirstColumn = null;
// the start position of current block
// the column index will store the offset from firstBlockPos
int blockStartPos = firstBlockPos;
// uncompressed current block size
int blockSize = 0;
// compressed output stream of current block
DataOutputStream blockOut = null;
for (Iterator<IColumn> it = columns.iterator(); it.hasNext(); ) {
column = it.next();
if ((blockFirstColumn == null) && (blockOut == null)) {
// start a new block
blockFirstColumn = column;
blockStartPos = dos.getLength();
blockSize = 0;
// get a new block output stream
blockOut = getBlockOutputStream(dos, context);
}
// serialize column
columnFamily.getColumnSerializer().serialize(column, blockOut);
// uncompressed block size
blockSize += column.serializedSize();
// if we hit the block size that we have to index after, go ahead and index it.
if (blockSize >= DatabaseDescriptor.getColumnIndexSize()) {
int blockWidth = releaseBlockOutputStream(blockOut, context);
assert blockWidth == blockSize;
blockOut = null;
int blockEndPos = dos.getLength();
IndexHelper.IndexInfo cIndexInfo =
new IndexHelper.IndexInfo(
blockFirstColumn.name(),
column.name(),
blockStartPos - firstBlockPos,
blockWidth,
blockEndPos - blockStartPos);
indexList.add(cIndexInfo);
indexSizeInBytes += cIndexInfo.serializedSize();
// to next block
blockFirstColumn = null;
}
}
if (blockOut != null) {
int blockWidth = releaseBlockOutputStream(blockOut, context);
assert blockWidth == blockSize;
blockOut = null;
int blockEndPos = dos.getLength();
IndexHelper.IndexInfo cIndexInfo =
new IndexHelper.IndexInfo(
blockFirstColumn.name(),
column.name(),
blockStartPos - firstBlockPos,
blockWidth,
blockEndPos - blockStartPos);
indexList.add(cIndexInfo);
indexSizeInBytes += cIndexInfo.serializedSize();
}
// the start position of column index
int indexStartPos = dos.getLength();
// serialize column index
BigdataColumnIndexer.serialize(indexList, indexSizeInBytes, dos);
// write out the size of index.
dos.writeInt(dos.getLength() - indexStartPos);
} catch (IOException e) {
logger.error(e.toString());
throw new RuntimeException(e);
} finally {
context.releaseCompressor();
}
}
