@Test(dataProvider = "shadow-cell-suffixes", timeOut = 10_000)
public void testShadowCellSuffixRemovalFromQualifier(byte[] shadowCellSuffixToTest)
throws IOException {
// Test removal from a correclty suffixed qualifier
byte[] suffixedQualifier =
com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest);
Cell cell = new KeyValue(row, family, suffixedQualifier, 1, Bytes.toBytes("value"));
byte[] resultedQualifier =
CellUtils.removeShadowCellSuffix(
cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
byte[] expectedQualifier = qualifier;
assertEquals(resultedQualifier, expectedQualifier);
// Test removal from a badly suffixed qualifier
byte[] badlySuffixedQualifier =
com.google.common.primitives.Bytes.concat(qualifier, Bytes.toBytes("BAD"));
Cell badCell = new KeyValue(row, family, badlySuffixedQualifier, 1, Bytes.toBytes("value"));
try {
CellUtils.removeShadowCellSuffix(
badCell.getQualifierArray(), badCell.getQualifierOffset(), badCell.getQualifierLength());
fail();
} catch (IllegalArgumentException e) {
// Expected
}
}
public static int compareCommonQualifierPrefix(Cell left, Cell right, int qualCommonPrefix) {
return Bytes.compareTo(
left.getQualifierArray(),
left.getQualifierOffset() + qualCommonPrefix,
left.getQualifierLength() - qualCommonPrefix,
right.getQualifierArray(),
right.getQualifierOffset() + qualCommonPrefix,
right.getQualifierLength() - qualCommonPrefix);
}
private static int findCommonPrefixInQualifierPart(
Cell left, Cell right, int qualifierCommonPrefix) {
return Bytes.findCommonPrefix(
left.getQualifierArray(),
right.getQualifierArray(),
left.getQualifierLength() - qualifierCommonPrefix,
right.getQualifierLength() - qualifierCommonPrefix,
left.getQualifierOffset() + qualifierCommonPrefix,
right.getQualifierOffset() + qualifierCommonPrefix);
}
private int compareTypeBytes(Cell key, Cell right) {
if (key.getFamilyLength() + key.getQualifierLength() == 0
&& key.getTypeByte() == Type.Minimum.getCode()) {
// left is "bigger", i.e. it appears later in the sorted order
return 1;
}
if (right.getFamilyLength() + right.getQualifierLength() == 0
&& right.getTypeByte() == Type.Minimum.getCode()) {
return -1;
}
return 0;
}
@Override
public void write(Cell cell) throws IOException {
if (encryptor == null) {
super.write(cell);
return;
}
byte[] iv = nextIv();
encryptor.setIv(iv);
encryptor.reset();
// TODO: Check if this is a cell for an encrypted CF. If not, we can
// write a 0 here to signal an unwrapped cell and just dump the KV bytes
// afterward
StreamUtils.writeRawVInt32(out, iv.length);
out.write(iv);
// TODO: Add support for WAL compression
ByteArrayOutputStream baos = new ByteArrayOutputStream();
OutputStream cout = encryptor.createEncryptionStream(baos);
int tlen = cell.getTagsLength();
// Write the KeyValue infrastructure as VInts.
StreamUtils.writeRawVInt32(cout, KeyValueUtil.keyLength(cell));
StreamUtils.writeRawVInt32(cout, cell.getValueLength());
// To support tags
StreamUtils.writeRawVInt32(cout, tlen);
// Write row, qualifier, and family
StreamUtils.writeRawVInt32(cout, cell.getRowLength());
cout.write(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
StreamUtils.writeRawVInt32(cout, cell.getFamilyLength());
cout.write(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
StreamUtils.writeRawVInt32(cout, cell.getQualifierLength());
cout.write(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
// Write the rest ie. ts, type, value and tags parts
StreamUtils.writeLong(cout, cell.getTimestamp());
cout.write(cell.getTypeByte());
cout.write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
if (tlen > 0) {
cout.write(cell.getTagsArray(), cell.getTagsOffset(), tlen);
}
cout.close();
StreamUtils.writeRawVInt32(out, baos.size());
baos.writeTo(out);
// Increment IV given the final payload length
incrementIv(baos.size());
}
@Override
public ReturnCode filterKeyValue(Cell kv) {
if (sortedPrefixes.size() == 0 || kv.getQualifierArray() == null) {
return ReturnCode.INCLUDE;
} else {
return filterColumn(kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength());
}
}
/**
* Find the column index which will replace the column name in the aggregated array and will be
* restored in Reducer
*
* @param cell KeyValue for the column
* @return column index for the specified cell or -1 if was not found
*/
private int findIndex(Cell cell) throws IOException {
byte[] familyName =
Bytes.copy(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
byte[] name =
Bytes.copy(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
byte[] cfn = Bytes.add(familyName, QueryConstants.NAMESPACE_SEPARATOR_BYTES, name);
if (columnIndexes.containsKey(cfn)) {
return columnIndexes.get(cfn);
}
return -1;
}
@Test(timeOut = 10_000)
public void testShadowCellSuffixConcatenationToQualifier() {
Cell cell = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
byte[] suffixedQualifier =
CellUtils.addShadowCellSuffix(
cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
byte[] expectedQualifier =
com.google.common.primitives.Bytes.concat(qualifier, SHADOW_CELL_SUFFIX);
assertEquals(suffixedQualifier, expectedQualifier);
}
@Override
public void write(Cell cell) throws IOException {
checkFlushed();
// Row
write(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
// Column family
write(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
// Qualifier
write(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
// Version
this.out.write(Bytes.toBytes(cell.getTimestamp()));
// Type
this.out.write(cell.getTypeByte());
// Value
write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
// Tags
write(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength());
// MvccVersion
this.out.write(Bytes.toBytes(cell.getMvccVersion()));
}
@Override
public ReturnCode filterKeyValue(Cell kv) {
// TODO have a column compare method in Cell
byte[] buffer = kv.getQualifierArray();
int qualifierOffset = kv.getQualifierOffset();
int qualifierLength = kv.getQualifierLength();
int cmpMin = 1;
if (this.minColumn != null) {
cmpMin =
Bytes.compareTo(
buffer, qualifierOffset, qualifierLength, this.minColumn, 0, this.minColumn.length);
}
if (cmpMin < 0) {
return ReturnCode.SEEK_NEXT_USING_HINT;
}
if (!this.minColumnInclusive && cmpMin == 0) {
return ReturnCode.SKIP;
}
if (this.maxColumn == null) {
return ReturnCode.INCLUDE;
}
int cmpMax =
Bytes.compareTo(
buffer, qualifierOffset, qualifierLength, this.maxColumn, 0, this.maxColumn.length);
if (this.maxColumnInclusive && cmpMax <= 0 || !this.maxColumnInclusive && cmpMax < 0) {
return ReturnCode.INCLUDE;
}
return ReturnCode.NEXT_ROW;
}