@Override
public void write(Cell cell) throws IOException {
if (encryptor == null) {
super.write(cell);
return;
}
byte[] iv = nextIv();
encryptor.setIv(iv);
encryptor.reset();
// TODO: Check if this is a cell for an encrypted CF. If not, we can
// write a 0 here to signal an unwrapped cell and just dump the KV bytes
// afterward
StreamUtils.writeRawVInt32(out, iv.length);
out.write(iv);
// TODO: Add support for WAL compression
ByteArrayOutputStream baos = new ByteArrayOutputStream();
OutputStream cout = encryptor.createEncryptionStream(baos);
int tlen = cell.getTagsLength();
// Write the KeyValue infrastructure as VInts.
StreamUtils.writeRawVInt32(cout, KeyValueUtil.keyLength(cell));
StreamUtils.writeRawVInt32(cout, cell.getValueLength());
// To support tags
StreamUtils.writeRawVInt32(cout, tlen);
// Write row, qualifier, and family
StreamUtils.writeRawVInt32(cout, cell.getRowLength());
cout.write(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
StreamUtils.writeRawVInt32(cout, cell.getFamilyLength());
cout.write(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
StreamUtils.writeRawVInt32(cout, cell.getQualifierLength());
cout.write(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
// Write the rest ie. ts, type, value and tags parts
StreamUtils.writeLong(cout, cell.getTimestamp());
cout.write(cell.getTypeByte());
cout.write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
if (tlen > 0) {
cout.write(cell.getTagsArray(), cell.getTagsOffset(), tlen);
}
cout.close();
StreamUtils.writeRawVInt32(out, baos.size());
baos.writeTo(out);
// Increment IV given the final payload length
incrementIv(baos.size());
}
public static void main(String[] args) throws IOException {
Configuration conf = HBaseConfiguration.create();
HBaseHelper helper = HBaseHelper.getHelper(conf);
helper.dropTable("testtable");
helper.createTable("testtable", "colfam1", "colfam2");
System.out.println("Adding rows to table...");
helper.fillTable("testtable", 1, 10, 10, "colfam1", "colfam2");
Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(TableName.valueOf("testtable"));
// vv SingleColumnValueFilterExample
SingleColumnValueFilter filter =
new SingleColumnValueFilter(
Bytes.toBytes("colfam1"),
Bytes.toBytes("col-5"),
CompareFilter.CompareOp.NOT_EQUAL,
new SubstringComparator("val-5"));
filter.setFilterIfMissing(true);
Scan scan = new Scan();
scan.setFilter(filter);
ResultScanner scanner = table.getScanner(scan);
// ^^ SingleColumnValueFilterExample
System.out.println("Results of scan:");
// vv SingleColumnValueFilterExample
for (Result result : scanner) {
for (Cell cell : result.rawCells()) {
System.out.println(
"Cell: "
+ cell
+ ", Value: "
+ Bytes.toString(
cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
}
scanner.close();
Get get = new Get(Bytes.toBytes("row-6"));
get.setFilter(filter);
Result result = table.get(get);
System.out.println("Result of get: ");
for (Cell cell : result.rawCells()) {
System.out.println(
"Cell: "
+ cell
+ ", Value: "
+ Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
// ^^ SingleColumnValueFilterExample
}
@Override
public void write(Cell cell) throws IOException {
checkFlushed();
// Row
write(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
// Column family
write(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
// Qualifier
write(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
// Version
this.out.write(Bytes.toBytes(cell.getTimestamp()));
// Type
this.out.write(cell.getTypeByte());
// Value
write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
// Tags
write(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength());
// MvccVersion
this.out.write(Bytes.toBytes(cell.getMvccVersion()));
}
@Test(timeOut = 10_000)
public void testCorrectMapingOfCellsToShadowCells() throws IOException {
// Create the required data
final byte[] validShadowCellQualifier =
com.google.common.primitives.Bytes.concat(qualifier, SHADOW_CELL_SUFFIX);
final byte[] qualifier2 = Bytes.toBytes("test-qual2");
final byte[] validShadowCellQualifier2 =
com.google.common.primitives.Bytes.concat(qualifier2, SHADOW_CELL_SUFFIX);
final byte[] qualifier3 = Bytes.toBytes("test-qual3");
Cell cell1 =
new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put
Cell dupCell1 =
new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put
Cell dupCell1WithAnotherValue =
new KeyValue(row, family, qualifier, 1, Bytes.toBytes("other-value"));
Cell delCell1 = new KeyValue(row, family, qualifier, 1, Type.Delete, Bytes.toBytes("value"));
Cell shadowCell1 =
new KeyValue(row, family, validShadowCellQualifier, 1, Bytes.toBytes("sc-value"));
Cell cell2 = new KeyValue(row, family, qualifier2, 1, Bytes.toBytes("value2"));
Cell shadowCell2 =
new KeyValue(row, family, validShadowCellQualifier2, 1, Bytes.toBytes("sc-value2"));
Cell cell3 = new KeyValue(row, family, qualifier3, 1, Bytes.toBytes("value3"));
// Check a list of cells with duplicate values
List<Cell> badListWithDups = new ArrayList<>();
badListWithDups.add(cell1);
badListWithDups.add(dupCell1WithAnotherValue);
// Check dup shadow cell with same MVCC is ignored
SortedMap<Cell, Optional<Cell>> cellsToShadowCells =
CellUtils.mapCellsToShadowCells(badListWithDups);
assertEquals(cellsToShadowCells.size(), 1, "There should be only 1 key-value maps");
assertTrue(cellsToShadowCells.containsKey(cell1));
KeyValue firstKey = (KeyValue) cellsToShadowCells.firstKey();
KeyValue lastKey = (KeyValue) cellsToShadowCells.lastKey();
assertTrue(firstKey.equals(lastKey));
assertTrue(
0
== Bytes.compareTo(
firstKey.getValueArray(),
firstKey.getValueOffset(),
firstKey.getValueLength(),
cell1.getValueArray(),
cell1.getValueOffset(),
cell1.getValueLength()),
"Should be equal");
// Modify dup shadow cell to have a greater MVCC and check that is replaced
HBaseShims.setKeyValueSequenceId((KeyValue) dupCell1WithAnotherValue, 1);
cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups);
assertEquals(cellsToShadowCells.size(), 1, "There should be only 1 key-value maps");
assertTrue(cellsToShadowCells.containsKey(dupCell1WithAnotherValue));
firstKey = (KeyValue) cellsToShadowCells.firstKey();
lastKey = (KeyValue) cellsToShadowCells.lastKey();
assertTrue(firstKey.equals(lastKey));
assertTrue(
0
== Bytes.compareTo(
firstKey.getValueArray(),
firstKey.getValueOffset(),
firstKey.getValueLength(),
dupCell1WithAnotherValue.getValueArray(),
dupCell1WithAnotherValue.getValueOffset(),
dupCell1WithAnotherValue.getValueLength()),
"Should be equal");
// Check a list of cells with duplicate values
List<Cell> cellListWithDups = new ArrayList<>();
cellListWithDups.add(cell1);
cellListWithDups.add(shadowCell1);
cellListWithDups.add(dupCell1); // Dup cell
cellListWithDups.add(delCell1); // Another Dup cell but with different type
cellListWithDups.add(cell2);
cellListWithDups.add(cell3);
cellListWithDups.add(shadowCell2);
cellsToShadowCells = CellUtils.mapCellsToShadowCells(cellListWithDups);
assertEquals(cellsToShadowCells.size(), 3, "There should be only 3 key-value maps");
assertTrue(cellsToShadowCells.get(cell1).get().equals(shadowCell1));
assertTrue(cellsToShadowCells.get(dupCell1).get().equals(shadowCell1));
assertFalse(
cellsToShadowCells.containsKey(delCell1)); // TODO This is strange and needs to be solved.
// The current algo avoids to put the delete cell
// as key after the put cell with same value was added
assertTrue(cellsToShadowCells.get(cell2).get().equals(shadowCell2));
assertTrue(cellsToShadowCells.get(cell3).equals(Optional.absent()));
}