@Test(dataProvider = "shadow-cell-suffixes", timeOut = 10_000) public void testShadowCellSuffixRemovalFromQualifier(byte[] shadowCellSuffixToTest) throws IOException { // Test removal from a correclty suffixed qualifier byte[] suffixedQualifier = com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest); Cell cell = new KeyValue(row, family, suffixedQualifier, 1, Bytes.toBytes("value")); byte[] resultedQualifier = CellUtils.removeShadowCellSuffix( cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); byte[] expectedQualifier = qualifier; assertEquals(resultedQualifier, expectedQualifier); // Test removal from a badly suffixed qualifier byte[] badlySuffixedQualifier = com.google.common.primitives.Bytes.concat(qualifier, Bytes.toBytes("BAD")); Cell badCell = new KeyValue(row, family, badlySuffixedQualifier, 1, Bytes.toBytes("value")); try { CellUtils.removeShadowCellSuffix( badCell.getQualifierArray(), badCell.getQualifierOffset(), badCell.getQualifierLength()); fail(); } catch (IllegalArgumentException e) { // Expected } }
public static int compareCommonQualifierPrefix(Cell left, Cell right, int qualCommonPrefix) { return Bytes.compareTo( left.getQualifierArray(), left.getQualifierOffset() + qualCommonPrefix, left.getQualifierLength() - qualCommonPrefix, right.getQualifierArray(), right.getQualifierOffset() + qualCommonPrefix, right.getQualifierLength() - qualCommonPrefix); }
private static int findCommonPrefixInQualifierPart( Cell left, Cell right, int qualifierCommonPrefix) { return Bytes.findCommonPrefix( left.getQualifierArray(), right.getQualifierArray(), left.getQualifierLength() - qualifierCommonPrefix, right.getQualifierLength() - qualifierCommonPrefix, left.getQualifierOffset() + qualifierCommonPrefix, right.getQualifierOffset() + qualifierCommonPrefix); }
private int compareTypeBytes(Cell key, Cell right) { if (key.getFamilyLength() + key.getQualifierLength() == 0 && key.getTypeByte() == Type.Minimum.getCode()) { // left is "bigger", i.e. it appears later in the sorted order return 1; } if (right.getFamilyLength() + right.getQualifierLength() == 0 && right.getTypeByte() == Type.Minimum.getCode()) { return -1; } return 0; }
@Override public void write(Cell cell) throws IOException { if (encryptor == null) { super.write(cell); return; } byte[] iv = nextIv(); encryptor.setIv(iv); encryptor.reset(); // TODO: Check if this is a cell for an encrypted CF. If not, we can // write a 0 here to signal an unwrapped cell and just dump the KV bytes // afterward StreamUtils.writeRawVInt32(out, iv.length); out.write(iv); // TODO: Add support for WAL compression ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStream cout = encryptor.createEncryptionStream(baos); int tlen = cell.getTagsLength(); // Write the KeyValue infrastructure as VInts. StreamUtils.writeRawVInt32(cout, KeyValueUtil.keyLength(cell)); StreamUtils.writeRawVInt32(cout, cell.getValueLength()); // To support tags StreamUtils.writeRawVInt32(cout, tlen); // Write row, qualifier, and family StreamUtils.writeRawVInt32(cout, cell.getRowLength()); cout.write(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); StreamUtils.writeRawVInt32(cout, cell.getFamilyLength()); cout.write(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()); StreamUtils.writeRawVInt32(cout, cell.getQualifierLength()); cout.write(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); // Write the rest ie. ts, type, value and tags parts StreamUtils.writeLong(cout, cell.getTimestamp()); cout.write(cell.getTypeByte()); cout.write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); if (tlen > 0) { cout.write(cell.getTagsArray(), cell.getTagsOffset(), tlen); } cout.close(); StreamUtils.writeRawVInt32(out, baos.size()); baos.writeTo(out); // Increment IV given the final payload length incrementIv(baos.size()); }
@Override public ReturnCode filterKeyValue(Cell kv) { if (sortedPrefixes.size() == 0 || kv.getQualifierArray() == null) { return ReturnCode.INCLUDE; } else { return filterColumn(kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()); } }
/** * Find the column index which will replace the column name in the aggregated array and will be * restored in Reducer * * @param cell KeyValue for the column * @return column index for the specified cell or -1 if was not found */ private int findIndex(Cell cell) throws IOException { byte[] familyName = Bytes.copy(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()); byte[] name = Bytes.copy(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); byte[] cfn = Bytes.add(familyName, QueryConstants.NAMESPACE_SEPARATOR_BYTES, name); if (columnIndexes.containsKey(cfn)) { return columnIndexes.get(cfn); } return -1; }
@Test(timeOut = 10_000) public void testShadowCellSuffixConcatenationToQualifier() { Cell cell = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); byte[] suffixedQualifier = CellUtils.addShadowCellSuffix( cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); byte[] expectedQualifier = com.google.common.primitives.Bytes.concat(qualifier, SHADOW_CELL_SUFFIX); assertEquals(suffixedQualifier, expectedQualifier); }
@Override public void write(Cell cell) throws IOException { checkFlushed(); // Row write(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); // Column family write(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()); // Qualifier write(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); // Version this.out.write(Bytes.toBytes(cell.getTimestamp())); // Type this.out.write(cell.getTypeByte()); // Value write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); // Tags write(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); // MvccVersion this.out.write(Bytes.toBytes(cell.getMvccVersion())); }
@Override public ReturnCode filterKeyValue(Cell kv) { // TODO have a column compare method in Cell byte[] buffer = kv.getQualifierArray(); int qualifierOffset = kv.getQualifierOffset(); int qualifierLength = kv.getQualifierLength(); int cmpMin = 1; if (this.minColumn != null) { cmpMin = Bytes.compareTo( buffer, qualifierOffset, qualifierLength, this.minColumn, 0, this.minColumn.length); } if (cmpMin < 0) { return ReturnCode.SEEK_NEXT_USING_HINT; } if (!this.minColumnInclusive && cmpMin == 0) { return ReturnCode.SKIP; } if (this.maxColumn == null) { return ReturnCode.INCLUDE; } int cmpMax = Bytes.compareTo( buffer, qualifierOffset, qualifierLength, this.maxColumn, 0, this.maxColumn.length); if (this.maxColumnInclusive && cmpMax <= 0 || !this.maxColumnInclusive && cmpMax < 0) { return ReturnCode.INCLUDE; } return ReturnCode.NEXT_ROW; }