/**
* @param encoding encoding used
* @param headerBytes dummy header bytes
* @param fileContext HFile meta data
*/
public HFileBlockDefaultEncodingContext(
DataBlockEncoding encoding, byte[] headerBytes, HFileContext fileContext) {
this.encodingAlgo = encoding;
this.fileContext = fileContext;
Compression.Algorithm compressionAlgorithm =
fileContext.getCompression() == null ? NONE : fileContext.getCompression();
if (compressionAlgorithm != NONE) {
compressor = compressionAlgorithm.getCompressor();
compressedByteStream = new ByteArrayOutputStream();
try {
compressionStream =
compressionAlgorithm.createPlainCompressionStream(compressedByteStream, compressor);
} catch (IOException e) {
throw new RuntimeException(
"Could not create compression stream for algorithm " + compressionAlgorithm, e);
}
}
Encryption.Context cryptoContext = fileContext.getEncryptionContext();
if (cryptoContext != Encryption.Context.NONE) {
cryptoByteStream = new ByteArrayOutputStream();
iv = new byte[cryptoContext.getCipher().getIvLength()];
new SecureRandom().nextBytes(iv);
}
dummyHeader =
Preconditions.checkNotNull(
headerBytes,
"Please pass HConstants.HFILEBLOCK_DUMMY_HEADER instead of null for param headerBytes");
}
@BeforeClass
public static void setUp() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
// Disable block cache in this test.
conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
conf.setInt("hfile.format.version", 3);
fs = FileSystem.get(conf);
cryptoContext = Encryption.newContext(conf);
String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
Cipher aes = Encryption.getCipher(conf, algorithm);
assertNotNull(aes);
cryptoContext.setCipher(aes);
byte[] key = new byte[aes.getKeyLength()];
RNG.nextBytes(key);
cryptoContext.setKey(key);
}
@Test(timeout = 20000)
public void testHFileEncryptionMetadata() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
CacheConfig cacheConf = new CacheConfig(conf);
HFileContext fileContext =
new HFileContextBuilder().withEncryptionContext(cryptoContext).build();
// write a simple encrypted hfile
Path path = new Path(TEST_UTIL.getDataTestDir(), "cryptometa.hfile");
FSDataOutputStream out = fs.create(path);
HFile.Writer writer =
HFile.getWriterFactory(conf, cacheConf)
.withOutputStream(out)
.withFileContext(fileContext)
.create();
try {
KeyValue kv = new KeyValue("foo".getBytes(), "f1".getBytes(), null, "value".getBytes());
writer.append(kv);
} finally {
writer.close();
out.close();
}
// read it back in and validate correct crypto metadata
HFile.Reader reader = HFile.createReader(fs, path, cacheConf, conf);
try {
reader.loadFileInfo();
FixedFileTrailer trailer = reader.getTrailer();
assertNotNull(trailer.getEncryptionKey());
Encryption.Context readerContext = reader.getFileContext().getEncryptionContext();
assertEquals(readerContext.getCipher().getName(), cryptoContext.getCipher().getName());
assertTrue(Bytes.equals(readerContext.getKeyBytes(), cryptoContext.getKeyBytes()));
} finally {
reader.close();
}
}
/**
* @param uncompressedBytesWithHeader
* @param blockType
* @param headerBytes
* @throws IOException
*/
protected void compressAfterEncoding(
byte[] uncompressedBytesWithHeader, BlockType blockType, byte[] headerBytes)
throws IOException {
this.uncompressedBytesWithHeader = uncompressedBytesWithHeader;
Encryption.Context cryptoContext = fileContext.getEncryptionContext();
if (cryptoContext != Encryption.Context.NONE) {
// Encrypted block format:
// +--------------------------+
// | byte iv length |
// +--------------------------+
// | iv data ... |
// +--------------------------+
// | encrypted block data ... |
// +--------------------------+
cryptoByteStream.reset();
// Write the block header (plaintext)
cryptoByteStream.write(headerBytes);
InputStream in;
int plaintextLength;
// Run any compression before encryption
if (fileContext.getCompression() != Compression.Algorithm.NONE) {
compressedByteStream.reset();
compressionStream.resetState();
compressionStream.write(
uncompressedBytesWithHeader,
headerBytes.length,
uncompressedBytesWithHeader.length - headerBytes.length);
compressionStream.flush();
compressionStream.finish();
byte[] plaintext = compressedByteStream.toByteArray();
plaintextLength = plaintext.length;
in = new ByteArrayInputStream(plaintext);
} else {
plaintextLength = uncompressedBytesWithHeader.length - headerBytes.length;
in =
new ByteArrayInputStream(
uncompressedBytesWithHeader, headerBytes.length, plaintextLength);
}
if (plaintextLength > 0) {
// Set up the cipher
Cipher cipher = cryptoContext.getCipher();
Encryptor encryptor = cipher.getEncryptor();
encryptor.setKey(cryptoContext.getKey());
// Set up the IV
int ivLength = iv.length;
Preconditions.checkState(ivLength <= Byte.MAX_VALUE, "IV length out of range");
cryptoByteStream.write(ivLength);
if (ivLength > 0) {
encryptor.setIv(iv);
cryptoByteStream.write(iv);
}
// Encrypt the data
Encryption.encrypt(cryptoByteStream, in, encryptor);
onDiskBytesWithHeader = cryptoByteStream.toByteArray();
// Increment the IV given the final block size
Encryption.incrementIv(iv, 1 + (onDiskBytesWithHeader.length / encryptor.getBlockSize()));
} else {
cryptoByteStream.write(0);
onDiskBytesWithHeader = cryptoByteStream.toByteArray();
}
} else {
if (this.fileContext.getCompression() != NONE) {
compressedByteStream.reset();
compressedByteStream.write(headerBytes);
compressionStream.resetState();
compressionStream.write(
uncompressedBytesWithHeader,
headerBytes.length,
uncompressedBytesWithHeader.length - headerBytes.length);
compressionStream.flush();
compressionStream.finish();
onDiskBytesWithHeader = compressedByteStream.toByteArray();
} else {
onDiskBytesWithHeader = uncompressedBytesWithHeader;
}
}
this.blockType = blockType;
}