/**
* Closes an HFile writer.
*
* @param hfileWriter The writer to close.
* @throws IOException If there is an error.
*/
private void closeWriter(HFile.Writer hfileWriter) throws IOException {
LOG.info("Closing HFile " + hfileWriter.getPath());
// Write file metadata:
hfileWriter.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, toBytes(mLatestTimestamp));
final String taskAttemptID = mContext.getTaskAttemptID().toString();
hfileWriter.appendFileInfo(StoreFile.BULKLOAD_TASK_KEY, toBytes(taskAttemptID));
hfileWriter.appendFileInfo(StoreFile.MAJOR_COMPACTION_KEY, toBytes(true));
ResourceUtils.closeOrLog(hfileWriter);
}
/** {@inheritDoc} */
@Override
public void write(HFileKeyValue entry, NullWritable unused) throws IOException {
final KeyValue kv = entry.getKeyValue();
kv.updateLatestStamp(mLatestTimestampBytes);
final long recordLength = kv.getLength();
if (mCurrentHFileSize + recordLength >= mMaxFileSizeBytes) {
// We can't fit this record in the current HFile without exceeding the max file size.
if (Arrays.equals(mCurrentRow, kv.getRow())) {
// But we're still adding data for a single row, so we can't close this HFile yet.
LOG.debug("Reached max HFile size, but waiting to finish this row before closing.");
} else {
// Close it and open a new one.
closeWriter(mWriter);
mWriter = openNewWriter();
}
}
mWriter.append(kv);
mTimeRangeTracker.includeTimestamp(kv);
mCurrentHFileSize += recordLength;
// Remember the row so we know when we are transitioning.
mCurrentRow = kv.getRow();
}
private static void createHFile(
Configuration conf, FileSystem fs, Path path, byte[] family, byte[] qualifier)
throws IOException {
HFileContext context = new HFileContextBuilder().build();
HFile.Writer writer =
HFile.getWriterFactory(conf, new CacheConfig(conf))
.withPath(fs, path)
.withFileContext(context)
.create();
long now = System.currentTimeMillis();
try {
for (int i = 1; i <= 9; i++) {
KeyValue kv =
new KeyValue(Bytes.toBytes(i + ""), family, qualifier, now, Bytes.toBytes(i + ""));
writer.append(kv);
}
} finally {
writer.close();
}
}
/** {@inheritDoc} */
@Override
public void close(TaskAttemptContext context) throws IOException {
if (null != mWriter) {
if (mCurrentHFileSize > 0) {
// Write out a timerange. This is the only Metadata we write out.
// See: HBASE-8055 and KIJIMR-204.
mWriter.appendFileInfo(
StoreFile.TIMERANGE_KEY, WritableUtils.toByteArray(mTimeRangeTracker));
}
mTimeRangeTracker = new TimeRangeTracker();
closeWriter(mWriter);
}
}
/** Create an HFile with the given number of rows with a specified value. */
public static void createHFile(
FileSystem fs, Path path, byte[] family, byte[] qualifier, byte[] value, int numRows)
throws IOException {
HFileContext context =
new HFileContextBuilder().withBlockSize(BLOCKSIZE).withCompression(COMPRESSION).build();
HFile.Writer writer =
HFile.getWriterFactory(conf, new CacheConfig(conf))
.withPath(fs, path)
.withFileContext(context)
.create();
long now = System.currentTimeMillis();
try {
// subtract 2 since iterateOnSplits doesn't include boundary keys
for (int i = 0; i < numRows; i++) {
KeyValue kv = new KeyValue(rowkey(i), family, qualifier, now, value);
writer.append(kv);
}
writer.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(now));
} finally {
writer.close();
}
}
@Test(timeout = 20000)
public void testHFileEncryptionMetadata() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
CacheConfig cacheConf = new CacheConfig(conf);
HFileContext fileContext =
new HFileContextBuilder().withEncryptionContext(cryptoContext).build();
// write a simple encrypted hfile
Path path = new Path(TEST_UTIL.getDataTestDir(), "cryptometa.hfile");
FSDataOutputStream out = fs.create(path);
HFile.Writer writer =
HFile.getWriterFactory(conf, cacheConf)
.withOutputStream(out)
.withFileContext(fileContext)
.create();
try {
KeyValue kv = new KeyValue("foo".getBytes(), "f1".getBytes(), null, "value".getBytes());
writer.append(kv);
} finally {
writer.close();
out.close();
}
// read it back in and validate correct crypto metadata
HFile.Reader reader = HFile.createReader(fs, path, cacheConf, conf);
try {
reader.loadFileInfo();
FixedFileTrailer trailer = reader.getTrailer();
assertNotNull(trailer.getEncryptionKey());
Encryption.Context readerContext = reader.getFileContext().getEncryptionContext();
assertEquals(readerContext.getCipher().getName(), cryptoContext.getCipher().getName());
assertTrue(Bytes.equals(readerContext.getKeyBytes(), cryptoContext.getKeyBytes()));
} finally {
reader.close();
}
}
@Test(timeout = 6000000)
public void testHFileEncryption() throws Exception {
// Create 1000 random test KVs
RedundantKVGenerator generator = new RedundantKVGenerator();
List<KeyValue> testKvs = generator.generateTestKeyValues(1000);
// Iterate through data block encoding and compression combinations
Configuration conf = TEST_UTIL.getConfiguration();
CacheConfig cacheConf = new CacheConfig(conf);
for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
for (Compression.Algorithm compression : TestHFileBlock.COMPRESSION_ALGORITHMS) {
HFileContext fileContext =
new HFileContextBuilder()
.withBlockSize(4096) // small blocks
.withEncryptionContext(cryptoContext)
.withCompression(compression)
.withDataBlockEncoding(encoding)
.build();
// write a new test HFile
LOG.info("Writing with " + fileContext);
Path path = new Path(TEST_UTIL.getDataTestDir(), UUID.randomUUID().toString() + ".hfile");
FSDataOutputStream out = fs.create(path);
HFile.Writer writer =
HFile.getWriterFactory(conf, cacheConf)
.withOutputStream(out)
.withFileContext(fileContext)
.create();
try {
for (KeyValue kv : testKvs) {
writer.append(kv);
}
} finally {
writer.close();
out.close();
}
// read it back in
LOG.info("Reading with " + fileContext);
int i = 0;
HFileScanner scanner = null;
HFile.Reader reader = HFile.createReader(fs, path, cacheConf, conf);
try {
reader.loadFileInfo();
FixedFileTrailer trailer = reader.getTrailer();
assertNotNull(trailer.getEncryptionKey());
scanner = reader.getScanner(false, false);
assertTrue("Initial seekTo failed", scanner.seekTo());
do {
Cell kv = scanner.getCell();
assertTrue(
"Read back an unexpected or invalid KV",
testKvs.contains(KeyValueUtil.ensureKeyValue(kv)));
i++;
} while (scanner.next());
} finally {
reader.close();
scanner.close();
}
assertEquals("Did not read back as many KVs as written", i, testKvs.size());
// Test random seeks with pread
LOG.info("Random seeking with " + fileContext);
reader = HFile.createReader(fs, path, cacheConf, conf);
try {
scanner = reader.getScanner(false, true);
assertTrue("Initial seekTo failed", scanner.seekTo());
for (i = 0; i < 100; i++) {
KeyValue kv = testKvs.get(RNG.nextInt(testKvs.size()));
assertEquals("Unable to find KV as expected: " + kv, scanner.seekTo(kv), 0);
}
} finally {
scanner.close();
reader.close();
}
}
}
}