@Test public void testReadingExistingVersion1HFile() throws IOException { URL url = TestHFileReaderV1.class.getResource("8e8ab58dcf39412da19833fcd8f687ac"); Path existingHFilePath = new Path(url.getPath()); HFile.Reader reader = HFile.createReader(fs, existingHFilePath, new CacheConfig(conf)); reader.loadFileInfo(); FixedFileTrailer trailer = reader.getTrailer(); assertEquals(N, reader.getEntries()); assertEquals(N, trailer.getEntryCount()); assertEquals(1, trailer.getMajorVersion()); assertEquals(Compression.Algorithm.GZ, trailer.getCompressionCodec()); for (boolean pread : new boolean[] {false, true}) { int totalDataSize = 0; int n = 0; HFileScanner scanner = reader.getScanner(false, pread); assertTrue(scanner.seekTo()); do { totalDataSize += scanner.getKey().limit() + scanner.getValue().limit() + Bytes.SIZEOF_INT * 2; ++n; } while (scanner.next()); // Add magic record sizes, one per data block. totalDataSize += 8 * trailer.getDataIndexCount(); assertEquals(N, n); assertEquals(trailer.getTotalUncompressedBytes(), totalDataSize); } reader.close(); }
@Test(timeout = 6000000) public void testHFileEncryption() throws Exception { // Create 1000 random test KVs RedundantKVGenerator generator = new RedundantKVGenerator(); List<KeyValue> testKvs = generator.generateTestKeyValues(1000); // Iterate through data block encoding and compression combinations Configuration conf = TEST_UTIL.getConfiguration(); CacheConfig cacheConf = new CacheConfig(conf); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { for (Compression.Algorithm compression : TestHFileBlock.COMPRESSION_ALGORITHMS) { HFileContext fileContext = new HFileContextBuilder() .withBlockSize(4096) // small blocks .withEncryptionContext(cryptoContext) .withCompression(compression) .withDataBlockEncoding(encoding) .build(); // write a new test HFile LOG.info("Writing with " + fileContext); Path path = new Path(TEST_UTIL.getDataTestDir(), UUID.randomUUID().toString() + ".hfile"); FSDataOutputStream out = fs.create(path); HFile.Writer writer = HFile.getWriterFactory(conf, cacheConf) .withOutputStream(out) .withFileContext(fileContext) .create(); try { for (KeyValue kv : testKvs) { writer.append(kv); } } finally { writer.close(); out.close(); } // read it back in LOG.info("Reading with " + fileContext); int i = 0; HFileScanner scanner = null; HFile.Reader reader = HFile.createReader(fs, path, cacheConf, conf); try { reader.loadFileInfo(); FixedFileTrailer trailer = reader.getTrailer(); assertNotNull(trailer.getEncryptionKey()); scanner = reader.getScanner(false, false); assertTrue("Initial seekTo failed", scanner.seekTo()); do { Cell kv = scanner.getCell(); assertTrue( "Read back an unexpected or invalid KV", testKvs.contains(KeyValueUtil.ensureKeyValue(kv))); i++; } while (scanner.next()); } finally { reader.close(); scanner.close(); } assertEquals("Did not read back as many KVs as written", i, testKvs.size()); // Test random seeks with pread LOG.info("Random seeking with " + fileContext); reader = HFile.createReader(fs, path, cacheConf, conf); try { scanner = reader.getScanner(false, true); assertTrue("Initial seekTo failed", scanner.seekTo()); for (i = 0; i < 100; i++) { KeyValue kv = testKvs.get(RNG.nextInt(testKvs.size())); assertEquals("Unable to find KV as expected: " + kv, scanner.seekTo(kv), 0); } } finally { scanner.close(); reader.close(); } } } }