Esempio n. 1
0
  @Test
  public void testReadingExistingVersion1HFile() throws IOException {
    URL url = TestHFileReaderV1.class.getResource("8e8ab58dcf39412da19833fcd8f687ac");
    Path existingHFilePath = new Path(url.getPath());
    HFile.Reader reader = HFile.createReader(fs, existingHFilePath, new CacheConfig(conf));
    reader.loadFileInfo();
    FixedFileTrailer trailer = reader.getTrailer();

    assertEquals(N, reader.getEntries());
    assertEquals(N, trailer.getEntryCount());
    assertEquals(1, trailer.getMajorVersion());
    assertEquals(Compression.Algorithm.GZ, trailer.getCompressionCodec());

    for (boolean pread : new boolean[] {false, true}) {
      int totalDataSize = 0;
      int n = 0;

      HFileScanner scanner = reader.getScanner(false, pread);
      assertTrue(scanner.seekTo());
      do {
        totalDataSize +=
            scanner.getKey().limit() + scanner.getValue().limit() + Bytes.SIZEOF_INT * 2;
        ++n;
      } while (scanner.next());

      // Add magic record sizes, one per data block.
      totalDataSize += 8 * trailer.getDataIndexCount();

      assertEquals(N, n);
      assertEquals(trailer.getTotalUncompressedBytes(), totalDataSize);
    }
    reader.close();
  }
Esempio n. 2
0
  @Test(timeout = 20000)
  public void testHFileEncryptionMetadata() throws Exception {
    Configuration conf = TEST_UTIL.getConfiguration();
    CacheConfig cacheConf = new CacheConfig(conf);
    HFileContext fileContext =
        new HFileContextBuilder().withEncryptionContext(cryptoContext).build();

    // write a simple encrypted hfile
    Path path = new Path(TEST_UTIL.getDataTestDir(), "cryptometa.hfile");
    FSDataOutputStream out = fs.create(path);
    HFile.Writer writer =
        HFile.getWriterFactory(conf, cacheConf)
            .withOutputStream(out)
            .withFileContext(fileContext)
            .create();
    try {
      KeyValue kv = new KeyValue("foo".getBytes(), "f1".getBytes(), null, "value".getBytes());
      writer.append(kv);
    } finally {
      writer.close();
      out.close();
    }

    // read it back in and validate correct crypto metadata
    HFile.Reader reader = HFile.createReader(fs, path, cacheConf, conf);
    try {
      reader.loadFileInfo();
      FixedFileTrailer trailer = reader.getTrailer();
      assertNotNull(trailer.getEncryptionKey());
      Encryption.Context readerContext = reader.getFileContext().getEncryptionContext();
      assertEquals(readerContext.getCipher().getName(), cryptoContext.getCipher().getName());
      assertTrue(Bytes.equals(readerContext.getKeyBytes(), cryptoContext.getKeyBytes()));
    } finally {
      reader.close();
    }
  }
Esempio n. 3
0
  @Test(timeout = 6000000)
  public void testHFileEncryption() throws Exception {
    // Create 1000 random test KVs
    RedundantKVGenerator generator = new RedundantKVGenerator();
    List<KeyValue> testKvs = generator.generateTestKeyValues(1000);

    // Iterate through data block encoding and compression combinations
    Configuration conf = TEST_UTIL.getConfiguration();
    CacheConfig cacheConf = new CacheConfig(conf);
    for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
      for (Compression.Algorithm compression : TestHFileBlock.COMPRESSION_ALGORITHMS) {
        HFileContext fileContext =
            new HFileContextBuilder()
                .withBlockSize(4096) // small blocks
                .withEncryptionContext(cryptoContext)
                .withCompression(compression)
                .withDataBlockEncoding(encoding)
                .build();
        // write a new test HFile
        LOG.info("Writing with " + fileContext);
        Path path = new Path(TEST_UTIL.getDataTestDir(), UUID.randomUUID().toString() + ".hfile");
        FSDataOutputStream out = fs.create(path);
        HFile.Writer writer =
            HFile.getWriterFactory(conf, cacheConf)
                .withOutputStream(out)
                .withFileContext(fileContext)
                .create();
        try {
          for (KeyValue kv : testKvs) {
            writer.append(kv);
          }
        } finally {
          writer.close();
          out.close();
        }

        // read it back in
        LOG.info("Reading with " + fileContext);
        int i = 0;
        HFileScanner scanner = null;
        HFile.Reader reader = HFile.createReader(fs, path, cacheConf, conf);
        try {
          reader.loadFileInfo();
          FixedFileTrailer trailer = reader.getTrailer();
          assertNotNull(trailer.getEncryptionKey());
          scanner = reader.getScanner(false, false);
          assertTrue("Initial seekTo failed", scanner.seekTo());
          do {
            Cell kv = scanner.getCell();
            assertTrue(
                "Read back an unexpected or invalid KV",
                testKvs.contains(KeyValueUtil.ensureKeyValue(kv)));
            i++;
          } while (scanner.next());
        } finally {
          reader.close();
          scanner.close();
        }

        assertEquals("Did not read back as many KVs as written", i, testKvs.size());

        // Test random seeks with pread
        LOG.info("Random seeking with " + fileContext);
        reader = HFile.createReader(fs, path, cacheConf, conf);
        try {
          scanner = reader.getScanner(false, true);
          assertTrue("Initial seekTo failed", scanner.seekTo());
          for (i = 0; i < 100; i++) {
            KeyValue kv = testKvs.get(RNG.nextInt(testKvs.size()));
            assertEquals("Unable to find KV as expected: " + kv, scanner.seekTo(kv), 0);
          }
        } finally {
          scanner.close();
          reader.close();
        }
      }
    }
  }