/**
   * @param encoding encoding used
   * @param headerBytes dummy header bytes
   * @param fileContext HFile meta data
   */
  public HFileBlockDefaultEncodingContext(
      DataBlockEncoding encoding, byte[] headerBytes, HFileContext fileContext) {
    this.encodingAlgo = encoding;
    this.fileContext = fileContext;
    Compression.Algorithm compressionAlgorithm =
        fileContext.getCompression() == null ? NONE : fileContext.getCompression();
    if (compressionAlgorithm != NONE) {
      compressor = compressionAlgorithm.getCompressor();
      compressedByteStream = new ByteArrayOutputStream();
      try {
        compressionStream =
            compressionAlgorithm.createPlainCompressionStream(compressedByteStream, compressor);
      } catch (IOException e) {
        throw new RuntimeException(
            "Could not create compression stream for algorithm " + compressionAlgorithm, e);
      }
    }

    Encryption.Context cryptoContext = fileContext.getEncryptionContext();
    if (cryptoContext != Encryption.Context.NONE) {
      cryptoByteStream = new ByteArrayOutputStream();
      iv = new byte[cryptoContext.getCipher().getIvLength()];
      new SecureRandom().nextBytes(iv);
    }

    dummyHeader =
        Preconditions.checkNotNull(
            headerBytes,
            "Please pass HConstants.HFILEBLOCK_DUMMY_HEADER instead of null for param headerBytes");
  }
예제 #2
0
  @BeforeClass
  public static void setUp() throws Exception {
    Configuration conf = TEST_UTIL.getConfiguration();
    // Disable block cache in this test.
    conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
    conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
    conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
    conf.setInt("hfile.format.version", 3);

    fs = FileSystem.get(conf);

    cryptoContext = Encryption.newContext(conf);
    String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
    Cipher aes = Encryption.getCipher(conf, algorithm);
    assertNotNull(aes);
    cryptoContext.setCipher(aes);
    byte[] key = new byte[aes.getKeyLength()];
    RNG.nextBytes(key);
    cryptoContext.setKey(key);
  }
예제 #3
0
  @Test(timeout = 20000)
  public void testHFileEncryptionMetadata() throws Exception {
    Configuration conf = TEST_UTIL.getConfiguration();
    CacheConfig cacheConf = new CacheConfig(conf);
    HFileContext fileContext =
        new HFileContextBuilder().withEncryptionContext(cryptoContext).build();

    // write a simple encrypted hfile
    Path path = new Path(TEST_UTIL.getDataTestDir(), "cryptometa.hfile");
    FSDataOutputStream out = fs.create(path);
    HFile.Writer writer =
        HFile.getWriterFactory(conf, cacheConf)
            .withOutputStream(out)
            .withFileContext(fileContext)
            .create();
    try {
      KeyValue kv = new KeyValue("foo".getBytes(), "f1".getBytes(), null, "value".getBytes());
      writer.append(kv);
    } finally {
      writer.close();
      out.close();
    }

    // read it back in and validate correct crypto metadata
    HFile.Reader reader = HFile.createReader(fs, path, cacheConf, conf);
    try {
      reader.loadFileInfo();
      FixedFileTrailer trailer = reader.getTrailer();
      assertNotNull(trailer.getEncryptionKey());
      Encryption.Context readerContext = reader.getFileContext().getEncryptionContext();
      assertEquals(readerContext.getCipher().getName(), cryptoContext.getCipher().getName());
      assertTrue(Bytes.equals(readerContext.getKeyBytes(), cryptoContext.getKeyBytes()));
    } finally {
      reader.close();
    }
  }
  /**
   * @param uncompressedBytesWithHeader
   * @param blockType
   * @param headerBytes
   * @throws IOException
   */
  protected void compressAfterEncoding(
      byte[] uncompressedBytesWithHeader, BlockType blockType, byte[] headerBytes)
      throws IOException {
    this.uncompressedBytesWithHeader = uncompressedBytesWithHeader;

    Encryption.Context cryptoContext = fileContext.getEncryptionContext();
    if (cryptoContext != Encryption.Context.NONE) {

      // Encrypted block format:
      // +--------------------------+
      // | byte iv length           |
      // +--------------------------+
      // | iv data ...              |
      // +--------------------------+
      // | encrypted block data ... |
      // +--------------------------+

      cryptoByteStream.reset();
      // Write the block header (plaintext)
      cryptoByteStream.write(headerBytes);

      InputStream in;
      int plaintextLength;
      // Run any compression before encryption
      if (fileContext.getCompression() != Compression.Algorithm.NONE) {
        compressedByteStream.reset();
        compressionStream.resetState();
        compressionStream.write(
            uncompressedBytesWithHeader,
            headerBytes.length,
            uncompressedBytesWithHeader.length - headerBytes.length);
        compressionStream.flush();
        compressionStream.finish();
        byte[] plaintext = compressedByteStream.toByteArray();
        plaintextLength = plaintext.length;
        in = new ByteArrayInputStream(plaintext);
      } else {
        plaintextLength = uncompressedBytesWithHeader.length - headerBytes.length;
        in =
            new ByteArrayInputStream(
                uncompressedBytesWithHeader, headerBytes.length, plaintextLength);
      }

      if (plaintextLength > 0) {

        // Set up the cipher
        Cipher cipher = cryptoContext.getCipher();
        Encryptor encryptor = cipher.getEncryptor();
        encryptor.setKey(cryptoContext.getKey());

        // Set up the IV
        int ivLength = iv.length;
        Preconditions.checkState(ivLength <= Byte.MAX_VALUE, "IV length out of range");
        cryptoByteStream.write(ivLength);
        if (ivLength > 0) {
          encryptor.setIv(iv);
          cryptoByteStream.write(iv);
        }

        // Encrypt the data
        Encryption.encrypt(cryptoByteStream, in, encryptor);

        onDiskBytesWithHeader = cryptoByteStream.toByteArray();

        // Increment the IV given the final block size
        Encryption.incrementIv(iv, 1 + (onDiskBytesWithHeader.length / encryptor.getBlockSize()));

      } else {

        cryptoByteStream.write(0);
        onDiskBytesWithHeader = cryptoByteStream.toByteArray();
      }

    } else {

      if (this.fileContext.getCompression() != NONE) {
        compressedByteStream.reset();
        compressedByteStream.write(headerBytes);
        compressionStream.resetState();
        compressionStream.write(
            uncompressedBytesWithHeader,
            headerBytes.length,
            uncompressedBytesWithHeader.length - headerBytes.length);
        compressionStream.flush();
        compressionStream.finish();
        onDiskBytesWithHeader = compressedByteStream.toByteArray();
      } else {
        onDiskBytesWithHeader = uncompressedBytesWithHeader;
      }
    }

    this.blockType = blockType;
  }