/** {@inheritDoc} */
      @Override
      public void write(HFileKeyValue entry, NullWritable unused) throws IOException {

        final KeyValue kv = entry.getKeyValue();
        kv.updateLatestStamp(mLatestTimestampBytes);

        final long recordLength = kv.getLength();
        if (mCurrentHFileSize + recordLength >= mMaxFileSizeBytes) {
          // We can't fit this record in the current HFile without exceeding the max file size.

          if (Arrays.equals(mCurrentRow, kv.getRow())) {
            // But we're still adding data for a single row, so we can't close this HFile yet.
            LOG.debug("Reached max HFile size, but waiting to finish this row before closing.");
          } else {
            // Close it and open a new one.
            closeWriter(mWriter);
            mWriter = openNewWriter();
          }
        }

        mWriter.append(kv);
        mTimeRangeTracker.includeTimestamp(kv);
        mCurrentHFileSize += recordLength;

        // Remember the row so we know when we are transitioning.
        mCurrentRow = kv.getRow();
      }
 private static void createHFile(
     Configuration conf, FileSystem fs, Path path, byte[] family, byte[] qualifier)
     throws IOException {
   HFileContext context = new HFileContextBuilder().build();
   HFile.Writer writer =
       HFile.getWriterFactory(conf, new CacheConfig(conf))
           .withPath(fs, path)
           .withFileContext(context)
           .create();
   long now = System.currentTimeMillis();
   try {
     for (int i = 1; i <= 9; i++) {
       KeyValue kv =
           new KeyValue(Bytes.toBytes(i + ""), family, qualifier, now, Bytes.toBytes(i + ""));
       writer.append(kv);
     }
   } finally {
     writer.close();
   }
 }
 /** Create an HFile with the given number of rows with a specified value. */
 public static void createHFile(
     FileSystem fs, Path path, byte[] family, byte[] qualifier, byte[] value, int numRows)
     throws IOException {
   HFileContext context =
       new HFileContextBuilder().withBlockSize(BLOCKSIZE).withCompression(COMPRESSION).build();
   HFile.Writer writer =
       HFile.getWriterFactory(conf, new CacheConfig(conf))
           .withPath(fs, path)
           .withFileContext(context)
           .create();
   long now = System.currentTimeMillis();
   try {
     // subtract 2 since iterateOnSplits doesn't include boundary keys
     for (int i = 0; i < numRows; i++) {
       KeyValue kv = new KeyValue(rowkey(i), family, qualifier, now, value);
       writer.append(kv);
     }
     writer.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(now));
   } finally {
     writer.close();
   }
 }
Beispiel #4
0
  @Test(timeout = 20000)
  public void testHFileEncryptionMetadata() throws Exception {
    Configuration conf = TEST_UTIL.getConfiguration();
    CacheConfig cacheConf = new CacheConfig(conf);
    HFileContext fileContext =
        new HFileContextBuilder().withEncryptionContext(cryptoContext).build();

    // write a simple encrypted hfile
    Path path = new Path(TEST_UTIL.getDataTestDir(), "cryptometa.hfile");
    FSDataOutputStream out = fs.create(path);
    HFile.Writer writer =
        HFile.getWriterFactory(conf, cacheConf)
            .withOutputStream(out)
            .withFileContext(fileContext)
            .create();
    try {
      KeyValue kv = new KeyValue("foo".getBytes(), "f1".getBytes(), null, "value".getBytes());
      writer.append(kv);
    } finally {
      writer.close();
      out.close();
    }

    // read it back in and validate correct crypto metadata
    HFile.Reader reader = HFile.createReader(fs, path, cacheConf, conf);
    try {
      reader.loadFileInfo();
      FixedFileTrailer trailer = reader.getTrailer();
      assertNotNull(trailer.getEncryptionKey());
      Encryption.Context readerContext = reader.getFileContext().getEncryptionContext();
      assertEquals(readerContext.getCipher().getName(), cryptoContext.getCipher().getName());
      assertTrue(Bytes.equals(readerContext.getKeyBytes(), cryptoContext.getKeyBytes()));
    } finally {
      reader.close();
    }
  }
Beispiel #5
0
  @Test(timeout = 6000000)
  public void testHFileEncryption() throws Exception {
    // Create 1000 random test KVs
    RedundantKVGenerator generator = new RedundantKVGenerator();
    List<KeyValue> testKvs = generator.generateTestKeyValues(1000);

    // Iterate through data block encoding and compression combinations
    Configuration conf = TEST_UTIL.getConfiguration();
    CacheConfig cacheConf = new CacheConfig(conf);
    for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
      for (Compression.Algorithm compression : TestHFileBlock.COMPRESSION_ALGORITHMS) {
        HFileContext fileContext =
            new HFileContextBuilder()
                .withBlockSize(4096) // small blocks
                .withEncryptionContext(cryptoContext)
                .withCompression(compression)
                .withDataBlockEncoding(encoding)
                .build();
        // write a new test HFile
        LOG.info("Writing with " + fileContext);
        Path path = new Path(TEST_UTIL.getDataTestDir(), UUID.randomUUID().toString() + ".hfile");
        FSDataOutputStream out = fs.create(path);
        HFile.Writer writer =
            HFile.getWriterFactory(conf, cacheConf)
                .withOutputStream(out)
                .withFileContext(fileContext)
                .create();
        try {
          for (KeyValue kv : testKvs) {
            writer.append(kv);
          }
        } finally {
          writer.close();
          out.close();
        }

        // read it back in
        LOG.info("Reading with " + fileContext);
        int i = 0;
        HFileScanner scanner = null;
        HFile.Reader reader = HFile.createReader(fs, path, cacheConf, conf);
        try {
          reader.loadFileInfo();
          FixedFileTrailer trailer = reader.getTrailer();
          assertNotNull(trailer.getEncryptionKey());
          scanner = reader.getScanner(false, false);
          assertTrue("Initial seekTo failed", scanner.seekTo());
          do {
            Cell kv = scanner.getCell();
            assertTrue(
                "Read back an unexpected or invalid KV",
                testKvs.contains(KeyValueUtil.ensureKeyValue(kv)));
            i++;
          } while (scanner.next());
        } finally {
          reader.close();
          scanner.close();
        }

        assertEquals("Did not read back as many KVs as written", i, testKvs.size());

        // Test random seeks with pread
        LOG.info("Random seeking with " + fileContext);
        reader = HFile.createReader(fs, path, cacheConf, conf);
        try {
          scanner = reader.getScanner(false, true);
          assertTrue("Initial seekTo failed", scanner.seekTo());
          for (i = 0; i < 100; i++) {
            KeyValue kv = testKvs.get(RNG.nextInt(testKvs.size()));
            assertEquals("Unable to find KV as expected: " + kv, scanner.seekTo(kv), 0);
          }
        } finally {
          scanner.close();
          reader.close();
        }
      }
    }
  }