예제 #1
0
  /**
   * Flushes the snapshot of the memstore. Flushes the mob data to the mob files, and flushes the
   * name of these mob files to HBase.
   *
   * @param snapshot The snapshot of the memstore.
   * @throws IOException
   */
  private void internalFlushCache(final MemStoreSnapshot snapshot) throws IOException {
    if (snapshot.getCellsCount() == 0) {
      return;
    }
    // generate the files into a temp directory.
    String tempPathString = context.getConfiguration().get(SweepJob.WORKING_FILES_DIR_KEY);
    StoreFile.Writer mobFileWriter =
        MobUtils.createWriter(
            conf,
            fs,
            hcd,
            partitionId.getDate(),
            new Path(tempPathString),
            snapshot.getCellsCount(),
            hcd.getCompactionCompression(),
            partitionId.getStartKey(),
            cacheConfig,
            cryptoContext);

    String relativePath = mobFileWriter.getPath().getName();
    LOG.info("Create files under a temp directory " + mobFileWriter.getPath().toString());

    byte[] referenceValue = Bytes.toBytes(relativePath);
    KeyValueScanner scanner = snapshot.getScanner();
    Cell cell = null;
    while (null != (cell = scanner.next())) {
      mobFileWriter.append(cell);
    }
    scanner.close();
    // Write out the log sequence number that corresponds to this output
    // hfile. The hfile is current up to and including logCacheFlushId.
    mobFileWriter.appendMetadata(Long.MAX_VALUE, false, snapshot.getCellsCount());
    mobFileWriter.close();

    MobUtils.commitFile(conf, fs, mobFileWriter.getPath(), mobFamilyDir, cacheConfig);
    context.getCounter(SweepCounter.FILE_AFTER_MERGE_OR_CLEAN).increment(1);
    // write reference/fileName back to the store files of HBase.
    scanner = snapshot.getScanner();
    scanner.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
    cell = null;
    Tag tableNameTag =
        new ArrayBackedTag(
            TagType.MOB_TABLE_NAME_TAG_TYPE, Bytes.toBytes(this.table.getName().toString()));
    long updatedCount = 0;
    while (null != (cell = scanner.next())) {
      KeyValue reference = MobUtils.createMobRefKeyValue(cell, referenceValue, tableNameTag);
      Put put =
          new Put(reference.getRowArray(), reference.getRowOffset(), reference.getRowLength());
      put.add(reference);
      table.mutate(put);
      updatedCount++;
    }
    table.flush();
    context.getCounter(SweepCounter.RECORDS_UPDATED).increment(updatedCount);
    scanner.close();
  }