Beispiel #1
0
  @Override
  public void setup(Context context) {
    Configuration conf = context.getConfiguration();
    crypto = new Rand();
    crypto.init(conf.get("key"));

    pattern = Pattern.compile(conf.get(PATTERN));
  }
Beispiel #2
0
  /**
   * Flushes the snapshot of the memstore. Flushes the mob data to the mob files, and flushes the
   * name of these mob files to HBase.
   *
   * @param snapshot The snapshot of the memstore.
   * @throws IOException
   */
  private void internalFlushCache(final MemStoreSnapshot snapshot) throws IOException {
    if (snapshot.getCellsCount() == 0) {
      return;
    }
    // generate the files into a temp directory.
    String tempPathString = context.getConfiguration().get(SweepJob.WORKING_FILES_DIR_KEY);
    StoreFile.Writer mobFileWriter =
        MobUtils.createWriter(
            conf,
            fs,
            hcd,
            partitionId.getDate(),
            new Path(tempPathString),
            snapshot.getCellsCount(),
            hcd.getCompactionCompression(),
            partitionId.getStartKey(),
            cacheConfig,
            cryptoContext);

    String relativePath = mobFileWriter.getPath().getName();
    LOG.info("Create files under a temp directory " + mobFileWriter.getPath().toString());

    byte[] referenceValue = Bytes.toBytes(relativePath);
    KeyValueScanner scanner = snapshot.getScanner();
    Cell cell = null;
    while (null != (cell = scanner.next())) {
      mobFileWriter.append(cell);
    }
    scanner.close();
    // Write out the log sequence number that corresponds to this output
    // hfile. The hfile is current up to and including logCacheFlushId.
    mobFileWriter.appendMetadata(Long.MAX_VALUE, false, snapshot.getCellsCount());
    mobFileWriter.close();

    MobUtils.commitFile(conf, fs, mobFileWriter.getPath(), mobFamilyDir, cacheConfig);
    context.getCounter(SweepCounter.FILE_AFTER_MERGE_OR_CLEAN).increment(1);
    // write reference/fileName back to the store files of HBase.
    scanner = snapshot.getScanner();
    scanner.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
    cell = null;
    Tag tableNameTag =
        new ArrayBackedTag(
            TagType.MOB_TABLE_NAME_TAG_TYPE, Bytes.toBytes(this.table.getName().toString()));
    long updatedCount = 0;
    while (null != (cell = scanner.next())) {
      KeyValue reference = MobUtils.createMobRefKeyValue(cell, referenceValue, tableNameTag);
      Put put =
          new Put(reference.getRowArray(), reference.getRowOffset(), reference.getRowLength());
      put.add(reference);
      table.mutate(put);
      updatedCount++;
    }
    table.flush();
    context.getCounter(SweepCounter.RECORDS_UPDATED).increment(updatedCount);
    scanner.close();
  }
Beispiel #3
0
    protected void setup(Context context) throws IOException, InterruptedException {
      Configuration conf = context.getConfiguration();
      fieldDelim = conf.get("field.delim", "[]");
      fieldDelimRegex = conf.get("field.delim.regex", "\\[\\]");

      String filePath = conf.get("histogram.schema.file.path");
      FileSystem dfs = FileSystem.get(conf);
      Path src = new Path(filePath);
      FSDataInputStream fs = dfs.open(src);
      ObjectMapper mapper = new ObjectMapper();
      schema = mapper.readValue(fs, RichAttributeSchema.class);
    }
Beispiel #4
0
 public MemStoreWrapper(
     Context context,
     FileSystem fs,
     BufferedMutator table,
     HColumnDescriptor hcd,
     MemStore memstore,
     CacheConfig cacheConfig)
     throws IOException {
   this.memstore = memstore;
   this.context = context;
   this.fs = fs;
   this.table = table;
   this.hcd = hcd;
   this.conf = context.getConfiguration();
   this.cacheConfig = cacheConfig;
   flushSize =
       this.conf.getLong(
           MobConstants.MOB_SWEEP_TOOL_COMPACTION_MEMSTORE_FLUSH_SIZE,
           MobConstants.DEFAULT_MOB_SWEEP_TOOL_COMPACTION_MEMSTORE_FLUSH_SIZE);
   mobFamilyDir = MobUtils.getMobFamilyPath(conf, table.getName(), hcd.getNameAsString());
   cryptoContext = EncryptionUtil.createEncryptionContext(conf, hcd);
 }
Beispiel #5
0
 @Override
 public void setup(Context context) {
   Configuration conf = context.getConfiguration();
   crypto = new Rand();
   crypto.init(conf.get("key"));
 }
 @Override
 protected void setup(Context context) throws IOException, InterruptedException {
   Configuration conf = context.getConfiguration();
   loc_text_index = Integer.parseInt(conf.get("loc_text_index"));
   loc_field_num = Integer.parseInt(conf.get("loc_field_num"));
 }
Beispiel #7
0
 protected void setup(Context context) throws IOException, InterruptedException {
   Configuration conf = context.getConfiguration();
   fieldDelim = conf.get("field.delim", "[]");
 }