@Override
public void setup(Context context) {
Configuration conf = context.getConfiguration();
crypto = new Rand();
crypto.init(conf.get("key"));
pattern = Pattern.compile(conf.get(PATTERN));
}
/**
* Flushes the snapshot of the memstore. Flushes the mob data to the mob files, and flushes the
* name of these mob files to HBase.
*
* @param snapshot The snapshot of the memstore.
* @throws IOException
*/
private void internalFlushCache(final MemStoreSnapshot snapshot) throws IOException {
if (snapshot.getCellsCount() == 0) {
return;
}
// generate the files into a temp directory.
String tempPathString = context.getConfiguration().get(SweepJob.WORKING_FILES_DIR_KEY);
StoreFile.Writer mobFileWriter =
MobUtils.createWriter(
conf,
fs,
hcd,
partitionId.getDate(),
new Path(tempPathString),
snapshot.getCellsCount(),
hcd.getCompactionCompression(),
partitionId.getStartKey(),
cacheConfig,
cryptoContext);
String relativePath = mobFileWriter.getPath().getName();
LOG.info("Create files under a temp directory " + mobFileWriter.getPath().toString());
byte[] referenceValue = Bytes.toBytes(relativePath);
KeyValueScanner scanner = snapshot.getScanner();
Cell cell = null;
while (null != (cell = scanner.next())) {
mobFileWriter.append(cell);
}
scanner.close();
// Write out the log sequence number that corresponds to this output
// hfile. The hfile is current up to and including logCacheFlushId.
mobFileWriter.appendMetadata(Long.MAX_VALUE, false, snapshot.getCellsCount());
mobFileWriter.close();
MobUtils.commitFile(conf, fs, mobFileWriter.getPath(), mobFamilyDir, cacheConfig);
context.getCounter(SweepCounter.FILE_AFTER_MERGE_OR_CLEAN).increment(1);
// write reference/fileName back to the store files of HBase.
scanner = snapshot.getScanner();
scanner.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
cell = null;
Tag tableNameTag =
new ArrayBackedTag(
TagType.MOB_TABLE_NAME_TAG_TYPE, Bytes.toBytes(this.table.getName().toString()));
long updatedCount = 0;
while (null != (cell = scanner.next())) {
KeyValue reference = MobUtils.createMobRefKeyValue(cell, referenceValue, tableNameTag);
Put put =
new Put(reference.getRowArray(), reference.getRowOffset(), reference.getRowLength());
put.add(reference);
table.mutate(put);
updatedCount++;
}
table.flush();
context.getCounter(SweepCounter.RECORDS_UPDATED).increment(updatedCount);
scanner.close();
}
protected void setup(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
fieldDelim = conf.get("field.delim", "[]");
fieldDelimRegex = conf.get("field.delim.regex", "\\[\\]");
String filePath = conf.get("histogram.schema.file.path");
FileSystem dfs = FileSystem.get(conf);
Path src = new Path(filePath);
FSDataInputStream fs = dfs.open(src);
ObjectMapper mapper = new ObjectMapper();
schema = mapper.readValue(fs, RichAttributeSchema.class);
}
public MemStoreWrapper(
Context context,
FileSystem fs,
BufferedMutator table,
HColumnDescriptor hcd,
MemStore memstore,
CacheConfig cacheConfig)
throws IOException {
this.memstore = memstore;
this.context = context;
this.fs = fs;
this.table = table;
this.hcd = hcd;
this.conf = context.getConfiguration();
this.cacheConfig = cacheConfig;
flushSize =
this.conf.getLong(
MobConstants.MOB_SWEEP_TOOL_COMPACTION_MEMSTORE_FLUSH_SIZE,
MobConstants.DEFAULT_MOB_SWEEP_TOOL_COMPACTION_MEMSTORE_FLUSH_SIZE);
mobFamilyDir = MobUtils.getMobFamilyPath(conf, table.getName(), hcd.getNameAsString());
cryptoContext = EncryptionUtil.createEncryptionContext(conf, hcd);
}
@Override
public void setup(Context context) {
Configuration conf = context.getConfiguration();
crypto = new Rand();
crypto.init(conf.get("key"));
}
@Override
protected void setup(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
loc_text_index = Integer.parseInt(conf.get("loc_text_index"));
loc_field_num = Integer.parseInt(conf.get("loc_field_num"));
}
protected void setup(Context context) throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
fieldDelim = conf.get("field.delim", "[]");
}