public void runMergeWorkload() throws IOException {
long maxKeyCount = prepareForMerge();
List<StoreFileScanner> scanners =
StoreFileScanner.getScannersForStoreFiles(inputStoreFiles, false, false);
HColumnDescriptor columnDescriptor =
new HColumnDescriptor(HFileReadWriteTest.class.getSimpleName());
columnDescriptor.setBlocksize(blockSize);
columnDescriptor.setBloomFilterType(bloomType);
columnDescriptor.setCompressionType(compression);
columnDescriptor.setDataBlockEncoding(dataBlockEncoding);
HRegionInfo regionInfo = new HRegionInfo();
HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
HRegion region = new HRegion(outputDir, null, fs, conf, regionInfo, htd, null);
HStore store = new HStore(outputDir, region, columnDescriptor, fs, conf);
StoreFile.Writer writer =
new StoreFile.WriterBuilder(conf, new CacheConfig(conf), fs, blockSize)
.withOutputDir(outputDir)
.withCompression(compression)
.withDataBlockEncoder(dataBlockEncoder)
.withBloomType(bloomType)
.withMaxKeyCount(maxKeyCount)
.withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE)
.withBytesPerChecksum(HFile.DEFAULT_BYTES_PER_CHECKSUM)
.build();
StatisticsPrinter statsPrinter = new StatisticsPrinter();
statsPrinter.startThread();
try {
performMerge(scanners, store, writer);
writer.close();
} finally {
statsPrinter.requestStop();
}
Path resultPath = writer.getPath();
resultPath = tryUsingSimpleOutputPath(resultPath);
long fileSize = fs.getFileStatus(resultPath).getLen();
LOG.info("Created " + resultPath + ", size " + fileSize);
System.out.println();
System.out.println("HFile information for " + resultPath);
System.out.println();
HFilePrettyPrinter hfpp = new HFilePrettyPrinter();
hfpp.run(new String[] {"-m", "-f", resultPath.toString()});
}
/* (non-Javadoc)
* @see com.mozilla.bagheera.hazelcast.persistence.MapStoreBase#init(com.hazelcast.core.HazelcastInstance, java.util.Properties, java.lang.String)
*/
@Override
public void init(HazelcastInstance hazelcastInstance, Properties properties, String mapName) {
super.init(hazelcastInstance, properties, mapName);
Configuration conf = HBaseConfiguration.create();
for (String name : properties.stringPropertyNames()) {
if (name.startsWith("hbase.")
|| name.startsWith("hadoop.")
|| name.startsWith("zookeeper.")) {
conf.set(name, properties.getProperty(name));
}
}
prefixDate =
Boolean.parseBoolean(properties.getProperty("hazelcast.hbase.key.prefix.date", "false"));
int hbasePoolSize = Integer.parseInt(properties.getProperty("hazelcast.hbase.pool.size", "10"));
tableName = Bytes.toBytes(properties.getProperty("hazelcast.hbase.table", mapName));
family = Bytes.toBytes(properties.getProperty("hazelcast.hbase.column.family", "data"));
qualifier = Bytes.toBytes(properties.getProperty("hazelcast.hbase.column.qualifier", "json"));
pool = new HTablePool(conf, hbasePoolSize);
try {
HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
if (!hbaseAdmin.tableExists(tableName)) {
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor columnDesc = new HColumnDescriptor(family);
columnDesc.setCompressionType(Algorithm.LZO);
columnDesc.setBlockCacheEnabled(true);
columnDesc.setBlocksize(65536);
columnDesc.setInMemory(false);
columnDesc.setMaxVersions(1);
columnDesc.setTimeToLive(Integer.MAX_VALUE);
desc.addFamily(columnDesc);
hbaseAdmin.createTable(desc);
}
} catch (Exception e) {
throw new RuntimeException("Error creating table!", e);
}
// register with MapStoreRepository
MapStoreRepository.addMapStore(mapName, this);
}
/**
* "name":"", "compression":"", "versions":"", "min_versions":"", "blocksize":"", "in_memory":"",
* "blockcache":"", "bloomfilter":"", "replication_scope":""
*
* @param hcd
* @param family
* @return
*/
private HColumnDescriptor getColumnDescriptor(JSONObject family) {
HColumnDescriptor hcd = null;
if (family == null) return hcd;
try {
if (family.has(XHBaseConstant.TABLE_DESC_FNAME)) {
hcd = new HColumnDescriptor((String) family.get(XHBaseConstant.TABLE_DESC_FNAME));
} else {
return hcd;
}
// set compression
if (family.has(XHBaseConstant.TABLE_DESC_COMPRESSION)) {
String compression = (String) family.get(XHBaseConstant.TABLE_DESC_COMPRESSION);
if (!compression.isEmpty()) {
if (compression.equalsIgnoreCase("gz")) {
hcd.setCompressionType(Compression.Algorithm.GZ);
} else if (compression.equalsIgnoreCase("lzo")) {
hcd.setCompressionType(Compression.Algorithm.LZO);
}
}
}
// set versions
if (family.has(XHBaseConstant.TABLE_DESC_VERSIONS)) {
String versions = (String) family.get(XHBaseConstant.TABLE_DESC_VERSIONS);
if (!versions.isEmpty()) {
hcd.setMaxVersions(Integer.valueOf(versions));
}
}
// set min versions
if (family.has(XHBaseConstant.TABLE_DESC_MINVERSIONS)) {
String min_versions = (String) family.get(XHBaseConstant.TABLE_DESC_MINVERSIONS);
if (!min_versions.isEmpty()) {
hcd.setMinVersions(Integer.valueOf(min_versions));
}
}
// set block size
if (family.has(XHBaseConstant.TABLE_DESC_BLOCKSIZE)) {
String block_size = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKSIZE);
if (!block_size.isEmpty()) {
hcd.setBlocksize(Integer.valueOf(block_size));
}
}
// set in memory
if (family.has(XHBaseConstant.TABLE_DESC_INMEMORY)) {
String in_memory = (String) family.get(XHBaseConstant.TABLE_DESC_INMEMORY);
if (!in_memory.isEmpty()) {
hcd.setInMemory(in_memory.equalsIgnoreCase("true") ? true : false);
}
}
// set block cache
if (family.has(XHBaseConstant.TABLE_DESC_BLOCKCACHE)) {
String block_cache = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKCACHE);
if (!block_cache.isEmpty()) {
hcd.setBlockCacheEnabled(block_cache.equalsIgnoreCase("true") ? true : false);
}
}
// set bloom filter
if (family.has(XHBaseConstant.TABLE_DESC_BLOOMFILTER)) {
String bloom_filter = (String) family.get(XHBaseConstant.TABLE_DESC_BLOOMFILTER);
if (!bloom_filter.isEmpty()) {
if (bloom_filter.equalsIgnoreCase("none")) {
hcd.setBloomFilterType(BloomType.NONE);
} else if (bloom_filter.equalsIgnoreCase("row")) {
hcd.setBloomFilterType(BloomType.ROW);
} else if (bloom_filter.equalsIgnoreCase("ROWCOL"))
hcd.setBloomFilterType(BloomType.ROWCOL); // TODO what is it?
}
}
// set replication scope
if (family.has(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE)) {
String replica_scope = (String) family.get(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE);
if (!replica_scope.isEmpty()) {
hcd.setScope(Integer.valueOf(replica_scope));
}
}
} catch (Exception e) {
e.printStackTrace();
}
return hcd;
}
