/** Apply column family options such as Bloom filters, compression, and data block encoding. */
protected void applyColumnFamilyOptions(byte[] tableName, byte[][] columnFamilies)
throws IOException {
HBaseAdmin admin = new HBaseAdmin(conf);
HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
LOG.info("Disabling table " + Bytes.toString(tableName));
admin.disableTable(tableName);
for (byte[] cf : columnFamilies) {
HColumnDescriptor columnDesc = tableDesc.getFamily(cf);
boolean isNewCf = columnDesc == null;
if (isNewCf) {
columnDesc = new HColumnDescriptor(cf);
}
if (bloomType != null) {
columnDesc.setBloomFilterType(bloomType);
}
if (compressAlgo != null) {
columnDesc.setCompressionType(compressAlgo);
}
if (dataBlockEncodingAlgo != null) {
columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo);
columnDesc.setEncodeOnDisk(!encodeInCacheOnly);
}
if (inMemoryCF) {
columnDesc.setInMemory(inMemoryCF);
}
if (isNewCf) {
admin.addColumn(tableName, columnDesc);
} else {
admin.modifyColumn(tableName, columnDesc);
}
}
LOG.info("Enabling table " + Bytes.toString(tableName));
admin.enableTable(tableName);
}
public void runMergeWorkload() throws IOException {
long maxKeyCount = prepareForMerge();
List<StoreFileScanner> scanners =
StoreFileScanner.getScannersForStoreFiles(inputStoreFiles, false, false);
HColumnDescriptor columnDescriptor =
new HColumnDescriptor(HFileReadWriteTest.class.getSimpleName());
columnDescriptor.setBlocksize(blockSize);
columnDescriptor.setBloomFilterType(bloomType);
columnDescriptor.setCompressionType(compression);
columnDescriptor.setDataBlockEncoding(dataBlockEncoding);
HRegionInfo regionInfo = new HRegionInfo();
HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
HRegion region = new HRegion(outputDir, null, fs, conf, regionInfo, htd, null);
HStore store = new HStore(outputDir, region, columnDescriptor, fs, conf);
StoreFile.Writer writer =
new StoreFile.WriterBuilder(conf, new CacheConfig(conf), fs, blockSize)
.withOutputDir(outputDir)
.withCompression(compression)
.withDataBlockEncoder(dataBlockEncoder)
.withBloomType(bloomType)
.withMaxKeyCount(maxKeyCount)
.withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE)
.withBytesPerChecksum(HFile.DEFAULT_BYTES_PER_CHECKSUM)
.build();
StatisticsPrinter statsPrinter = new StatisticsPrinter();
statsPrinter.startThread();
try {
performMerge(scanners, store, writer);
writer.close();
} finally {
statsPrinter.requestStop();
}
Path resultPath = writer.getPath();
resultPath = tryUsingSimpleOutputPath(resultPath);
long fileSize = fs.getFileStatus(resultPath).getLen();
LOG.info("Created " + resultPath + ", size " + fileSize);
System.out.println();
System.out.println("HFile information for " + resultPath);
System.out.println();
HFilePrettyPrinter hfpp = new HFilePrettyPrinter();
hfpp.run(new String[] {"-m", "-f", resultPath.toString()});
}
private void runTest(
String testName,
TableName tableName,
BloomType bloomType,
boolean preCreateTable,
byte[][] tableSplitKeys,
byte[][][] hfileRanges)
throws Exception {
HTableDescriptor htd = new HTableDescriptor(tableName);
HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY);
familyDesc.setBloomFilterType(bloomType);
htd.addFamily(familyDesc);
runTest(testName, htd, bloomType, preCreateTable, tableSplitKeys, hfileRanges);
}
/**
* Create a Mob table.
*
* @param util
* @param tableName
* @param families
* @return An HTable instance for the created table.
* @throws IOException
*/
public static Table createMobTable(
final HBaseTestingUtility util, final TableName tableName, final byte[]... families)
throws IOException {
HTableDescriptor htd = new HTableDescriptor(tableName);
for (byte[] family : families) {
HColumnDescriptor hcd = new HColumnDescriptor(family);
// Disable blooms (they are on by default as of 0.95) but we disable them
// here because
// tests have hard coded counts of what to expect in block cache, etc.,
// and blooms being
// on is interfering.
hcd.setBloomFilterType(BloomType.NONE);
hcd.setMobEnabled(true);
hcd.setMobThreshold(0L);
htd.addFamily(hcd);
}
util.getHBaseAdmin().createTable(htd);
// HBaseAdmin only waits for regions to appear in hbase:meta we should wait
// until they are assigned
util.waitUntilAllRegionsAssigned(htd.getTableName());
return ConnectionFactory.createConnection(util.getConfiguration()).getTable(htd.getTableName());
}
/**
* "name":"", "compression":"", "versions":"", "min_versions":"", "blocksize":"", "in_memory":"",
* "blockcache":"", "bloomfilter":"", "replication_scope":""
*
* @param hcd
* @param family
* @return
*/
private HColumnDescriptor getColumnDescriptor(JSONObject family) {
HColumnDescriptor hcd = null;
if (family == null) return hcd;
try {
if (family.has(XHBaseConstant.TABLE_DESC_FNAME)) {
hcd = new HColumnDescriptor((String) family.get(XHBaseConstant.TABLE_DESC_FNAME));
} else {
return hcd;
}
// set compression
if (family.has(XHBaseConstant.TABLE_DESC_COMPRESSION)) {
String compression = (String) family.get(XHBaseConstant.TABLE_DESC_COMPRESSION);
if (!compression.isEmpty()) {
if (compression.equalsIgnoreCase("gz")) {
hcd.setCompressionType(Compression.Algorithm.GZ);
} else if (compression.equalsIgnoreCase("lzo")) {
hcd.setCompressionType(Compression.Algorithm.LZO);
}
}
}
// set versions
if (family.has(XHBaseConstant.TABLE_DESC_VERSIONS)) {
String versions = (String) family.get(XHBaseConstant.TABLE_DESC_VERSIONS);
if (!versions.isEmpty()) {
hcd.setMaxVersions(Integer.valueOf(versions));
}
}
// set min versions
if (family.has(XHBaseConstant.TABLE_DESC_MINVERSIONS)) {
String min_versions = (String) family.get(XHBaseConstant.TABLE_DESC_MINVERSIONS);
if (!min_versions.isEmpty()) {
hcd.setMinVersions(Integer.valueOf(min_versions));
}
}
// set block size
if (family.has(XHBaseConstant.TABLE_DESC_BLOCKSIZE)) {
String block_size = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKSIZE);
if (!block_size.isEmpty()) {
hcd.setBlocksize(Integer.valueOf(block_size));
}
}
// set in memory
if (family.has(XHBaseConstant.TABLE_DESC_INMEMORY)) {
String in_memory = (String) family.get(XHBaseConstant.TABLE_DESC_INMEMORY);
if (!in_memory.isEmpty()) {
hcd.setInMemory(in_memory.equalsIgnoreCase("true") ? true : false);
}
}
// set block cache
if (family.has(XHBaseConstant.TABLE_DESC_BLOCKCACHE)) {
String block_cache = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKCACHE);
if (!block_cache.isEmpty()) {
hcd.setBlockCacheEnabled(block_cache.equalsIgnoreCase("true") ? true : false);
}
}
// set bloom filter
if (family.has(XHBaseConstant.TABLE_DESC_BLOOMFILTER)) {
String bloom_filter = (String) family.get(XHBaseConstant.TABLE_DESC_BLOOMFILTER);
if (!bloom_filter.isEmpty()) {
if (bloom_filter.equalsIgnoreCase("none")) {
hcd.setBloomFilterType(BloomType.NONE);
} else if (bloom_filter.equalsIgnoreCase("row")) {
hcd.setBloomFilterType(BloomType.ROW);
} else if (bloom_filter.equalsIgnoreCase("ROWCOL"))
hcd.setBloomFilterType(BloomType.ROWCOL); // TODO what is it?
}
}
// set replication scope
if (family.has(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE)) {
String replica_scope = (String) family.get(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE);
if (!replica_scope.isEmpty()) {
hcd.setScope(Integer.valueOf(replica_scope));
}
}
} catch (Exception e) {
e.printStackTrace();
}
return hcd;
}
