/** Apply column family options such as Bloom filters, compression, and data block encoding. */
protected void applyColumnFamilyOptions(byte[] tableName, byte[][] columnFamilies)
throws IOException {
HBaseAdmin admin = new HBaseAdmin(conf);
HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
LOG.info("Disabling table " + Bytes.toString(tableName));
admin.disableTable(tableName);
for (byte[] cf : columnFamilies) {
HColumnDescriptor columnDesc = tableDesc.getFamily(cf);
boolean isNewCf = columnDesc == null;
if (isNewCf) {
columnDesc = new HColumnDescriptor(cf);
}
if (bloomType != null) {
columnDesc.setBloomFilterType(bloomType);
}
if (compressAlgo != null) {
columnDesc.setCompressionType(compressAlgo);
}
if (dataBlockEncodingAlgo != null) {
columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo);
columnDesc.setEncodeOnDisk(!encodeInCacheOnly);
}
if (inMemoryCF) {
columnDesc.setInMemory(inMemoryCF);
}
if (isNewCf) {
admin.addColumn(tableName, columnDesc);
} else {
admin.modifyColumn(tableName, columnDesc);
}
}
LOG.info("Enabling table " + Bytes.toString(tableName));
admin.enableTable(tableName);
}
public void runMergeWorkload() throws IOException {
long maxKeyCount = prepareForMerge();
List<StoreFileScanner> scanners =
StoreFileScanner.getScannersForStoreFiles(inputStoreFiles, false, false);
HColumnDescriptor columnDescriptor =
new HColumnDescriptor(HFileReadWriteTest.class.getSimpleName());
columnDescriptor.setBlocksize(blockSize);
columnDescriptor.setBloomFilterType(bloomType);
columnDescriptor.setCompressionType(compression);
columnDescriptor.setDataBlockEncoding(dataBlockEncoding);
HRegionInfo regionInfo = new HRegionInfo();
HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
HRegion region = new HRegion(outputDir, null, fs, conf, regionInfo, htd, null);
HStore store = new HStore(outputDir, region, columnDescriptor, fs, conf);
StoreFile.Writer writer =
new StoreFile.WriterBuilder(conf, new CacheConfig(conf), fs, blockSize)
.withOutputDir(outputDir)
.withCompression(compression)
.withDataBlockEncoder(dataBlockEncoder)
.withBloomType(bloomType)
.withMaxKeyCount(maxKeyCount)
.withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE)
.withBytesPerChecksum(HFile.DEFAULT_BYTES_PER_CHECKSUM)
.build();
StatisticsPrinter statsPrinter = new StatisticsPrinter();
statsPrinter.startThread();
try {
performMerge(scanners, store, writer);
writer.close();
} finally {
statsPrinter.requestStop();
}
Path resultPath = writer.getPath();
resultPath = tryUsingSimpleOutputPath(resultPath);
long fileSize = fs.getFileStatus(resultPath).getLen();
LOG.info("Created " + resultPath + ", size " + fileSize);
System.out.println();
System.out.println("HFile information for " + resultPath);
System.out.println();
HFilePrettyPrinter hfpp = new HFilePrettyPrinter();
hfpp.run(new String[] {"-m", "-f", resultPath.toString()});
}
private void ensureColumnFamilyExists(String tableName, String columnFamily)
throws StorageException {
HBaseAdmin adm = getAdminInterface();
HTableDescriptor desc = ensureTableExists(tableName);
Preconditions.checkNotNull(desc);
HColumnDescriptor cf = desc.getFamily(columnFamily.getBytes());
// Create our column family, if necessary
if (cf == null) {
try {
adm.disableTable(tableName);
desc.addFamily(
new HColumnDescriptor(columnFamily).setCompressionType(Compression.Algorithm.GZ));
adm.modifyTable(tableName.getBytes(), desc);
try {
logger.debug(
"Added HBase ColumnFamily {}, waiting for 1 sec. to propogate.", columnFamily);
Thread.sleep(1000L);
} catch (InterruptedException ie) {
throw new TemporaryStorageException(ie);
}
adm.enableTable(tableName);
} catch (TableNotFoundException ee) {
logger.error("TableNotFoundException", ee);
throw new PermanentStorageException(ee);
} catch (org.apache.hadoop.hbase.TableExistsException ee) {
logger.debug("Swallowing exception {}", ee);
} catch (IOException ee) {
throw new TemporaryStorageException(ee);
}
} else { // check if compression was enabled, if not - enable it
if (cf.getCompressionType() == null
|| cf.getCompressionType() == Compression.Algorithm.NONE) {
try {
adm.disableTable(tableName);
adm.modifyColumn(tableName, cf.setCompressionType(Compression.Algorithm.GZ));
adm.enableTable(tableName);
} catch (IOException e) {
throw new TemporaryStorageException(e);
}
}
}
}
public static void createTableInner(String tableName, int splitSize, String splitPrefix)
throws Exception {
Configuration conf = HBaseConfiguration.create();
HBaseAdmin hAdmin = new HBaseAdmin(conf);
HColumnDescriptor columnDescriptor = new HColumnDescriptor(Bytes.toBytes("f"));
columnDescriptor.setCompressionType(Algorithm.SNAPPY);
columnDescriptor.setMaxVersions(3);
HTableDescriptor descriptor = new HTableDescriptor(TableName.valueOf(tableName));
descriptor.addFamily(columnDescriptor);
hAdmin.createTable(descriptor, Util.genSplitKeysAlphaDig(splitPrefix, splitSize));
hAdmin.close();
}
/* (non-Javadoc)
* @see com.mozilla.bagheera.hazelcast.persistence.MapStoreBase#init(com.hazelcast.core.HazelcastInstance, java.util.Properties, java.lang.String)
*/
@Override
public void init(HazelcastInstance hazelcastInstance, Properties properties, String mapName) {
super.init(hazelcastInstance, properties, mapName);
Configuration conf = HBaseConfiguration.create();
for (String name : properties.stringPropertyNames()) {
if (name.startsWith("hbase.")
|| name.startsWith("hadoop.")
|| name.startsWith("zookeeper.")) {
conf.set(name, properties.getProperty(name));
}
}
prefixDate =
Boolean.parseBoolean(properties.getProperty("hazelcast.hbase.key.prefix.date", "false"));
int hbasePoolSize = Integer.parseInt(properties.getProperty("hazelcast.hbase.pool.size", "10"));
tableName = Bytes.toBytes(properties.getProperty("hazelcast.hbase.table", mapName));
family = Bytes.toBytes(properties.getProperty("hazelcast.hbase.column.family", "data"));
qualifier = Bytes.toBytes(properties.getProperty("hazelcast.hbase.column.qualifier", "json"));
pool = new HTablePool(conf, hbasePoolSize);
try {
HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
if (!hbaseAdmin.tableExists(tableName)) {
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor columnDesc = new HColumnDescriptor(family);
columnDesc.setCompressionType(Algorithm.LZO);
columnDesc.setBlockCacheEnabled(true);
columnDesc.setBlocksize(65536);
columnDesc.setInMemory(false);
columnDesc.setMaxVersions(1);
columnDesc.setTimeToLive(Integer.MAX_VALUE);
desc.addFamily(columnDesc);
hbaseAdmin.createTable(desc);
}
} catch (Exception e) {
throw new RuntimeException("Error creating table!", e);
}
// register with MapStoreRepository
MapStoreRepository.addMapStore(mapName, this);
}
private synchronized void initialiseTable() {
if (this.table == null) {
try {
HBaseStoreManager hbaseMgr = (HBaseStoreManager) storeMgr;
Configuration config = hbaseMgr.getHbaseConfig();
HBaseAdmin admin = new HBaseAdmin(config);
try {
if (!admin.tableExists(this.tableName)) {
if (!storeMgr.getSchemaHandler().isAutoCreateTables()) {
throw new NucleusUserException(Localiser.msg("040011", tableName));
}
NucleusLogger.VALUEGENERATION.debug(
"IncrementGenerator: Creating Table '" + this.tableName + "'");
HTableDescriptor ht = new HTableDescriptor(this.tableName);
HColumnDescriptor hcd = new HColumnDescriptor(INCREMENT_COL_NAME);
hcd.setCompressionType(Algorithm.NONE);
hcd.setMaxVersions(1);
ht.addFamily(hcd);
admin.createTable(ht);
}
} finally {
admin.close();
}
this.table = new HTable(config, this.tableName);
if (!this.table.exists(new Get(Bytes.toBytes(key)))) {
long initialValue = 0;
if (properties.containsKey("key-initial-value")) {
initialValue = Long.valueOf(properties.getProperty("key-initial-value")) - 1;
}
this.table.put(
new Put(Bytes.toBytes(key))
.add(
Bytes.toBytes(INCREMENT_COL_NAME),
Bytes.toBytes(INCREMENT_COL_NAME),
Bytes.toBytes(initialValue)));
}
} catch (IOException ex) {
NucleusLogger.VALUEGENERATION.fatal("Error instantiating IncrementGenerator", ex);
}
}
}
/**
* "name":"", "compression":"", "versions":"", "min_versions":"", "blocksize":"", "in_memory":"",
* "blockcache":"", "bloomfilter":"", "replication_scope":""
*
* @param hcd
* @param family
* @return
*/
private HColumnDescriptor getColumnDescriptor(JSONObject family) {
HColumnDescriptor hcd = null;
if (family == null) return hcd;
try {
if (family.has(XHBaseConstant.TABLE_DESC_FNAME)) {
hcd = new HColumnDescriptor((String) family.get(XHBaseConstant.TABLE_DESC_FNAME));
} else {
return hcd;
}
// set compression
if (family.has(XHBaseConstant.TABLE_DESC_COMPRESSION)) {
String compression = (String) family.get(XHBaseConstant.TABLE_DESC_COMPRESSION);
if (!compression.isEmpty()) {
if (compression.equalsIgnoreCase("gz")) {
hcd.setCompressionType(Compression.Algorithm.GZ);
} else if (compression.equalsIgnoreCase("lzo")) {
hcd.setCompressionType(Compression.Algorithm.LZO);
}
}
}
// set versions
if (family.has(XHBaseConstant.TABLE_DESC_VERSIONS)) {
String versions = (String) family.get(XHBaseConstant.TABLE_DESC_VERSIONS);
if (!versions.isEmpty()) {
hcd.setMaxVersions(Integer.valueOf(versions));
}
}
// set min versions
if (family.has(XHBaseConstant.TABLE_DESC_MINVERSIONS)) {
String min_versions = (String) family.get(XHBaseConstant.TABLE_DESC_MINVERSIONS);
if (!min_versions.isEmpty()) {
hcd.setMinVersions(Integer.valueOf(min_versions));
}
}
// set block size
if (family.has(XHBaseConstant.TABLE_DESC_BLOCKSIZE)) {
String block_size = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKSIZE);
if (!block_size.isEmpty()) {
hcd.setBlocksize(Integer.valueOf(block_size));
}
}
// set in memory
if (family.has(XHBaseConstant.TABLE_DESC_INMEMORY)) {
String in_memory = (String) family.get(XHBaseConstant.TABLE_DESC_INMEMORY);
if (!in_memory.isEmpty()) {
hcd.setInMemory(in_memory.equalsIgnoreCase("true") ? true : false);
}
}
// set block cache
if (family.has(XHBaseConstant.TABLE_DESC_BLOCKCACHE)) {
String block_cache = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKCACHE);
if (!block_cache.isEmpty()) {
hcd.setBlockCacheEnabled(block_cache.equalsIgnoreCase("true") ? true : false);
}
}
// set bloom filter
if (family.has(XHBaseConstant.TABLE_DESC_BLOOMFILTER)) {
String bloom_filter = (String) family.get(XHBaseConstant.TABLE_DESC_BLOOMFILTER);
if (!bloom_filter.isEmpty()) {
if (bloom_filter.equalsIgnoreCase("none")) {
hcd.setBloomFilterType(BloomType.NONE);
} else if (bloom_filter.equalsIgnoreCase("row")) {
hcd.setBloomFilterType(BloomType.ROW);
} else if (bloom_filter.equalsIgnoreCase("ROWCOL"))
hcd.setBloomFilterType(BloomType.ROWCOL); // TODO what is it?
}
}
// set replication scope
if (family.has(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE)) {
String replica_scope = (String) family.get(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE);
if (!replica_scope.isEmpty()) {
hcd.setScope(Integer.valueOf(replica_scope));
}
}
} catch (Exception e) {
e.printStackTrace();
}
return hcd;
}
