/** Apply column family options such as Bloom filters, compression, and data block encoding. */
protected void applyColumnFamilyOptions(byte[] tableName, byte[][] columnFamilies)
throws IOException {
HBaseAdmin admin = new HBaseAdmin(conf);
HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
LOG.info("Disabling table " + Bytes.toString(tableName));
admin.disableTable(tableName);
for (byte[] cf : columnFamilies) {
HColumnDescriptor columnDesc = tableDesc.getFamily(cf);
boolean isNewCf = columnDesc == null;
if (isNewCf) {
columnDesc = new HColumnDescriptor(cf);
}
if (bloomType != null) {
columnDesc.setBloomFilterType(bloomType);
}
if (compressAlgo != null) {
columnDesc.setCompressionType(compressAlgo);
}
if (dataBlockEncodingAlgo != null) {
columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo);
columnDesc.setEncodeOnDisk(!encodeInCacheOnly);
}
if (inMemoryCF) {
columnDesc.setInMemory(inMemoryCF);
}
if (isNewCf) {
admin.addColumn(tableName, columnDesc);
} else {
admin.modifyColumn(tableName, columnDesc);
}
}
LOG.info("Enabling table " + Bytes.toString(tableName));
admin.enableTable(tableName);
}
/**
* Enable in memory caching for .META.
*/
public static void setInfoFamilyCachingForMeta(final boolean b) {
for (HColumnDescriptor hcd:
HTableDescriptor.META_TABLEDESC.getColumnFamilies()) {
if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
hcd.setBlockCacheEnabled(b);
hcd.setInMemory(b);
}
}
}
/* (non-Javadoc)
* @see com.mozilla.bagheera.hazelcast.persistence.MapStoreBase#init(com.hazelcast.core.HazelcastInstance, java.util.Properties, java.lang.String)
*/
@Override
public void init(HazelcastInstance hazelcastInstance, Properties properties, String mapName) {
super.init(hazelcastInstance, properties, mapName);
Configuration conf = HBaseConfiguration.create();
for (String name : properties.stringPropertyNames()) {
if (name.startsWith("hbase.")
|| name.startsWith("hadoop.")
|| name.startsWith("zookeeper.")) {
conf.set(name, properties.getProperty(name));
}
}
prefixDate =
Boolean.parseBoolean(properties.getProperty("hazelcast.hbase.key.prefix.date", "false"));
int hbasePoolSize = Integer.parseInt(properties.getProperty("hazelcast.hbase.pool.size", "10"));
tableName = Bytes.toBytes(properties.getProperty("hazelcast.hbase.table", mapName));
family = Bytes.toBytes(properties.getProperty("hazelcast.hbase.column.family", "data"));
qualifier = Bytes.toBytes(properties.getProperty("hazelcast.hbase.column.qualifier", "json"));
pool = new HTablePool(conf, hbasePoolSize);
try {
HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
if (!hbaseAdmin.tableExists(tableName)) {
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor columnDesc = new HColumnDescriptor(family);
columnDesc.setCompressionType(Algorithm.LZO);
columnDesc.setBlockCacheEnabled(true);
columnDesc.setBlocksize(65536);
columnDesc.setInMemory(false);
columnDesc.setMaxVersions(1);
columnDesc.setTimeToLive(Integer.MAX_VALUE);
desc.addFamily(columnDesc);
hbaseAdmin.createTable(desc);
}
} catch (Exception e) {
throw new RuntimeException("Error creating table!", e);
}
// register with MapStoreRepository
MapStoreRepository.addMapStore(mapName, this);
}
/**
* "name":"", "compression":"", "versions":"", "min_versions":"", "blocksize":"", "in_memory":"",
* "blockcache":"", "bloomfilter":"", "replication_scope":""
*
* @param hcd
* @param family
* @return
*/
private HColumnDescriptor getColumnDescriptor(JSONObject family) {
HColumnDescriptor hcd = null;
if (family == null) return hcd;
try {
if (family.has(XHBaseConstant.TABLE_DESC_FNAME)) {
hcd = new HColumnDescriptor((String) family.get(XHBaseConstant.TABLE_DESC_FNAME));
} else {
return hcd;
}
// set compression
if (family.has(XHBaseConstant.TABLE_DESC_COMPRESSION)) {
String compression = (String) family.get(XHBaseConstant.TABLE_DESC_COMPRESSION);
if (!compression.isEmpty()) {
if (compression.equalsIgnoreCase("gz")) {
hcd.setCompressionType(Compression.Algorithm.GZ);
} else if (compression.equalsIgnoreCase("lzo")) {
hcd.setCompressionType(Compression.Algorithm.LZO);
}
}
}
// set versions
if (family.has(XHBaseConstant.TABLE_DESC_VERSIONS)) {
String versions = (String) family.get(XHBaseConstant.TABLE_DESC_VERSIONS);
if (!versions.isEmpty()) {
hcd.setMaxVersions(Integer.valueOf(versions));
}
}
// set min versions
if (family.has(XHBaseConstant.TABLE_DESC_MINVERSIONS)) {
String min_versions = (String) family.get(XHBaseConstant.TABLE_DESC_MINVERSIONS);
if (!min_versions.isEmpty()) {
hcd.setMinVersions(Integer.valueOf(min_versions));
}
}
// set block size
if (family.has(XHBaseConstant.TABLE_DESC_BLOCKSIZE)) {
String block_size = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKSIZE);
if (!block_size.isEmpty()) {
hcd.setBlocksize(Integer.valueOf(block_size));
}
}
// set in memory
if (family.has(XHBaseConstant.TABLE_DESC_INMEMORY)) {
String in_memory = (String) family.get(XHBaseConstant.TABLE_DESC_INMEMORY);
if (!in_memory.isEmpty()) {
hcd.setInMemory(in_memory.equalsIgnoreCase("true") ? true : false);
}
}
// set block cache
if (family.has(XHBaseConstant.TABLE_DESC_BLOCKCACHE)) {
String block_cache = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKCACHE);
if (!block_cache.isEmpty()) {
hcd.setBlockCacheEnabled(block_cache.equalsIgnoreCase("true") ? true : false);
}
}
// set bloom filter
if (family.has(XHBaseConstant.TABLE_DESC_BLOOMFILTER)) {
String bloom_filter = (String) family.get(XHBaseConstant.TABLE_DESC_BLOOMFILTER);
if (!bloom_filter.isEmpty()) {
if (bloom_filter.equalsIgnoreCase("none")) {
hcd.setBloomFilterType(BloomType.NONE);
} else if (bloom_filter.equalsIgnoreCase("row")) {
hcd.setBloomFilterType(BloomType.ROW);
} else if (bloom_filter.equalsIgnoreCase("ROWCOL"))
hcd.setBloomFilterType(BloomType.ROWCOL); // TODO what is it?
}
}
// set replication scope
if (family.has(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE)) {
String replica_scope = (String) family.get(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE);
if (!replica_scope.isEmpty()) {
hcd.setScope(Integer.valueOf(replica_scope));
}
}
} catch (Exception e) {
e.printStackTrace();
}
return hcd;
}
