/** Apply column family options such as Bloom filters, compression, and data block encoding. */ protected void applyColumnFamilyOptions(byte[] tableName, byte[][] columnFamilies) throws IOException { HBaseAdmin admin = new HBaseAdmin(conf); HTableDescriptor tableDesc = admin.getTableDescriptor(tableName); LOG.info("Disabling table " + Bytes.toString(tableName)); admin.disableTable(tableName); for (byte[] cf : columnFamilies) { HColumnDescriptor columnDesc = tableDesc.getFamily(cf); boolean isNewCf = columnDesc == null; if (isNewCf) { columnDesc = new HColumnDescriptor(cf); } if (bloomType != null) { columnDesc.setBloomFilterType(bloomType); } if (compressAlgo != null) { columnDesc.setCompressionType(compressAlgo); } if (dataBlockEncodingAlgo != null) { columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo); columnDesc.setEncodeOnDisk(!encodeInCacheOnly); } if (inMemoryCF) { columnDesc.setInMemory(inMemoryCF); } if (isNewCf) { admin.addColumn(tableName, columnDesc); } else { admin.modifyColumn(tableName, columnDesc); } } LOG.info("Enabling table " + Bytes.toString(tableName)); admin.enableTable(tableName); }
/** * Enable in memory caching for .META. */ public static void setInfoFamilyCachingForMeta(final boolean b) { for (HColumnDescriptor hcd: HTableDescriptor.META_TABLEDESC.getColumnFamilies()) { if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) { hcd.setBlockCacheEnabled(b); hcd.setInMemory(b); } } }
/* (non-Javadoc) * @see com.mozilla.bagheera.hazelcast.persistence.MapStoreBase#init(com.hazelcast.core.HazelcastInstance, java.util.Properties, java.lang.String) */ @Override public void init(HazelcastInstance hazelcastInstance, Properties properties, String mapName) { super.init(hazelcastInstance, properties, mapName); Configuration conf = HBaseConfiguration.create(); for (String name : properties.stringPropertyNames()) { if (name.startsWith("hbase.") || name.startsWith("hadoop.") || name.startsWith("zookeeper.")) { conf.set(name, properties.getProperty(name)); } } prefixDate = Boolean.parseBoolean(properties.getProperty("hazelcast.hbase.key.prefix.date", "false")); int hbasePoolSize = Integer.parseInt(properties.getProperty("hazelcast.hbase.pool.size", "10")); tableName = Bytes.toBytes(properties.getProperty("hazelcast.hbase.table", mapName)); family = Bytes.toBytes(properties.getProperty("hazelcast.hbase.column.family", "data")); qualifier = Bytes.toBytes(properties.getProperty("hazelcast.hbase.column.qualifier", "json")); pool = new HTablePool(conf, hbasePoolSize); try { HBaseAdmin hbaseAdmin = new HBaseAdmin(conf); if (!hbaseAdmin.tableExists(tableName)) { HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor columnDesc = new HColumnDescriptor(family); columnDesc.setCompressionType(Algorithm.LZO); columnDesc.setBlockCacheEnabled(true); columnDesc.setBlocksize(65536); columnDesc.setInMemory(false); columnDesc.setMaxVersions(1); columnDesc.setTimeToLive(Integer.MAX_VALUE); desc.addFamily(columnDesc); hbaseAdmin.createTable(desc); } } catch (Exception e) { throw new RuntimeException("Error creating table!", e); } // register with MapStoreRepository MapStoreRepository.addMapStore(mapName, this); }
/** * "name":"", "compression":"", "versions":"", "min_versions":"", "blocksize":"", "in_memory":"", * "blockcache":"", "bloomfilter":"", "replication_scope":"" * * @param hcd * @param family * @return */ private HColumnDescriptor getColumnDescriptor(JSONObject family) { HColumnDescriptor hcd = null; if (family == null) return hcd; try { if (family.has(XHBaseConstant.TABLE_DESC_FNAME)) { hcd = new HColumnDescriptor((String) family.get(XHBaseConstant.TABLE_DESC_FNAME)); } else { return hcd; } // set compression if (family.has(XHBaseConstant.TABLE_DESC_COMPRESSION)) { String compression = (String) family.get(XHBaseConstant.TABLE_DESC_COMPRESSION); if (!compression.isEmpty()) { if (compression.equalsIgnoreCase("gz")) { hcd.setCompressionType(Compression.Algorithm.GZ); } else if (compression.equalsIgnoreCase("lzo")) { hcd.setCompressionType(Compression.Algorithm.LZO); } } } // set versions if (family.has(XHBaseConstant.TABLE_DESC_VERSIONS)) { String versions = (String) family.get(XHBaseConstant.TABLE_DESC_VERSIONS); if (!versions.isEmpty()) { hcd.setMaxVersions(Integer.valueOf(versions)); } } // set min versions if (family.has(XHBaseConstant.TABLE_DESC_MINVERSIONS)) { String min_versions = (String) family.get(XHBaseConstant.TABLE_DESC_MINVERSIONS); if (!min_versions.isEmpty()) { hcd.setMinVersions(Integer.valueOf(min_versions)); } } // set block size if (family.has(XHBaseConstant.TABLE_DESC_BLOCKSIZE)) { String block_size = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKSIZE); if (!block_size.isEmpty()) { hcd.setBlocksize(Integer.valueOf(block_size)); } } // set in memory if (family.has(XHBaseConstant.TABLE_DESC_INMEMORY)) { String in_memory = (String) family.get(XHBaseConstant.TABLE_DESC_INMEMORY); if (!in_memory.isEmpty()) { hcd.setInMemory(in_memory.equalsIgnoreCase("true") ? true : false); } } // set block cache if (family.has(XHBaseConstant.TABLE_DESC_BLOCKCACHE)) { String block_cache = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKCACHE); if (!block_cache.isEmpty()) { hcd.setBlockCacheEnabled(block_cache.equalsIgnoreCase("true") ? true : false); } } // set bloom filter if (family.has(XHBaseConstant.TABLE_DESC_BLOOMFILTER)) { String bloom_filter = (String) family.get(XHBaseConstant.TABLE_DESC_BLOOMFILTER); if (!bloom_filter.isEmpty()) { if (bloom_filter.equalsIgnoreCase("none")) { hcd.setBloomFilterType(BloomType.NONE); } else if (bloom_filter.equalsIgnoreCase("row")) { hcd.setBloomFilterType(BloomType.ROW); } else if (bloom_filter.equalsIgnoreCase("ROWCOL")) hcd.setBloomFilterType(BloomType.ROWCOL); // TODO what is it? } } // set replication scope if (family.has(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE)) { String replica_scope = (String) family.get(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE); if (!replica_scope.isEmpty()) { hcd.setScope(Integer.valueOf(replica_scope)); } } } catch (Exception e) { e.printStackTrace(); } return hcd; }