/** Apply column family options such as Bloom filters, compression, and data block encoding. */ protected void applyColumnFamilyOptions(byte[] tableName, byte[][] columnFamilies) throws IOException { HBaseAdmin admin = new HBaseAdmin(conf); HTableDescriptor tableDesc = admin.getTableDescriptor(tableName); LOG.info("Disabling table " + Bytes.toString(tableName)); admin.disableTable(tableName); for (byte[] cf : columnFamilies) { HColumnDescriptor columnDesc = tableDesc.getFamily(cf); boolean isNewCf = columnDesc == null; if (isNewCf) { columnDesc = new HColumnDescriptor(cf); } if (bloomType != null) { columnDesc.setBloomFilterType(bloomType); } if (compressAlgo != null) { columnDesc.setCompressionType(compressAlgo); } if (dataBlockEncodingAlgo != null) { columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo); columnDesc.setEncodeOnDisk(!encodeInCacheOnly); } if (inMemoryCF) { columnDesc.setInMemory(inMemoryCF); } if (isNewCf) { admin.addColumn(tableName, columnDesc); } else { admin.modifyColumn(tableName, columnDesc); } } LOG.info("Enabling table " + Bytes.toString(tableName)); admin.enableTable(tableName); }
public void runMergeWorkload() throws IOException { long maxKeyCount = prepareForMerge(); List<StoreFileScanner> scanners = StoreFileScanner.getScannersForStoreFiles(inputStoreFiles, false, false); HColumnDescriptor columnDescriptor = new HColumnDescriptor(HFileReadWriteTest.class.getSimpleName()); columnDescriptor.setBlocksize(blockSize); columnDescriptor.setBloomFilterType(bloomType); columnDescriptor.setCompressionType(compression); columnDescriptor.setDataBlockEncoding(dataBlockEncoding); HRegionInfo regionInfo = new HRegionInfo(); HTableDescriptor htd = new HTableDescriptor(TABLE_NAME); HRegion region = new HRegion(outputDir, null, fs, conf, regionInfo, htd, null); HStore store = new HStore(outputDir, region, columnDescriptor, fs, conf); StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, new CacheConfig(conf), fs, blockSize) .withOutputDir(outputDir) .withCompression(compression) .withDataBlockEncoder(dataBlockEncoder) .withBloomType(bloomType) .withMaxKeyCount(maxKeyCount) .withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE) .withBytesPerChecksum(HFile.DEFAULT_BYTES_PER_CHECKSUM) .build(); StatisticsPrinter statsPrinter = new StatisticsPrinter(); statsPrinter.startThread(); try { performMerge(scanners, store, writer); writer.close(); } finally { statsPrinter.requestStop(); } Path resultPath = writer.getPath(); resultPath = tryUsingSimpleOutputPath(resultPath); long fileSize = fs.getFileStatus(resultPath).getLen(); LOG.info("Created " + resultPath + ", size " + fileSize); System.out.println(); System.out.println("HFile information for " + resultPath); System.out.println(); HFilePrettyPrinter hfpp = new HFilePrettyPrinter(); hfpp.run(new String[] {"-m", "-f", resultPath.toString()}); }
private void runTest( String testName, TableName tableName, BloomType bloomType, boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges) throws Exception { HTableDescriptor htd = new HTableDescriptor(tableName); HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY); familyDesc.setBloomFilterType(bloomType); htd.addFamily(familyDesc); runTest(testName, htd, bloomType, preCreateTable, tableSplitKeys, hfileRanges); }
/** * Create a Mob table. * * @param util * @param tableName * @param families * @return An HTable instance for the created table. * @throws IOException */ public static Table createMobTable( final HBaseTestingUtility util, final TableName tableName, final byte[]... families) throws IOException { HTableDescriptor htd = new HTableDescriptor(tableName); for (byte[] family : families) { HColumnDescriptor hcd = new HColumnDescriptor(family); // Disable blooms (they are on by default as of 0.95) but we disable them // here because // tests have hard coded counts of what to expect in block cache, etc., // and blooms being // on is interfering. hcd.setBloomFilterType(BloomType.NONE); hcd.setMobEnabled(true); hcd.setMobThreshold(0L); htd.addFamily(hcd); } util.getHBaseAdmin().createTable(htd); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait // until they are assigned util.waitUntilAllRegionsAssigned(htd.getTableName()); return ConnectionFactory.createConnection(util.getConfiguration()).getTable(htd.getTableName()); }
/** * "name":"", "compression":"", "versions":"", "min_versions":"", "blocksize":"", "in_memory":"", * "blockcache":"", "bloomfilter":"", "replication_scope":"" * * @param hcd * @param family * @return */ private HColumnDescriptor getColumnDescriptor(JSONObject family) { HColumnDescriptor hcd = null; if (family == null) return hcd; try { if (family.has(XHBaseConstant.TABLE_DESC_FNAME)) { hcd = new HColumnDescriptor((String) family.get(XHBaseConstant.TABLE_DESC_FNAME)); } else { return hcd; } // set compression if (family.has(XHBaseConstant.TABLE_DESC_COMPRESSION)) { String compression = (String) family.get(XHBaseConstant.TABLE_DESC_COMPRESSION); if (!compression.isEmpty()) { if (compression.equalsIgnoreCase("gz")) { hcd.setCompressionType(Compression.Algorithm.GZ); } else if (compression.equalsIgnoreCase("lzo")) { hcd.setCompressionType(Compression.Algorithm.LZO); } } } // set versions if (family.has(XHBaseConstant.TABLE_DESC_VERSIONS)) { String versions = (String) family.get(XHBaseConstant.TABLE_DESC_VERSIONS); if (!versions.isEmpty()) { hcd.setMaxVersions(Integer.valueOf(versions)); } } // set min versions if (family.has(XHBaseConstant.TABLE_DESC_MINVERSIONS)) { String min_versions = (String) family.get(XHBaseConstant.TABLE_DESC_MINVERSIONS); if (!min_versions.isEmpty()) { hcd.setMinVersions(Integer.valueOf(min_versions)); } } // set block size if (family.has(XHBaseConstant.TABLE_DESC_BLOCKSIZE)) { String block_size = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKSIZE); if (!block_size.isEmpty()) { hcd.setBlocksize(Integer.valueOf(block_size)); } } // set in memory if (family.has(XHBaseConstant.TABLE_DESC_INMEMORY)) { String in_memory = (String) family.get(XHBaseConstant.TABLE_DESC_INMEMORY); if (!in_memory.isEmpty()) { hcd.setInMemory(in_memory.equalsIgnoreCase("true") ? true : false); } } // set block cache if (family.has(XHBaseConstant.TABLE_DESC_BLOCKCACHE)) { String block_cache = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKCACHE); if (!block_cache.isEmpty()) { hcd.setBlockCacheEnabled(block_cache.equalsIgnoreCase("true") ? true : false); } } // set bloom filter if (family.has(XHBaseConstant.TABLE_DESC_BLOOMFILTER)) { String bloom_filter = (String) family.get(XHBaseConstant.TABLE_DESC_BLOOMFILTER); if (!bloom_filter.isEmpty()) { if (bloom_filter.equalsIgnoreCase("none")) { hcd.setBloomFilterType(BloomType.NONE); } else if (bloom_filter.equalsIgnoreCase("row")) { hcd.setBloomFilterType(BloomType.ROW); } else if (bloom_filter.equalsIgnoreCase("ROWCOL")) hcd.setBloomFilterType(BloomType.ROWCOL); // TODO what is it? } } // set replication scope if (family.has(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE)) { String replica_scope = (String) family.get(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE); if (!replica_scope.isEmpty()) { hcd.setScope(Integer.valueOf(replica_scope)); } } } catch (Exception e) { e.printStackTrace(); } return hcd; }