public static void main(String[] args) throws MasterNotRunningException, ZooKeeperConnectionException, IOException { Configuration conf = HBaseConfiguration.create(); conf.set( "hbase.zookeeper.quorum", "192.168.10.163:2181,192.168.10.164:2181,192.168.10.165:2181"); // admin用户 HBaseAdmin admin = new HBaseAdmin(conf); // table HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("people")); // info列簇 HColumnDescriptor hcd_info = new HColumnDescriptor("info"); hcd_info.setMaxVersions(3); // data列簇 HColumnDescriptor hcd_data = new HColumnDescriptor("data"); // 将列簇添加到htable中 htd.addFamily(hcd_info); htd.addFamily(hcd_data); // 创建表 admin.createTable(htd); // 关闭连接 admin.close(); }
public static void createTableTest(String tableStr) { try { Configuration conf = HBaseConfiguration.create(); byte[] tableName = Bytes.toBytes(tableStr); List<byte[]> families = new ArrayList<byte[]>(); families.add(f0); if (conf == null) { throw new IOException("conf not set."); } else { HBaseAdmin admin = new HBaseAdmin(conf); if (!admin.tableExists(tableName)) { HTableDescriptor tableDesc = new HTableDescriptor(tableName); for (byte[] family : families) { HColumnDescriptor hc = new HColumnDescriptor(family); hc.setMaxVersions(1); tableDesc.addFamily(hc); } admin.createTable(tableDesc); } else { throw new IOException("table " + new String(tableName) + " already exists."); } } System.out.println("DONE."); } catch (IOException e) { e.printStackTrace(); } }
// create 'mytable', {NAME=>'colfam:', COMPRESSION=>'lzo'} private synchronized HTableDescriptor createTableDescription( String tableName, String[] metrics, int[] max_version) { // log.info("entry: "+tableName + ":"+metrics); HTableDescriptor td = new HTableDescriptor(tableName); try { for (int i = 0; i < metrics.length; i++) { String colName = metrics[i]; if (colName == null || colName.length() == 0) { log.info("Invalid table schema content, contains empty name column."); throw new Exception("Invalid table schema content, contains empty name column."); } HColumnDescriptor hcd = new HColumnDescriptor(colName); hcd.setMaxVersions(max_version[i]); // hcd.setBloomFilterType(BloomType.ROWCOL); // compress it and require to install LZO // hcd.setCompressionType(Compression.Algorithm.GZ); td.addFamily(hcd); } // td.setMaxFileSize(1073741824); } catch (Exception e) { // log.error(e.fillInStackTrace()); e.printStackTrace(); } // log.info("exit"); return td; }
private HTableDescriptor getTableDesc(TableName tableName, byte[]... families) { HTableDescriptor htd = new HTableDescriptor(tableName); for (byte[] family : families) { HColumnDescriptor hcd = new HColumnDescriptor(family); // Set default to be three versions. hcd.setMaxVersions(Integer.MAX_VALUE); htd.addFamily(hcd); } return htd; }
public static synchronized void createTableIfNotExists() throws Exception { HBaseAdmin admin = HBaseUtils.getHBaseAdmin(); HTableDescriptor htd = new HTableDescriptor(TABLE_NAME); HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); htd.addFamily(hcd); hcd.setMaxVersions(1); if (!admin.tableExists(TABLE_NAME)) { admin.createTable(htd); } }
private HTable createTable(byte[] tableName, byte[][] columnFamilies) throws Exception { HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); for (byte[] family : columnFamilies) { HColumnDescriptor columnDesc = new HColumnDescriptor(family); columnDesc.setMaxVersions(Integer.MAX_VALUE); desc.addFamily(columnDesc); } desc.addCoprocessor(TransactionProcessor.class.getName()); hBaseAdmin.createTable(desc); testUtil.waitTableAvailable(tableName, 5000); return new HTable(testUtil.getConfiguration(), tableName); }
public static void createTableInner(String tableName, int splitSize, String splitPrefix) throws Exception { Configuration conf = HBaseConfiguration.create(); HBaseAdmin hAdmin = new HBaseAdmin(conf); HColumnDescriptor columnDescriptor = new HColumnDescriptor(Bytes.toBytes("f")); columnDescriptor.setCompressionType(Algorithm.SNAPPY); columnDescriptor.setMaxVersions(3); HTableDescriptor descriptor = new HTableDescriptor(TableName.valueOf(tableName)); descriptor.addFamily(columnDescriptor); hAdmin.createTable(descriptor, Util.genSplitKeysAlphaDig(splitPrefix, splitSize)); hAdmin.close(); }
@SuppressWarnings("deprecation") @Before public void setUp() throws Exception { HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor(family); hcd.setMobEnabled(true); hcd.setMobThreshold(3L); hcd.setMaxVersions(4); desc.addFamily(hcd); admin = TEST_UTIL.getHBaseAdmin(); admin.createTable(desc); table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()) .getBufferedMutator(TableName.valueOf(tableName)); }
public static void createTable(Configuration hbaseConf, String tableName) throws IOException { HBaseAdmin admin = new HBaseAdmin(hbaseConf); if (!admin.tableExists(tableName)) { HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); HColumnDescriptor datafam = new HColumnDescriptor(HBaseTimestampStorage.TSO_FAMILY); datafam.setMaxVersions(3); desc.addFamily(datafam); admin.createTable(desc); } if (admin.isTableDisabled(tableName)) { admin.enableTable(tableName); } admin.close(); LOG.info("Table {} created successfully", tableName); }
/* (non-Javadoc) * @see com.mozilla.bagheera.hazelcast.persistence.MapStoreBase#init(com.hazelcast.core.HazelcastInstance, java.util.Properties, java.lang.String) */ @Override public void init(HazelcastInstance hazelcastInstance, Properties properties, String mapName) { super.init(hazelcastInstance, properties, mapName); Configuration conf = HBaseConfiguration.create(); for (String name : properties.stringPropertyNames()) { if (name.startsWith("hbase.") || name.startsWith("hadoop.") || name.startsWith("zookeeper.")) { conf.set(name, properties.getProperty(name)); } } prefixDate = Boolean.parseBoolean(properties.getProperty("hazelcast.hbase.key.prefix.date", "false")); int hbasePoolSize = Integer.parseInt(properties.getProperty("hazelcast.hbase.pool.size", "10")); tableName = Bytes.toBytes(properties.getProperty("hazelcast.hbase.table", mapName)); family = Bytes.toBytes(properties.getProperty("hazelcast.hbase.column.family", "data")); qualifier = Bytes.toBytes(properties.getProperty("hazelcast.hbase.column.qualifier", "json")); pool = new HTablePool(conf, hbasePoolSize); try { HBaseAdmin hbaseAdmin = new HBaseAdmin(conf); if (!hbaseAdmin.tableExists(tableName)) { HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor columnDesc = new HColumnDescriptor(family); columnDesc.setCompressionType(Algorithm.LZO); columnDesc.setBlockCacheEnabled(true); columnDesc.setBlocksize(65536); columnDesc.setInMemory(false); columnDesc.setMaxVersions(1); columnDesc.setTimeToLive(Integer.MAX_VALUE); desc.addFamily(columnDesc); hbaseAdmin.createTable(desc); } } catch (Exception e) { throw new RuntimeException("Error creating table!", e); } // register with MapStoreRepository MapStoreRepository.addMapStore(mapName, this); }
private synchronized void initialiseTable() { if (this.table == null) { try { HBaseStoreManager hbaseMgr = (HBaseStoreManager) storeMgr; Configuration config = hbaseMgr.getHbaseConfig(); HBaseAdmin admin = new HBaseAdmin(config); try { if (!admin.tableExists(this.tableName)) { if (!storeMgr.getSchemaHandler().isAutoCreateTables()) { throw new NucleusUserException(Localiser.msg("040011", tableName)); } NucleusLogger.VALUEGENERATION.debug( "IncrementGenerator: Creating Table '" + this.tableName + "'"); HTableDescriptor ht = new HTableDescriptor(this.tableName); HColumnDescriptor hcd = new HColumnDescriptor(INCREMENT_COL_NAME); hcd.setCompressionType(Algorithm.NONE); hcd.setMaxVersions(1); ht.addFamily(hcd); admin.createTable(ht); } } finally { admin.close(); } this.table = new HTable(config, this.tableName); if (!this.table.exists(new Get(Bytes.toBytes(key)))) { long initialValue = 0; if (properties.containsKey("key-initial-value")) { initialValue = Long.valueOf(properties.getProperty("key-initial-value")) - 1; } this.table.put( new Put(Bytes.toBytes(key)) .add( Bytes.toBytes(INCREMENT_COL_NAME), Bytes.toBytes(INCREMENT_COL_NAME), Bytes.toBytes(initialValue))); } } catch (IOException ex) { NucleusLogger.VALUEGENERATION.fatal("Error instantiating IncrementGenerator", ex); } } }
public static void createAndConfigBaseTable( Configuration conf, byte[] tableName, byte[] columnFamily, String[] indexedColumnNames) throws IOException { // create a table with column familiy columnFamily HBaseAdmin admin = new HBaseAdmin(conf); HTableDescriptor desc = new HTableDescriptor(tableName); // specify indexable columns. for (int i = 0; i < indexedColumnNames.length; i++) { desc.setValue( INDEX_INDICATOR + (i + 1), Bytes.toString(columnFamily) + INDEX_DELIMITOR + indexedColumnNames[i]); } HColumnDescriptor descColFamily = new HColumnDescriptor(columnFamily); // configure to set KEEP_DELETED_CELLS => 'true' descColFamily.setKeepDeletedCells(true); descColFamily.setTimeToLive(HConstants.FOREVER); descColFamily.setMaxVersions(Integer.MAX_VALUE); desc.addFamily(descColFamily); admin.createTable(desc); }
/** * Verify schema modification takes. * * @throws IOException * @throws InterruptedException */ @Test(timeout = 300000) public void testOnlineChangeTableSchema() throws IOException, InterruptedException { final TableName tableName = TableName.valueOf("changeTableSchemaOnline"); TEST_UTIL .getMiniHBaseCluster() .getMaster() .getConfiguration() .setBoolean("hbase.online.schema.update.enable", true); HTableDescriptor[] tables = admin.listTables(); int numTables = tables.length; TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); tables = this.admin.listTables(); assertEquals(numTables + 1, tables.length); // FIRST, do htabledescriptor changes. HTableDescriptor htd = this.admin.getTableDescriptor(tableName); // Make a copy and assert copy is good. HTableDescriptor copy = new HTableDescriptor(htd); assertTrue(htd.equals(copy)); // Now amend the copy. Introduce differences. long newFlushSize = htd.getMemStoreFlushSize() / 2; if (newFlushSize <= 0) { newFlushSize = HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE / 2; } copy.setMemStoreFlushSize(newFlushSize); final String key = "anyoldkey"; assertTrue(htd.getValue(key) == null); copy.setValue(key, key); boolean expectedException = false; try { admin.modifyTable(tableName, copy); } catch (TableNotDisabledException re) { expectedException = true; } assertFalse(expectedException); HTableDescriptor modifiedHtd = this.admin.getTableDescriptor(tableName); assertFalse(htd.equals(modifiedHtd)); assertTrue(copy.equals(modifiedHtd)); assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize()); assertEquals(key, modifiedHtd.getValue(key)); // Now work on column family changes. int countOfFamilies = modifiedHtd.getFamilies().size(); assertTrue(countOfFamilies > 0); HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next(); int maxversions = hcd.getMaxVersions(); final int newMaxVersions = maxversions + 1; hcd.setMaxVersions(newMaxVersions); final byte[] hcdName = hcd.getName(); expectedException = false; try { this.admin.modifyColumn(tableName, hcd); } catch (TableNotDisabledException re) { expectedException = true; } assertFalse(expectedException); modifiedHtd = this.admin.getTableDescriptor(tableName); HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName); assertEquals(newMaxVersions, modifiedHcd.getMaxVersions()); // Try adding a column assertFalse(this.admin.isTableDisabled(tableName)); final String xtracolName = "xtracol"; HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName); xtracol.setValue(xtracolName, xtracolName); expectedException = false; try { this.admin.addColumn(tableName, xtracol); } catch (TableNotDisabledException re) { expectedException = true; } // Add column should work even if the table is enabled assertFalse(expectedException); modifiedHtd = this.admin.getTableDescriptor(tableName); hcd = modifiedHtd.getFamily(xtracol.getName()); assertTrue(hcd != null); assertTrue(hcd.getValue(xtracolName).equals(xtracolName)); // Delete the just-added column. this.admin.deleteColumn(tableName, xtracol.getName()); modifiedHtd = this.admin.getTableDescriptor(tableName); hcd = modifiedHtd.getFamily(xtracol.getName()); assertTrue(hcd == null); // Delete the table this.admin.disableTable(tableName); this.admin.deleteTable(tableName); this.admin.listTables(); assertFalse(this.admin.tableExists(tableName)); }
/** @throws java.lang.Exception */ @BeforeClass public static void setUpBeforeClass() throws Exception { conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); // smaller log roll size to trigger more events conf1.setFloat("hbase.regionserver.logroll.multiplier", 0.0003f); conf1.setInt("replication.source.size.capacity", 10240); conf1.setLong("replication.source.sleepforretries", 100); conf1.setInt("hbase.regionserver.maxlogs", 10); conf1.setLong("hbase.master.logcleaner.ttl", 10); conf1.setInt("zookeeper.recovery.retry", 1); conf1.setInt("zookeeper.recovery.retry.intervalmill", 10); conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT); conf1.setBoolean("dfs.support.append", true); conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); conf1.setInt("replication.stats.thread.period.seconds", 5); conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false); utility1 = new HBaseTestingUtility(conf1); utility1.startMiniZKCluster(); MiniZooKeeperCluster miniZK = utility1.getZkCluster(); // Have to reget conf1 in case zk cluster location different // than default conf1 = utility1.getConfiguration(); zkw1 = new ZooKeeperWatcher(conf1, "cluster1", null, true); admin = new ReplicationAdmin(conf1); LOG.info("Setup first Zk"); // Base conf2 on conf1 so it gets the right zk cluster. conf2 = HBaseConfiguration.create(conf1); conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); conf2.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT); conf2.setBoolean("dfs.support.append", true); conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false); utility2 = new HBaseTestingUtility(conf2); utility2.setZkCluster(miniZK); zkw2 = new ZooKeeperWatcher(conf2, "cluster2", null, true); admin.addPeer("2", utility2.getClusterKey()); LOG.info("Setup second Zk"); CONF_WITH_LOCALFS = HBaseConfiguration.create(conf1); utility1.startMiniCluster(2); utility2.startMiniCluster(2); HTableDescriptor table = new HTableDescriptor(TableName.valueOf(tableName)); HColumnDescriptor fam = new HColumnDescriptor(famName); fam.setMaxVersions(3); fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); table.addFamily(fam); fam = new HColumnDescriptor(noRepfamName); table.addFamily(fam); HBaseAdmin admin1 = new HBaseAdmin(conf1); HBaseAdmin admin2 = new HBaseAdmin(conf2); admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); htable1 = new HTable(conf1, tableName); htable1.setWriteBufferSize(1024); htable2 = new HTable(conf2, tableName); }
/** * "name":"", "compression":"", "versions":"", "min_versions":"", "blocksize":"", "in_memory":"", * "blockcache":"", "bloomfilter":"", "replication_scope":"" * * @param hcd * @param family * @return */ private HColumnDescriptor getColumnDescriptor(JSONObject family) { HColumnDescriptor hcd = null; if (family == null) return hcd; try { if (family.has(XHBaseConstant.TABLE_DESC_FNAME)) { hcd = new HColumnDescriptor((String) family.get(XHBaseConstant.TABLE_DESC_FNAME)); } else { return hcd; } // set compression if (family.has(XHBaseConstant.TABLE_DESC_COMPRESSION)) { String compression = (String) family.get(XHBaseConstant.TABLE_DESC_COMPRESSION); if (!compression.isEmpty()) { if (compression.equalsIgnoreCase("gz")) { hcd.setCompressionType(Compression.Algorithm.GZ); } else if (compression.equalsIgnoreCase("lzo")) { hcd.setCompressionType(Compression.Algorithm.LZO); } } } // set versions if (family.has(XHBaseConstant.TABLE_DESC_VERSIONS)) { String versions = (String) family.get(XHBaseConstant.TABLE_DESC_VERSIONS); if (!versions.isEmpty()) { hcd.setMaxVersions(Integer.valueOf(versions)); } } // set min versions if (family.has(XHBaseConstant.TABLE_DESC_MINVERSIONS)) { String min_versions = (String) family.get(XHBaseConstant.TABLE_DESC_MINVERSIONS); if (!min_versions.isEmpty()) { hcd.setMinVersions(Integer.valueOf(min_versions)); } } // set block size if (family.has(XHBaseConstant.TABLE_DESC_BLOCKSIZE)) { String block_size = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKSIZE); if (!block_size.isEmpty()) { hcd.setBlocksize(Integer.valueOf(block_size)); } } // set in memory if (family.has(XHBaseConstant.TABLE_DESC_INMEMORY)) { String in_memory = (String) family.get(XHBaseConstant.TABLE_DESC_INMEMORY); if (!in_memory.isEmpty()) { hcd.setInMemory(in_memory.equalsIgnoreCase("true") ? true : false); } } // set block cache if (family.has(XHBaseConstant.TABLE_DESC_BLOCKCACHE)) { String block_cache = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKCACHE); if (!block_cache.isEmpty()) { hcd.setBlockCacheEnabled(block_cache.equalsIgnoreCase("true") ? true : false); } } // set bloom filter if (family.has(XHBaseConstant.TABLE_DESC_BLOOMFILTER)) { String bloom_filter = (String) family.get(XHBaseConstant.TABLE_DESC_BLOOMFILTER); if (!bloom_filter.isEmpty()) { if (bloom_filter.equalsIgnoreCase("none")) { hcd.setBloomFilterType(BloomType.NONE); } else if (bloom_filter.equalsIgnoreCase("row")) { hcd.setBloomFilterType(BloomType.ROW); } else if (bloom_filter.equalsIgnoreCase("ROWCOL")) hcd.setBloomFilterType(BloomType.ROWCOL); // TODO what is it? } } // set replication scope if (family.has(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE)) { String replica_scope = (String) family.get(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE); if (!replica_scope.isEmpty()) { hcd.setScope(Integer.valueOf(replica_scope)); } } } catch (Exception e) { e.printStackTrace(); } return hcd; }
/** * Verify schema modification takes. * * @throws IOException */ @Test public void testChangeTableSchema() throws IOException { final byte[] tableName = Bytes.toBytes("changeTableSchema"); HTableDescriptor[] tables = admin.listTables(); int numTables = tables.length; TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY); tables = this.admin.listTables(); assertEquals(numTables + 1, tables.length); // FIRST, do htabledescriptor changes. HTableDescriptor htd = this.admin.getTableDescriptor(tableName); // Make a copy and assert copy is good. HTableDescriptor copy = new HTableDescriptor(htd); assertTrue(htd.equals(copy)); // Now amend the copy. Introduce differences. long newFlushSize = htd.getMemStoreFlushSize() / 2; copy.setMemStoreFlushSize(newFlushSize); final String key = "anyoldkey"; assertTrue(htd.getValue(key) == null); copy.setValue(key, key); boolean expectedException = false; try { this.admin.modifyTable(tableName, copy); } catch (TableNotDisabledException re) { expectedException = true; } assertTrue(expectedException); this.admin.disableTable(tableName); assertTrue(this.admin.isTableDisabled(tableName)); modifyTable(tableName, copy); HTableDescriptor modifiedHtd = this.admin.getTableDescriptor(tableName); // Assert returned modifiedhcd is same as the copy. assertFalse(htd.equals(modifiedHtd)); assertTrue(copy.equals(modifiedHtd)); assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize()); assertEquals(key, modifiedHtd.getValue(key)); // Reenable table to test it fails if not disabled. this.admin.enableTable(tableName); assertFalse(this.admin.isTableDisabled(tableName)); // Now work on column family changes. int countOfFamilies = modifiedHtd.getFamilies().size(); assertTrue(countOfFamilies > 0); HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next(); int maxversions = hcd.getMaxVersions(); final int newMaxVersions = maxversions + 1; hcd.setMaxVersions(newMaxVersions); final byte[] hcdName = hcd.getName(); expectedException = false; try { this.admin.modifyColumn(tableName, hcd); } catch (TableNotDisabledException re) { expectedException = true; } assertTrue(expectedException); this.admin.disableTable(tableName); assertTrue(this.admin.isTableDisabled(tableName)); // Modify Column is synchronous this.admin.modifyColumn(tableName, hcd); modifiedHtd = this.admin.getTableDescriptor(tableName); HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName); assertEquals(newMaxVersions, modifiedHcd.getMaxVersions()); // Try adding a column // Reenable table to test it fails if not disabled. this.admin.enableTable(tableName); assertFalse(this.admin.isTableDisabled(tableName)); final String xtracolName = "xtracol"; HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName); xtracol.setValue(xtracolName, xtracolName); try { this.admin.addColumn(tableName, xtracol); } catch (TableNotDisabledException re) { expectedException = true; } assertTrue(expectedException); this.admin.disableTable(tableName); assertTrue(this.admin.isTableDisabled(tableName)); this.admin.addColumn(tableName, xtracol); modifiedHtd = this.admin.getTableDescriptor(tableName); hcd = modifiedHtd.getFamily(xtracol.getName()); assertTrue(hcd != null); assertTrue(hcd.getValue(xtracolName).equals(xtracolName)); // Delete the just-added column. this.admin.deleteColumn(tableName, xtracol.getName()); modifiedHtd = this.admin.getTableDescriptor(tableName); hcd = modifiedHtd.getFamily(xtracol.getName()); assertTrue(hcd == null); // Delete the table this.admin.deleteTable(tableName); this.admin.listTables(); assertFalse(this.admin.tableExists(tableName)); }
public boolean doTest() throws InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, IOException, InterruptedException { /** create config */ conf_h = HBaseConfiguration.create(); conf_h.set("hbase.zookeeper.quorum", "localhost"); conf_h.set("hbase.zookeeper.property.clientPort", "2181"); Connection con_h = null; try { con_h = ConnectionFactory.createConnection(conf_h); } catch (IOException e) { e.printStackTrace(); } Admin admin = con_h.getAdmin(); HTableDescriptor tableDesc = new HTableDescriptor(tableName_chi); HColumnDescriptor colFamDesc = new HColumnDescriptor("count"); colFamDesc.setMaxVersions(1); tableDesc.addFamily(colFamDesc); admin.createTable(tableDesc); /** counting and insert in chiTable */ Scan scan = new Scan(); scan.addColumn(Bytes.toBytes("products"), Bytes.toBytes("product_category_id")); scan.addColumn(Bytes.toBytes("orders"), Bytes.toBytes("order_date")); // Creates a new Job with no particular Cluster Job job = Job.getInstance(conf_h, "Count"); // Job.getInstance(Configuration conf, String JobName) job.setJarByClass( ChiSquaredTest2_abc.class); // Set the Jar by finding where a given class came from // initTableMapperJob(String table, Scan scan, Class<? extends TableMapper> mapper, Class<?> // outputKeyClass, Class<?> outputValueClass, org.apache.hadoop.mapreduce.Job job) TableMapReduceUtil.initTableMapperJob( "retail_order", scan, Map1.class, Text.class, IntWritable.class, job); // initTableReducerJob(String table, Class<? extends TableReducer> reducer, // org.apache.hadoop.mapreduce.Job job) TableMapReduceUtil.initTableReducerJob("chiTable", Reduce1.class, job); // boolean waitForCompletion(boolean verbose), verbose - print the progress to the user job.waitForCompletion(true); // Submit the job to the cluster and wait for it to finish /** extract value from chiTable */ int totalY = 0; int totalN = 0; ArrayList<CellOfHTable> chiTable = new ArrayList<CellOfHTable>(); Table table_h = con_h.getTable(tableName_chi); Scan s = new Scan(); s.addFamily(Bytes.toBytes("count")); ResultScanner results = table_h.getScanner(s); for (Result r : results) { CellOfHTable c = new CellOfHTable( r.getRow(), r.getValue(Bytes.toBytes("count"), Bytes.toBytes("Y")) == null ? Bytes.toBytes(0) : r.getValue(Bytes.toBytes("count"), Bytes.toBytes("Y")), r.getValue(Bytes.toBytes("count"), Bytes.toBytes("N")) == null ? Bytes.toBytes(0) : r.getValue( Bytes.toBytes("count"), Bytes.toBytes("N"))); // (id, count_Y, count_N) chiTable.add(c); totalY = totalY + c.countY; totalN = totalN + c.countN; } results.close(); table_h.close(); admin.disableTable(tableName_chi); admin.deleteTable(tableName_chi); double chisquare = 0.0; for (int i = 0; i < chiTable.size(); i++) { CellOfHTable c = chiTable.get(i); double expectY = (double) (c.countY + c.countN) * (double) totalY / (double) (totalY + totalN); chisquare = chisquare + (((double) c.countY - expectY) * ((double) c.countY - expectY) / expectY); double expectN = (double) (c.countY + c.countN) * (double) totalN / (double) (totalY + totalN); chisquare = chisquare + (((double) c.countN - expectN) * ((double) c.countN - expectN) / expectN); } System.out.println(chisquare); ChiSquareDist csd = new ChiSquareDist((chiTable.size() - 1)); if (chisquare > csd.inverseF(1.0 - alpha)) { return true; } return false; }
@BeforeClass public static void setUpBeforeClass() throws Exception { conf1.setInt("hfile.format.version", 3); conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); conf1.setInt("replication.source.size.capacity", 10240); conf1.setLong("replication.source.sleepforretries", 100); conf1.setInt("hbase.regionserver.maxlogs", 10); conf1.setLong("hbase.master.logcleaner.ttl", 10); conf1.setInt("zookeeper.recovery.retry", 1); conf1.setInt("zookeeper.recovery.retry.intervalmill", 10); conf1.setBoolean("dfs.support.append", true); conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); conf1.setInt("replication.stats.thread.period.seconds", 5); conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false); conf1.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName()); conf1.setStrings( CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, TestCoprocessorForTagsAtSource.class.getName()); utility1 = new HBaseTestingUtility(conf1); utility1.startMiniZKCluster(); MiniZooKeeperCluster miniZK = utility1.getZkCluster(); // Have to reget conf1 in case zk cluster location different // than default conf1 = utility1.getConfiguration(); replicationAdmin = new ReplicationAdmin(conf1); LOG.info("Setup first Zk"); // Base conf2 on conf1 so it gets the right zk cluster. conf2 = HBaseConfiguration.create(conf1); conf2.setInt("hfile.format.version", 3); conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); conf2.setBoolean("dfs.support.append", true); conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false); conf2.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName()); conf2.setStrings( CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, TestCoprocessorForTagsAtSink.class.getName()); utility2 = new HBaseTestingUtility(conf2); utility2.setZkCluster(miniZK); replicationAdmin.addPeer("2", utility2.getClusterKey()); LOG.info("Setup second Zk"); utility1.startMiniCluster(2); utility2.startMiniCluster(2); HTableDescriptor table = new HTableDescriptor(TABLE_NAME); HColumnDescriptor fam = new HColumnDescriptor(FAMILY); fam.setMaxVersions(3); fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); table.addFamily(fam); try (Connection conn = ConnectionFactory.createConnection(conf1); Admin admin = conn.getAdmin()) { admin.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); } try (Connection conn = ConnectionFactory.createConnection(conf2); Admin admin = conn.getAdmin()) { admin.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); } htable1 = utility1.getConnection().getTable(TABLE_NAME); htable2 = utility2.getConnection().getTable(TABLE_NAME); }