public static void main(String[] args)
throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
Configuration conf = HBaseConfiguration.create();
conf.set(
"hbase.zookeeper.quorum", "192.168.10.163:2181,192.168.10.164:2181,192.168.10.165:2181");
// admin用户
HBaseAdmin admin = new HBaseAdmin(conf);
// table
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("people"));
// info列簇
HColumnDescriptor hcd_info = new HColumnDescriptor("info");
hcd_info.setMaxVersions(3);
// data列簇
HColumnDescriptor hcd_data = new HColumnDescriptor("data");
// 将列簇添加到htable中
htd.addFamily(hcd_info);
htd.addFamily(hcd_data);
// 创建表
admin.createTable(htd);
// 关闭连接
admin.close();
}
public static void createTableTest(String tableStr) {
try {
Configuration conf = HBaseConfiguration.create();
byte[] tableName = Bytes.toBytes(tableStr);
List<byte[]> families = new ArrayList<byte[]>();
families.add(f0);
if (conf == null) {
throw new IOException("conf not set.");
} else {
HBaseAdmin admin = new HBaseAdmin(conf);
if (!admin.tableExists(tableName)) {
HTableDescriptor tableDesc = new HTableDescriptor(tableName);
for (byte[] family : families) {
HColumnDescriptor hc = new HColumnDescriptor(family);
hc.setMaxVersions(1);
tableDesc.addFamily(hc);
}
admin.createTable(tableDesc);
} else {
throw new IOException("table " + new String(tableName) + " already exists.");
}
}
System.out.println("DONE.");
} catch (IOException e) {
e.printStackTrace();
}
}
// create 'mytable', {NAME=>'colfam:', COMPRESSION=>'lzo'}
private synchronized HTableDescriptor createTableDescription(
String tableName, String[] metrics, int[] max_version) {
// log.info("entry: "+tableName + ":"+metrics);
HTableDescriptor td = new HTableDescriptor(tableName);
try {
for (int i = 0; i < metrics.length; i++) {
String colName = metrics[i];
if (colName == null || colName.length() == 0) {
log.info("Invalid table schema content, contains empty name column.");
throw new Exception("Invalid table schema content, contains empty name column.");
}
HColumnDescriptor hcd = new HColumnDescriptor(colName);
hcd.setMaxVersions(max_version[i]);
// hcd.setBloomFilterType(BloomType.ROWCOL);
// compress it and require to install LZO
// hcd.setCompressionType(Compression.Algorithm.GZ);
td.addFamily(hcd);
}
// td.setMaxFileSize(1073741824);
} catch (Exception e) {
// log.error(e.fillInStackTrace());
e.printStackTrace();
}
// log.info("exit");
return td;
}
private HTableDescriptor getTableDesc(TableName tableName, byte[]... families) {
HTableDescriptor htd = new HTableDescriptor(tableName);
for (byte[] family : families) {
HColumnDescriptor hcd = new HColumnDescriptor(family);
// Set default to be three versions.
hcd.setMaxVersions(Integer.MAX_VALUE);
htd.addFamily(hcd);
}
return htd;
}
public static synchronized void createTableIfNotExists() throws Exception {
HBaseAdmin admin = HBaseUtils.getHBaseAdmin();
HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
htd.addFamily(hcd);
hcd.setMaxVersions(1);
if (!admin.tableExists(TABLE_NAME)) {
admin.createTable(htd);
}
}
private HTable createTable(byte[] tableName, byte[][] columnFamilies) throws Exception {
HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
for (byte[] family : columnFamilies) {
HColumnDescriptor columnDesc = new HColumnDescriptor(family);
columnDesc.setMaxVersions(Integer.MAX_VALUE);
desc.addFamily(columnDesc);
}
desc.addCoprocessor(TransactionProcessor.class.getName());
hBaseAdmin.createTable(desc);
testUtil.waitTableAvailable(tableName, 5000);
return new HTable(testUtil.getConfiguration(), tableName);
}
public static void createTableInner(String tableName, int splitSize, String splitPrefix)
throws Exception {
Configuration conf = HBaseConfiguration.create();
HBaseAdmin hAdmin = new HBaseAdmin(conf);
HColumnDescriptor columnDescriptor = new HColumnDescriptor(Bytes.toBytes("f"));
columnDescriptor.setCompressionType(Algorithm.SNAPPY);
columnDescriptor.setMaxVersions(3);
HTableDescriptor descriptor = new HTableDescriptor(TableName.valueOf(tableName));
descriptor.addFamily(columnDescriptor);
hAdmin.createTable(descriptor, Util.genSplitKeysAlphaDig(splitPrefix, splitSize));
hAdmin.close();
}
@SuppressWarnings("deprecation")
@Before
public void setUp() throws Exception {
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor hcd = new HColumnDescriptor(family);
hcd.setMobEnabled(true);
hcd.setMobThreshold(3L);
hcd.setMaxVersions(4);
desc.addFamily(hcd);
admin = TEST_UTIL.getHBaseAdmin();
admin.createTable(desc);
table =
ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())
.getBufferedMutator(TableName.valueOf(tableName));
}
public static void createTable(Configuration hbaseConf, String tableName) throws IOException {
HBaseAdmin admin = new HBaseAdmin(hbaseConf);
if (!admin.tableExists(tableName)) {
HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
HColumnDescriptor datafam = new HColumnDescriptor(HBaseTimestampStorage.TSO_FAMILY);
datafam.setMaxVersions(3);
desc.addFamily(datafam);
admin.createTable(desc);
}
if (admin.isTableDisabled(tableName)) {
admin.enableTable(tableName);
}
admin.close();
LOG.info("Table {} created successfully", tableName);
}
/* (non-Javadoc)
* @see com.mozilla.bagheera.hazelcast.persistence.MapStoreBase#init(com.hazelcast.core.HazelcastInstance, java.util.Properties, java.lang.String)
*/
@Override
public void init(HazelcastInstance hazelcastInstance, Properties properties, String mapName) {
super.init(hazelcastInstance, properties, mapName);
Configuration conf = HBaseConfiguration.create();
for (String name : properties.stringPropertyNames()) {
if (name.startsWith("hbase.")
|| name.startsWith("hadoop.")
|| name.startsWith("zookeeper.")) {
conf.set(name, properties.getProperty(name));
}
}
prefixDate =
Boolean.parseBoolean(properties.getProperty("hazelcast.hbase.key.prefix.date", "false"));
int hbasePoolSize = Integer.parseInt(properties.getProperty("hazelcast.hbase.pool.size", "10"));
tableName = Bytes.toBytes(properties.getProperty("hazelcast.hbase.table", mapName));
family = Bytes.toBytes(properties.getProperty("hazelcast.hbase.column.family", "data"));
qualifier = Bytes.toBytes(properties.getProperty("hazelcast.hbase.column.qualifier", "json"));
pool = new HTablePool(conf, hbasePoolSize);
try {
HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
if (!hbaseAdmin.tableExists(tableName)) {
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor columnDesc = new HColumnDescriptor(family);
columnDesc.setCompressionType(Algorithm.LZO);
columnDesc.setBlockCacheEnabled(true);
columnDesc.setBlocksize(65536);
columnDesc.setInMemory(false);
columnDesc.setMaxVersions(1);
columnDesc.setTimeToLive(Integer.MAX_VALUE);
desc.addFamily(columnDesc);
hbaseAdmin.createTable(desc);
}
} catch (Exception e) {
throw new RuntimeException("Error creating table!", e);
}
// register with MapStoreRepository
MapStoreRepository.addMapStore(mapName, this);
}
private synchronized void initialiseTable() {
if (this.table == null) {
try {
HBaseStoreManager hbaseMgr = (HBaseStoreManager) storeMgr;
Configuration config = hbaseMgr.getHbaseConfig();
HBaseAdmin admin = new HBaseAdmin(config);
try {
if (!admin.tableExists(this.tableName)) {
if (!storeMgr.getSchemaHandler().isAutoCreateTables()) {
throw new NucleusUserException(Localiser.msg("040011", tableName));
}
NucleusLogger.VALUEGENERATION.debug(
"IncrementGenerator: Creating Table '" + this.tableName + "'");
HTableDescriptor ht = new HTableDescriptor(this.tableName);
HColumnDescriptor hcd = new HColumnDescriptor(INCREMENT_COL_NAME);
hcd.setCompressionType(Algorithm.NONE);
hcd.setMaxVersions(1);
ht.addFamily(hcd);
admin.createTable(ht);
}
} finally {
admin.close();
}
this.table = new HTable(config, this.tableName);
if (!this.table.exists(new Get(Bytes.toBytes(key)))) {
long initialValue = 0;
if (properties.containsKey("key-initial-value")) {
initialValue = Long.valueOf(properties.getProperty("key-initial-value")) - 1;
}
this.table.put(
new Put(Bytes.toBytes(key))
.add(
Bytes.toBytes(INCREMENT_COL_NAME),
Bytes.toBytes(INCREMENT_COL_NAME),
Bytes.toBytes(initialValue)));
}
} catch (IOException ex) {
NucleusLogger.VALUEGENERATION.fatal("Error instantiating IncrementGenerator", ex);
}
}
}
public static void createAndConfigBaseTable(
Configuration conf, byte[] tableName, byte[] columnFamily, String[] indexedColumnNames)
throws IOException {
// create a table with column familiy columnFamily
HBaseAdmin admin = new HBaseAdmin(conf);
HTableDescriptor desc = new HTableDescriptor(tableName);
// specify indexable columns.
for (int i = 0; i < indexedColumnNames.length; i++) {
desc.setValue(
INDEX_INDICATOR + (i + 1),
Bytes.toString(columnFamily) + INDEX_DELIMITOR + indexedColumnNames[i]);
}
HColumnDescriptor descColFamily = new HColumnDescriptor(columnFamily);
// configure to set KEEP_DELETED_CELLS => 'true'
descColFamily.setKeepDeletedCells(true);
descColFamily.setTimeToLive(HConstants.FOREVER);
descColFamily.setMaxVersions(Integer.MAX_VALUE);
desc.addFamily(descColFamily);
admin.createTable(desc);
}
/**
* Verify schema modification takes.
*
* @throws IOException
* @throws InterruptedException
*/
@Test(timeout = 300000)
public void testOnlineChangeTableSchema() throws IOException, InterruptedException {
final TableName tableName = TableName.valueOf("changeTableSchemaOnline");
TEST_UTIL
.getMiniHBaseCluster()
.getMaster()
.getConfiguration()
.setBoolean("hbase.online.schema.update.enable", true);
HTableDescriptor[] tables = admin.listTables();
int numTables = tables.length;
TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
tables = this.admin.listTables();
assertEquals(numTables + 1, tables.length);
// FIRST, do htabledescriptor changes.
HTableDescriptor htd = this.admin.getTableDescriptor(tableName);
// Make a copy and assert copy is good.
HTableDescriptor copy = new HTableDescriptor(htd);
assertTrue(htd.equals(copy));
// Now amend the copy. Introduce differences.
long newFlushSize = htd.getMemStoreFlushSize() / 2;
if (newFlushSize <= 0) {
newFlushSize = HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE / 2;
}
copy.setMemStoreFlushSize(newFlushSize);
final String key = "anyoldkey";
assertTrue(htd.getValue(key) == null);
copy.setValue(key, key);
boolean expectedException = false;
try {
admin.modifyTable(tableName, copy);
} catch (TableNotDisabledException re) {
expectedException = true;
}
assertFalse(expectedException);
HTableDescriptor modifiedHtd = this.admin.getTableDescriptor(tableName);
assertFalse(htd.equals(modifiedHtd));
assertTrue(copy.equals(modifiedHtd));
assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize());
assertEquals(key, modifiedHtd.getValue(key));
// Now work on column family changes.
int countOfFamilies = modifiedHtd.getFamilies().size();
assertTrue(countOfFamilies > 0);
HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next();
int maxversions = hcd.getMaxVersions();
final int newMaxVersions = maxversions + 1;
hcd.setMaxVersions(newMaxVersions);
final byte[] hcdName = hcd.getName();
expectedException = false;
try {
this.admin.modifyColumn(tableName, hcd);
} catch (TableNotDisabledException re) {
expectedException = true;
}
assertFalse(expectedException);
modifiedHtd = this.admin.getTableDescriptor(tableName);
HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName);
assertEquals(newMaxVersions, modifiedHcd.getMaxVersions());
// Try adding a column
assertFalse(this.admin.isTableDisabled(tableName));
final String xtracolName = "xtracol";
HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName);
xtracol.setValue(xtracolName, xtracolName);
expectedException = false;
try {
this.admin.addColumn(tableName, xtracol);
} catch (TableNotDisabledException re) {
expectedException = true;
}
// Add column should work even if the table is enabled
assertFalse(expectedException);
modifiedHtd = this.admin.getTableDescriptor(tableName);
hcd = modifiedHtd.getFamily(xtracol.getName());
assertTrue(hcd != null);
assertTrue(hcd.getValue(xtracolName).equals(xtracolName));
// Delete the just-added column.
this.admin.deleteColumn(tableName, xtracol.getName());
modifiedHtd = this.admin.getTableDescriptor(tableName);
hcd = modifiedHtd.getFamily(xtracol.getName());
assertTrue(hcd == null);
// Delete the table
this.admin.disableTable(tableName);
this.admin.deleteTable(tableName);
this.admin.listTables();
assertFalse(this.admin.tableExists(tableName));
}
/** @throws java.lang.Exception */
@BeforeClass
public static void setUpBeforeClass() throws Exception {
conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
// smaller log roll size to trigger more events
conf1.setFloat("hbase.regionserver.logroll.multiplier", 0.0003f);
conf1.setInt("replication.source.size.capacity", 10240);
conf1.setLong("replication.source.sleepforretries", 100);
conf1.setInt("hbase.regionserver.maxlogs", 10);
conf1.setLong("hbase.master.logcleaner.ttl", 10);
conf1.setInt("zookeeper.recovery.retry", 1);
conf1.setInt("zookeeper.recovery.retry.intervalmill", 10);
conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
conf1.setBoolean("dfs.support.append", true);
conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
conf1.setInt("replication.stats.thread.period.seconds", 5);
conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
utility1 = new HBaseTestingUtility(conf1);
utility1.startMiniZKCluster();
MiniZooKeeperCluster miniZK = utility1.getZkCluster();
// Have to reget conf1 in case zk cluster location different
// than default
conf1 = utility1.getConfiguration();
zkw1 = new ZooKeeperWatcher(conf1, "cluster1", null, true);
admin = new ReplicationAdmin(conf1);
LOG.info("Setup first Zk");
// Base conf2 on conf1 so it gets the right zk cluster.
conf2 = HBaseConfiguration.create(conf1);
conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
conf2.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
conf2.setBoolean("dfs.support.append", true);
conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false);
utility2 = new HBaseTestingUtility(conf2);
utility2.setZkCluster(miniZK);
zkw2 = new ZooKeeperWatcher(conf2, "cluster2", null, true);
admin.addPeer("2", utility2.getClusterKey());
LOG.info("Setup second Zk");
CONF_WITH_LOCALFS = HBaseConfiguration.create(conf1);
utility1.startMiniCluster(2);
utility2.startMiniCluster(2);
HTableDescriptor table = new HTableDescriptor(TableName.valueOf(tableName));
HColumnDescriptor fam = new HColumnDescriptor(famName);
fam.setMaxVersions(3);
fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
table.addFamily(fam);
fam = new HColumnDescriptor(noRepfamName);
table.addFamily(fam);
HBaseAdmin admin1 = new HBaseAdmin(conf1);
HBaseAdmin admin2 = new HBaseAdmin(conf2);
admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
htable1 = new HTable(conf1, tableName);
htable1.setWriteBufferSize(1024);
htable2 = new HTable(conf2, tableName);
}
/**
* "name":"", "compression":"", "versions":"", "min_versions":"", "blocksize":"", "in_memory":"",
* "blockcache":"", "bloomfilter":"", "replication_scope":""
*
* @param hcd
* @param family
* @return
*/
private HColumnDescriptor getColumnDescriptor(JSONObject family) {
HColumnDescriptor hcd = null;
if (family == null) return hcd;
try {
if (family.has(XHBaseConstant.TABLE_DESC_FNAME)) {
hcd = new HColumnDescriptor((String) family.get(XHBaseConstant.TABLE_DESC_FNAME));
} else {
return hcd;
}
// set compression
if (family.has(XHBaseConstant.TABLE_DESC_COMPRESSION)) {
String compression = (String) family.get(XHBaseConstant.TABLE_DESC_COMPRESSION);
if (!compression.isEmpty()) {
if (compression.equalsIgnoreCase("gz")) {
hcd.setCompressionType(Compression.Algorithm.GZ);
} else if (compression.equalsIgnoreCase("lzo")) {
hcd.setCompressionType(Compression.Algorithm.LZO);
}
}
}
// set versions
if (family.has(XHBaseConstant.TABLE_DESC_VERSIONS)) {
String versions = (String) family.get(XHBaseConstant.TABLE_DESC_VERSIONS);
if (!versions.isEmpty()) {
hcd.setMaxVersions(Integer.valueOf(versions));
}
}
// set min versions
if (family.has(XHBaseConstant.TABLE_DESC_MINVERSIONS)) {
String min_versions = (String) family.get(XHBaseConstant.TABLE_DESC_MINVERSIONS);
if (!min_versions.isEmpty()) {
hcd.setMinVersions(Integer.valueOf(min_versions));
}
}
// set block size
if (family.has(XHBaseConstant.TABLE_DESC_BLOCKSIZE)) {
String block_size = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKSIZE);
if (!block_size.isEmpty()) {
hcd.setBlocksize(Integer.valueOf(block_size));
}
}
// set in memory
if (family.has(XHBaseConstant.TABLE_DESC_INMEMORY)) {
String in_memory = (String) family.get(XHBaseConstant.TABLE_DESC_INMEMORY);
if (!in_memory.isEmpty()) {
hcd.setInMemory(in_memory.equalsIgnoreCase("true") ? true : false);
}
}
// set block cache
if (family.has(XHBaseConstant.TABLE_DESC_BLOCKCACHE)) {
String block_cache = (String) family.get(XHBaseConstant.TABLE_DESC_BLOCKCACHE);
if (!block_cache.isEmpty()) {
hcd.setBlockCacheEnabled(block_cache.equalsIgnoreCase("true") ? true : false);
}
}
// set bloom filter
if (family.has(XHBaseConstant.TABLE_DESC_BLOOMFILTER)) {
String bloom_filter = (String) family.get(XHBaseConstant.TABLE_DESC_BLOOMFILTER);
if (!bloom_filter.isEmpty()) {
if (bloom_filter.equalsIgnoreCase("none")) {
hcd.setBloomFilterType(BloomType.NONE);
} else if (bloom_filter.equalsIgnoreCase("row")) {
hcd.setBloomFilterType(BloomType.ROW);
} else if (bloom_filter.equalsIgnoreCase("ROWCOL"))
hcd.setBloomFilterType(BloomType.ROWCOL); // TODO what is it?
}
}
// set replication scope
if (family.has(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE)) {
String replica_scope = (String) family.get(XHBaseConstant.TABLE_DESC_REPLICATIONSCOPE);
if (!replica_scope.isEmpty()) {
hcd.setScope(Integer.valueOf(replica_scope));
}
}
} catch (Exception e) {
e.printStackTrace();
}
return hcd;
}
/**
* Verify schema modification takes.
*
* @throws IOException
*/
@Test
public void testChangeTableSchema() throws IOException {
final byte[] tableName = Bytes.toBytes("changeTableSchema");
HTableDescriptor[] tables = admin.listTables();
int numTables = tables.length;
TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
tables = this.admin.listTables();
assertEquals(numTables + 1, tables.length);
// FIRST, do htabledescriptor changes.
HTableDescriptor htd = this.admin.getTableDescriptor(tableName);
// Make a copy and assert copy is good.
HTableDescriptor copy = new HTableDescriptor(htd);
assertTrue(htd.equals(copy));
// Now amend the copy. Introduce differences.
long newFlushSize = htd.getMemStoreFlushSize() / 2;
copy.setMemStoreFlushSize(newFlushSize);
final String key = "anyoldkey";
assertTrue(htd.getValue(key) == null);
copy.setValue(key, key);
boolean expectedException = false;
try {
this.admin.modifyTable(tableName, copy);
} catch (TableNotDisabledException re) {
expectedException = true;
}
assertTrue(expectedException);
this.admin.disableTable(tableName);
assertTrue(this.admin.isTableDisabled(tableName));
modifyTable(tableName, copy);
HTableDescriptor modifiedHtd = this.admin.getTableDescriptor(tableName);
// Assert returned modifiedhcd is same as the copy.
assertFalse(htd.equals(modifiedHtd));
assertTrue(copy.equals(modifiedHtd));
assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize());
assertEquals(key, modifiedHtd.getValue(key));
// Reenable table to test it fails if not disabled.
this.admin.enableTable(tableName);
assertFalse(this.admin.isTableDisabled(tableName));
// Now work on column family changes.
int countOfFamilies = modifiedHtd.getFamilies().size();
assertTrue(countOfFamilies > 0);
HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next();
int maxversions = hcd.getMaxVersions();
final int newMaxVersions = maxversions + 1;
hcd.setMaxVersions(newMaxVersions);
final byte[] hcdName = hcd.getName();
expectedException = false;
try {
this.admin.modifyColumn(tableName, hcd);
} catch (TableNotDisabledException re) {
expectedException = true;
}
assertTrue(expectedException);
this.admin.disableTable(tableName);
assertTrue(this.admin.isTableDisabled(tableName));
// Modify Column is synchronous
this.admin.modifyColumn(tableName, hcd);
modifiedHtd = this.admin.getTableDescriptor(tableName);
HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName);
assertEquals(newMaxVersions, modifiedHcd.getMaxVersions());
// Try adding a column
// Reenable table to test it fails if not disabled.
this.admin.enableTable(tableName);
assertFalse(this.admin.isTableDisabled(tableName));
final String xtracolName = "xtracol";
HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName);
xtracol.setValue(xtracolName, xtracolName);
try {
this.admin.addColumn(tableName, xtracol);
} catch (TableNotDisabledException re) {
expectedException = true;
}
assertTrue(expectedException);
this.admin.disableTable(tableName);
assertTrue(this.admin.isTableDisabled(tableName));
this.admin.addColumn(tableName, xtracol);
modifiedHtd = this.admin.getTableDescriptor(tableName);
hcd = modifiedHtd.getFamily(xtracol.getName());
assertTrue(hcd != null);
assertTrue(hcd.getValue(xtracolName).equals(xtracolName));
// Delete the just-added column.
this.admin.deleteColumn(tableName, xtracol.getName());
modifiedHtd = this.admin.getTableDescriptor(tableName);
hcd = modifiedHtd.getFamily(xtracol.getName());
assertTrue(hcd == null);
// Delete the table
this.admin.deleteTable(tableName);
this.admin.listTables();
assertFalse(this.admin.tableExists(tableName));
}
public boolean doTest()
throws InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException,
IOException, InterruptedException {
/** create config */
conf_h = HBaseConfiguration.create();
conf_h.set("hbase.zookeeper.quorum", "localhost");
conf_h.set("hbase.zookeeper.property.clientPort", "2181");
Connection con_h = null;
try {
con_h = ConnectionFactory.createConnection(conf_h);
} catch (IOException e) {
e.printStackTrace();
}
Admin admin = con_h.getAdmin();
HTableDescriptor tableDesc = new HTableDescriptor(tableName_chi);
HColumnDescriptor colFamDesc = new HColumnDescriptor("count");
colFamDesc.setMaxVersions(1);
tableDesc.addFamily(colFamDesc);
admin.createTable(tableDesc);
/** counting and insert in chiTable */
Scan scan = new Scan();
scan.addColumn(Bytes.toBytes("products"), Bytes.toBytes("product_category_id"));
scan.addColumn(Bytes.toBytes("orders"), Bytes.toBytes("order_date"));
// Creates a new Job with no particular Cluster
Job job =
Job.getInstance(conf_h, "Count"); // Job.getInstance(Configuration conf, String JobName)
job.setJarByClass(
ChiSquaredTest2_abc.class); // Set the Jar by finding where a given class came from
// initTableMapperJob(String table, Scan scan, Class<? extends TableMapper> mapper, Class<?>
// outputKeyClass, Class<?> outputValueClass, org.apache.hadoop.mapreduce.Job job)
TableMapReduceUtil.initTableMapperJob(
"retail_order", scan, Map1.class, Text.class, IntWritable.class, job);
// initTableReducerJob(String table, Class<? extends TableReducer> reducer,
// org.apache.hadoop.mapreduce.Job job)
TableMapReduceUtil.initTableReducerJob("chiTable", Reduce1.class, job);
// boolean waitForCompletion(boolean verbose), verbose - print the progress to the user
job.waitForCompletion(true); // Submit the job to the cluster and wait for it to finish
/** extract value from chiTable */
int totalY = 0;
int totalN = 0;
ArrayList<CellOfHTable> chiTable = new ArrayList<CellOfHTable>();
Table table_h = con_h.getTable(tableName_chi);
Scan s = new Scan();
s.addFamily(Bytes.toBytes("count"));
ResultScanner results = table_h.getScanner(s);
for (Result r : results) {
CellOfHTable c =
new CellOfHTable(
r.getRow(),
r.getValue(Bytes.toBytes("count"), Bytes.toBytes("Y")) == null
? Bytes.toBytes(0)
: r.getValue(Bytes.toBytes("count"), Bytes.toBytes("Y")),
r.getValue(Bytes.toBytes("count"), Bytes.toBytes("N")) == null
? Bytes.toBytes(0)
: r.getValue(
Bytes.toBytes("count"), Bytes.toBytes("N"))); // (id, count_Y, count_N)
chiTable.add(c);
totalY = totalY + c.countY;
totalN = totalN + c.countN;
}
results.close();
table_h.close();
admin.disableTable(tableName_chi);
admin.deleteTable(tableName_chi);
double chisquare = 0.0;
for (int i = 0; i < chiTable.size(); i++) {
CellOfHTable c = chiTable.get(i);
double expectY =
(double) (c.countY + c.countN) * (double) totalY / (double) (totalY + totalN);
chisquare =
chisquare + (((double) c.countY - expectY) * ((double) c.countY - expectY) / expectY);
double expectN =
(double) (c.countY + c.countN) * (double) totalN / (double) (totalY + totalN);
chisquare =
chisquare + (((double) c.countN - expectN) * ((double) c.countN - expectN) / expectN);
}
System.out.println(chisquare);
ChiSquareDist csd = new ChiSquareDist((chiTable.size() - 1));
if (chisquare > csd.inverseF(1.0 - alpha)) {
return true;
}
return false;
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
conf1.setInt("hfile.format.version", 3);
conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
conf1.setInt("replication.source.size.capacity", 10240);
conf1.setLong("replication.source.sleepforretries", 100);
conf1.setInt("hbase.regionserver.maxlogs", 10);
conf1.setLong("hbase.master.logcleaner.ttl", 10);
conf1.setInt("zookeeper.recovery.retry", 1);
conf1.setInt("zookeeper.recovery.retry.intervalmill", 10);
conf1.setBoolean("dfs.support.append", true);
conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
conf1.setInt("replication.stats.thread.period.seconds", 5);
conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
conf1.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
conf1.setStrings(
CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
TestCoprocessorForTagsAtSource.class.getName());
utility1 = new HBaseTestingUtility(conf1);
utility1.startMiniZKCluster();
MiniZooKeeperCluster miniZK = utility1.getZkCluster();
// Have to reget conf1 in case zk cluster location different
// than default
conf1 = utility1.getConfiguration();
replicationAdmin = new ReplicationAdmin(conf1);
LOG.info("Setup first Zk");
// Base conf2 on conf1 so it gets the right zk cluster.
conf2 = HBaseConfiguration.create(conf1);
conf2.setInt("hfile.format.version", 3);
conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
conf2.setBoolean("dfs.support.append", true);
conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false);
conf2.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
conf2.setStrings(
CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
TestCoprocessorForTagsAtSink.class.getName());
utility2 = new HBaseTestingUtility(conf2);
utility2.setZkCluster(miniZK);
replicationAdmin.addPeer("2", utility2.getClusterKey());
LOG.info("Setup second Zk");
utility1.startMiniCluster(2);
utility2.startMiniCluster(2);
HTableDescriptor table = new HTableDescriptor(TABLE_NAME);
HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
fam.setMaxVersions(3);
fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
table.addFamily(fam);
try (Connection conn = ConnectionFactory.createConnection(conf1);
Admin admin = conn.getAdmin()) {
admin.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
}
try (Connection conn = ConnectionFactory.createConnection(conf2);
Admin admin = conn.getAdmin()) {
admin.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
}
htable1 = utility1.getConnection().getTable(TABLE_NAME);
htable2 = utility2.getConnection().getTable(TABLE_NAME);
}
