@Override protected int doWork() throws Exception { Connection connection = null; Admin admin = null; try { connection = ConnectionFactory.createConnection(getConf()); admin = connection.getAdmin(); HBaseProtos.SnapshotDescription.Type type = HBaseProtos.SnapshotDescription.Type.FLUSH; if (snapshotType != null) { type = ProtobufUtil.createProtosSnapShotDescType(snapshotName); } admin.snapshot( new SnapshotDescription(snapshotName, tableName, ProtobufUtil.createSnapshotType(type))); } catch (Exception e) { return -1; } finally { if (admin != null) { admin.close(); } if (connection != null) { connection.close(); } } return 0; }
/** * Sweeps the mob files on one column family. It deletes the unused mob files and merges the small * mob files into bigger ones. * * @param tableName The current table name in string format. * @param familyName The column family name. * @return 0 if success, 2 if job aborted with an exception, 3 if unable to start due to other * compaction,4 if mr job was unsuccessful * @throws IOException * @throws InterruptedException * @throws ClassNotFoundException * @throws KeeperException * @throws ServiceException */ int sweepFamily(String tableName, String familyName) throws IOException, InterruptedException, ClassNotFoundException, KeeperException, ServiceException { Configuration conf = getConf(); // make sure the target HBase exists. HBaseAdmin.checkHBaseAvailable(conf); Connection connection = ConnectionFactory.createConnection(getConf()); Admin admin = connection.getAdmin(); try { FileSystem fs = FileSystem.get(conf); TableName tn = TableName.valueOf(tableName); HTableDescriptor htd = admin.getTableDescriptor(tn); HColumnDescriptor family = htd.getFamily(Bytes.toBytes(familyName)); if (family == null || !family.isMobEnabled()) { throw new IOException("Column family " + familyName + " is not a MOB column family"); } SweepJob job = new SweepJob(conf, fs); // Run the sweeping return job.sweep(tn, family); } catch (Exception e) { System.err.println("Job aborted due to exception " + e); return 2; // job failed } finally { try { admin.close(); } catch (IOException e) { System.out.println("Failed to close the HBaseAdmin: " + e.getMessage()); } try { connection.close(); } catch (IOException e) { System.out.println("Failed to close the connection: " + e.getMessage()); } } }
private static void createTable() throws Exception { try { Configuration configuration = HBaseConfiguration.create(); HBaseAdmin.checkHBaseAvailable(configuration); Connection connection = ConnectionFactory.createConnection(configuration); // Instantiating HbaseAdmin class Admin admin = connection.getAdmin(); // Instantiating table descriptor class HTableDescriptor stockTableDesc = new HTableDescriptor(TableName.valueOf(Constants.STOCK_DATES_TABLE)); // Adding column families to table descriptor HColumnDescriptor stock_0414 = new HColumnDescriptor(Constants.STOCK_DATES_CF); stockTableDesc.addFamily(stock_0414); // Execute the table through admin if (!admin.tableExists(stockTableDesc.getTableName())) { admin.createTable(stockTableDesc); System.out.println("Stock table created !!!"); } // Load hbase-site.xml HBaseConfiguration.addHbaseResources(configuration); } catch (ServiceException e) { log.error("Error occurred while creating HBase tables", e); throw new Exception("Error occurred while creating HBase tables", e); } }
public Map<String, Long> getRegionSizes(String tableName) { Map<String, Long> regions = new HashMap<>(); try { final Table table = connection.getTable(TableName.valueOf(tableName)); RegionLocator regionLocator = connection.getRegionLocator(table.getName()); List<HRegionLocation> tableRegionInfos = regionLocator.getAllRegionLocations(); Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR); for (HRegionLocation regionInfo : tableRegionInfos) { tableRegions.add(regionInfo.getRegionInfo().getRegionName()); } ClusterStatus clusterStatus = connection.getAdmin().getClusterStatus(); Collection<ServerName> servers = clusterStatus.getServers(); final long megaByte = 1024L * 1024L; for (ServerName serverName : servers) { ServerLoad serverLoad = clusterStatus.getLoad(serverName); for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) { byte[] regionId = regionLoad.getName(); if (tableRegions.contains(regionId)) { long regionSizeBytes = regionLoad.getStorefileSizeMB() * megaByte; regions.put(regionLoad.getNameAsString(), regionSizeBytes); } } } } catch (IOException e) { e.printStackTrace(); } return regions; }
public static boolean createTableOrOverwrite(String tableName, String[] columnFamily) { if (tableName == null && columnFamily == null) { return false; } Connection connection = null; try { connection = ConnectionFactory.createConnection(config); Admin admin = connection.getAdmin(); if (admin.tableExists(TableName.valueOf(tableName))) { admin.disableTable(TableName.valueOf(tableName)); admin.deleteTable(TableName.valueOf(tableName)); } HTableDescriptor table = new HTableDescriptor(TableName.valueOf(tableName)); for (String cf : columnFamily) { table.addFamily(new HColumnDescriptor(cf)); } admin.createTable(table); System.out.println("create table successfully."); return true; } catch (IOException e) { e.printStackTrace(); return false; } finally { try { connection.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } }
/** * 查询所有表 * * @return 所以表 / null * @throws Exception */ public static List<HTableDescriptor> queryALLTable() throws Exception { Connection conn = null; HBaseAdmin admin = null; try { conn = ConnectionFactory.createConnection(conf); admin = (HBaseAdmin) conn.getAdmin(); if (admin != null) { HTableDescriptor[] listTables = admin.listTables(); if (null != listTables && listTables.length > 0) { return Arrays.asList(listTables); } } } catch (Exception e) { e.printStackTrace(); } finally { try { if (null != admin) { admin.close(); } } catch (IOException e) { logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage()); } try { if (null != conn) { conn.close(); } } catch (Exception e) { logger.error("Connection close exception, errMsg:{}", e.getMessage()); } } return null; }
/** * 删除指定表名 * * @param rowKey */ public void deleteTable(byte[] rowKey) { Connection conn = null; HBaseAdmin admin = null; try { conn = ConnectionFactory.createConnection(conf); admin = (HBaseAdmin) conn.getAdmin(); // 在删除一张表前,要使其失效 admin.disableTable(rowKey); admin.deleteTable(rowKey); admin.enableTable(rowKey); } catch (Exception e) { logger.error("HBaseAdmin deleteTable exception, errMsg:{}", e.getMessage()); } finally { try { if (null != admin) { admin.close(); } } catch (IOException e) { logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage()); } try { if (null != conn) { conn.close(); } } catch (Exception e) { logger.error("Connection close exception, errMsg:{}", e.getMessage()); } } }
/** * 删除指定名称的列簇 * * @param tableName 表名 * @param columnFamilyName 列族 */ public static void deleteFamily(byte[] tableName, String columnFamilyName) { Connection conn = null; HBaseAdmin admin = null; try { conn = ConnectionFactory.createConnection(conf); admin = (HBaseAdmin) conn.getAdmin(); admin.deleteColumn(tableName, columnFamilyName); } catch (Exception e) { logger.error("HBaseAdmin deleteColumn exception, errMsg:{}", e.getMessage()); } finally { try { if (null != admin) { admin.close(); } } catch (IOException e) { logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage()); } try { if (null != conn) { conn.close(); } } catch (Exception e) { logger.error("Connection close exception, errMsg:{}", e.getMessage()); } } }
/** * Find all column families that are replicated from this cluster * * @return the full list of the replicated column families of this cluster as: tableName, family * name, replicationType * <p>Currently replicationType is Global. In the future, more replication types may be * extended here. For example 1) the replication may only apply to selected peers instead of * all peers 2) the replicationType may indicate the host Cluster servers as Slave for the * table:columnFam. */ public List<HashMap<String, String>> listReplicated() throws IOException { List<HashMap<String, String>> replicationColFams = new ArrayList<HashMap<String, String>>(); Admin admin = connection.getAdmin(); HTableDescriptor[] tables; try { tables = admin.listTables(); } finally { if (admin != null) admin.close(); } for (HTableDescriptor table : tables) { HColumnDescriptor[] columns = table.getColumnFamilies(); String tableName = table.getNameAsString(); for (HColumnDescriptor column : columns) { if (column.getScope() != HConstants.REPLICATION_SCOPE_LOCAL) { // At this moment, the columfam is replicated to all peers HashMap<String, String> replicationEntry = new HashMap<String, String>(); replicationEntry.put(TNAME, tableName); replicationEntry.put(CFNAME, column.getNameAsString()); replicationEntry.put(REPLICATIONTYPE, REPLICATIONGLOBAL); replicationColFams.add(replicationEntry); } } } return replicationColFams; }
public boolean tableExists(String tableName) { try { Admin admin = connection.getAdmin(); return admin.tableExists(TableName.valueOf(tableName)); } catch (IOException e) { e.printStackTrace(); } return false; }
protected HTableDescriptor[] getTables(final Configuration configuration) throws IOException { HTableDescriptor[] htbls = null; try (Connection connection = ConnectionFactory.createConnection(configuration)) { try (Admin admin = connection.getAdmin()) { htbls = admin.listTables(); } } return htbls; }
public static Admin getAadmin(Connection connection) { Admin admin = null; try { admin = connection.getAdmin(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } return admin; }
public void createTable(String tableName, List<String> columnFamilies) { try { Admin admin = connection.getAdmin(); HTableDescriptor descriptor = new HTableDescriptor(TableName.valueOf(tableName)); for (String family : columnFamilies) { descriptor.addFamily(new HColumnDescriptor(family)); } admin.createTable(descriptor); } catch (IOException e) { e.printStackTrace(); } }
@Before public void init() { conf = HBaseConfiguration.create(); try { conn = ConnectionFactory.createConnection(conf); admin = conn.getAdmin(); tn = TableName.valueOf(tableName); table = conn.getTable(tn); initData(); } catch (Exception e) { e.printStackTrace(); } }
public boolean removeTable(String tableName) { try { Admin admin = connection.getAdmin(); TableName t = TableName.valueOf(tableName); if (admin.tableExists(t)) { admin.disableTable(t); admin.deleteTable(t); return true; } } catch (IOException e) { e.printStackTrace(); } return false; }
protected void verifyNamespaces() throws IOException { Connection connection = getConnection(); Admin admin = connection.getAdmin(); // iterating concurrent map for (String nsName : namespaceMap.keySet()) { try { Assert.assertTrue( "Namespace: " + nsName + " in namespaceMap does not exist", admin.getNamespaceDescriptor(nsName) != null); } catch (NamespaceNotFoundException nsnfe) { Assert.fail( "Namespace: " + nsName + " in namespaceMap does not exist: " + nsnfe.getMessage()); } } admin.close(); }
@Test(timeout = 60000) public void testPriorityRegionIsOpenedWithSeparateThreadPool() throws Exception { ThreadPoolExecutor exec = getRS().getExecutorService().getExecutorThreadPool(ExecutorType.RS_OPEN_PRIORITY_REGION); assertEquals(0, exec.getCompletedTaskCount()); HTableDescriptor htd = new HTableDescriptor(tableName); htd.setPriority(HConstants.HIGH_QOS); htd.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration()); Admin admin = connection.getAdmin()) { admin.createTable(htd); } assertEquals(1, exec.getCompletedTaskCount()); }
public void manualTest(String args[]) throws Exception { Configuration conf = HBaseConfiguration.create(); util = new HBaseTestingUtility(conf); if ("newtable".equals(args[0])) { TableName tname = TableName.valueOf(args[1]); byte[][] splitKeys = generateRandomSplitKeys(4); try (Table table = util.createTable(tname, FAMILIES, splitKeys)) {} } else if ("incremental".equals(args[0])) { TableName tname = TableName.valueOf(args[1]); try (Connection c = ConnectionFactory.createConnection(conf); Admin admin = c.getAdmin(); RegionLocator regionLocator = c.getRegionLocator(tname)) { Path outDir = new Path("incremental-out"); runIncrementalPELoad(conf, admin.getTableDescriptor(tname), regionLocator, outDir); } } else { throw new RuntimeException("usage: TestHFileOutputFormat2 newtable | incremental"); } }
protected void verifyTables() throws IOException { Connection connection = getConnection(); Admin admin = connection.getAdmin(); // iterating concurrent map for (TableName tableName : enabledTables.keySet()) { Assert.assertTrue( "Table: " + tableName + " in enabledTables is not enabled", admin.isTableEnabled(tableName)); } for (TableName tableName : disabledTables.keySet()) { Assert.assertTrue( "Table: " + tableName + " in disabledTables is not disabled", admin.isTableDisabled(tableName)); } for (TableName tableName : deletedTables.keySet()) { Assert.assertFalse( "Table: " + tableName + " in deletedTables is not deleted", admin.tableExists(tableName)); } admin.close(); }
/** * 创建一张表 * * @param table * @throws ParamIsNullException 参数为空 * @throws TableExistsException 表已存在 */ public static void createTable(HTableDescriptor table) throws ParamIsNullException, TableExistsException { if (null == table) { throw new ParamIsNullException("参数不能为空"); } logger.info("create table begin... , table:{}", table.toString()); Connection conn = null; HBaseAdmin admin = null; String tableName = table.getNameAsString(); try { logger.info("获取connection"); conn = ConnectionFactory.createConnection(conf); logger.info("获取admin"); admin = (HBaseAdmin) conn.getAdmin(); /** 表已存在 */ if (admin.tableExists(Bytes.toBytes(tableName))) { throw new TableExistsException(tableName); } logger.info("create..."); admin.createTable(table); logger.info("table create success, tableName:{}", tableName); } catch (IOException e) { logger.error("table create fail, tableName:{}, errMsg:{}", tableName, e); } finally { if (null != admin) { try { admin.close(); } catch (IOException e) { logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage()); } } if (null != conn) { try { conn.close(); } catch (IOException e) { logger.error("Connection close exception, errMsg:{}", e.getMessage()); } } } }
/** * 添加一个列族 * * @param tableName * @param family 列族 * @throws ParamIsNullException 参数为空 * @throws TableNotFoundException 表不存在/获取表连接失败 */ public static void addColumnFamily(String tableName, HColumnDescriptor family) throws ParamIsNullException, TableNotFoundException { if (null == tableName) { throw new ParamIsNullException("tableName不能为空"); } if (null == family) { throw new ParamIsNullException("HColumnDescriptor不能为空"); } Connection conn = null; HBaseAdmin admin = null; try { conn = ConnectionFactory.createConnection(conf); admin = (HBaseAdmin) conn.getAdmin(); /** 表不存在 */ if (!admin.tableExists(Bytes.toBytes(tableName))) { throw new TableNotFoundException(tableName); } HTableDescriptor table = admin.getTableDescriptor(Bytes.toBytes(tableName)); table.addFamily(family); } catch (IOException e) { logger.error("获取Hbase连接发生异常, errMsg:{}", e.getMessage()); } finally { if (null != admin) { try { admin.close(); } catch (IOException e) { logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage()); } } if (null != conn) { try { conn.close(); } catch (IOException e) { logger.error("Connection close exception, errMsg:{}", e.getMessage()); } } } }
public static void main(String[] args) throws IOException, InterruptedException { Configuration conf = HBaseConfiguration.create(); HBaseHelper helper = HBaseHelper.getHelper(conf); helper.dropTable("testtable"); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); // vv CreateTableWithNamespaceExample /*[*/ NamespaceDescriptor namespace = NamespaceDescriptor.create("testspace").build(); admin.createNamespace(namespace); /*]*/ TableName tableName = TableName.valueOf("testspace", "testtable"); HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor coldef = new HColumnDescriptor(Bytes.toBytes("colfam1")); desc.addFamily(coldef); admin.createTable(desc); // ^^ CreateTableWithNamespaceExample boolean avail = admin.isTableAvailable(tableName); System.out.println("Table available: " + avail); }
/** * Connect to peer and check the table descriptor on peer: * * <ol> * <li>Create the same table on peer when not exist. * <li>Throw exception if the table exists on peer cluster but descriptors are not same. * </ol> * * @param tableName name of the table to sync to the peer * @param splits table split keys * @throws IOException */ private void checkAndSyncTableDescToPeers(final TableName tableName, final byte[][] splits) throws IOException { List<ReplicationPeer> repPeers = listValidReplicationPeers(); if (repPeers == null || repPeers.size() <= 0) { throw new IllegalArgumentException("Found no peer cluster for replication."); } for (ReplicationPeer repPeer : repPeers) { Configuration peerConf = repPeer.getConfiguration(); HTableDescriptor htd = null; try (Connection conn = ConnectionFactory.createConnection(peerConf); Admin admin = this.connection.getAdmin(); Admin repHBaseAdmin = conn.getAdmin()) { htd = admin.getTableDescriptor(tableName); HTableDescriptor peerHtd = null; if (!repHBaseAdmin.tableExists(tableName)) { repHBaseAdmin.createTable(htd, splits); } else { peerHtd = repHBaseAdmin.getTableDescriptor(tableName); if (peerHtd == null) { throw new IllegalArgumentException( "Failed to get table descriptor for table " + tableName.getNameAsString() + " from peer cluster " + repPeer.getId()); } else if (!peerHtd.equals(htd)) { throw new IllegalArgumentException( "Table " + tableName.getNameAsString() + " exists in peer cluster " + repPeer.getId() + ", but the table descriptors are not same when comapred with source cluster." + " Thus can not enable the table's replication switch."); } } } } }
public void convertH() throws SQLException, IOException, ClassNotFoundException, IllegalAccessException, InstantiationException { /** create HBase connection */ conf_h = HBaseConfiguration.create(); conf_h.set("hbase.zookeeper.quorum", "localhost"); conf_h.set("hbase.zookeeper.property.clientPort", "2181"); try { conn_h = ConnectionFactory.createConnection(conf_h); } catch (IOException e) { e.printStackTrace(); } /** create Mysql connection */ try { Class.forName("com.mysql.jdbc.Driver").newInstance(); // con_sql = DriverManager.getConnection("jdbc:mysql:///retail_db", "root", "cloudera"); conn_sql = DriverManager.getConnection("jdbc:mysql:///retail_db", "root", "cloudera"); if (!conn_sql.isClosed()) { System.out.println("Successfully connected to MySQL server..."); } } catch (Exception e) { System.err.println(e.getMessage()); } /** create table and family */ Admin admin = conn_h.getAdmin(); HTableDescriptor tableDesc = new HTableDescriptor(tableName_retail); // create Family Name String[] familyName = {"order_items", "orders", "products"}; for (String name : familyName) { HColumnDescriptor colDesc = new HColumnDescriptor(name); tableDesc.addFamily(colDesc); } admin.createTable(tableDesc); /** query from mysql */ if (conn_sql != null) { Statement stat = conn_sql.createStatement(); stat.executeQuery( "select * from products " + "inner join order_items on product_id = order_item_product_id " + "inner join orders on order_item_order_id = order_id "); ResultSet rs = stat.getResultSet(); while (rs.next()) { // order_items String rowKey = rs.getString("order_item_id"); Put put = new Put(Bytes.toBytes(rowKey)); // new Put(rowKey) put.addColumn( Bytes.toBytes("order_items"), Bytes.toBytes("order_item_id"), Bytes.toBytes(rowKey)); String order_item_order_id = rs.getString("order_item_order_id"); put.addColumn( Bytes.toBytes("order_items"), Bytes.toBytes("order_item_order_id"), Bytes.toBytes(order_item_order_id)); String order_item_product_id = rs.getString("order_item_product_id"); put.addColumn( Bytes.toBytes("order_items"), Bytes.toBytes("order_item_product_id"), Bytes.toBytes(order_item_product_id)); String order_item_quantity = rs.getString("order_item_quantity"); put.addColumn( Bytes.toBytes("order_items"), Bytes.toBytes("order_item_quantity"), Bytes.toBytes(order_item_quantity)); String order_item_subtotal = rs.getString("order_item_subtotal"); put.addColumn( Bytes.toBytes("order_items"), Bytes.toBytes("order_item_subtotal"), Bytes.toBytes(order_item_subtotal)); String order_item_product_price = rs.getString("order_item_product_price"); put.addColumn( Bytes.toBytes("order_items"), Bytes.toBytes("order_item_product_price"), Bytes.toBytes(order_item_product_price)); // orders String order_id = rs.getString("order_id"); put.addColumn(Bytes.toBytes("orders"), Bytes.toBytes("order_id"), Bytes.toBytes(order_id)); String order_date = rs.getString("order_date"); put.addColumn( Bytes.toBytes("orders"), Bytes.toBytes("order_date"), Bytes.toBytes(order_date)); String order_customer_id = rs.getString("order_customer_id"); put.addColumn( Bytes.toBytes("orders"), Bytes.toBytes("order_customer_id"), Bytes.toBytes(order_customer_id)); String order_status = rs.getString("order_status"); put.addColumn( Bytes.toBytes("orders"), Bytes.toBytes("order_status"), Bytes.toBytes(order_status)); // products String product_id = rs.getString("product_id"); put.addColumn( Bytes.toBytes("products"), Bytes.toBytes("product_id"), Bytes.toBytes(product_id)); String product_category_id = rs.getString("product_category_id"); put.addColumn( Bytes.toBytes("products"), Bytes.toBytes("product_category_id"), Bytes.toBytes(product_category_id)); String product_name = rs.getString("product_name"); put.addColumn( Bytes.toBytes("products"), Bytes.toBytes("product_name"), Bytes.toBytes(product_name)); String product_description = rs.getString("product_description"); put.addColumn( Bytes.toBytes("products"), Bytes.toBytes("product_description"), Bytes.toBytes(product_description)); String product_price = rs.getString("product_price"); put.addColumn( Bytes.toBytes("products"), Bytes.toBytes("product_price"), Bytes.toBytes(product_price)); String product_image = rs.getString("product_image"); put.addColumn( Bytes.toBytes("products"), Bytes.toBytes("product_image"), Bytes.toBytes(product_image)); // put in table Table table = conn_h.getTable(tableName_retail); table.put(put); table.close(); } rs.close(); stat.close(); } conn_h.close(); conn_sql.close(); }
public boolean doTest() throws InstantiationException, IllegalAccessException, ClassNotFoundException, SQLException, IOException, InterruptedException { /** create config */ conf_h = HBaseConfiguration.create(); conf_h.set("hbase.zookeeper.quorum", "localhost"); conf_h.set("hbase.zookeeper.property.clientPort", "2181"); Connection con_h = null; try { con_h = ConnectionFactory.createConnection(conf_h); } catch (IOException e) { e.printStackTrace(); } Admin admin = con_h.getAdmin(); HTableDescriptor tableDesc = new HTableDescriptor(tableName_chi); HColumnDescriptor colFamDesc = new HColumnDescriptor("count"); colFamDesc.setMaxVersions(1); tableDesc.addFamily(colFamDesc); admin.createTable(tableDesc); /** counting and insert in chiTable */ Scan scan = new Scan(); scan.addColumn(Bytes.toBytes("products"), Bytes.toBytes("product_category_id")); scan.addColumn(Bytes.toBytes("orders"), Bytes.toBytes("order_date")); // Creates a new Job with no particular Cluster Job job = Job.getInstance(conf_h, "Count"); // Job.getInstance(Configuration conf, String JobName) job.setJarByClass( ChiSquaredTest2_abc.class); // Set the Jar by finding where a given class came from // initTableMapperJob(String table, Scan scan, Class<? extends TableMapper> mapper, Class<?> // outputKeyClass, Class<?> outputValueClass, org.apache.hadoop.mapreduce.Job job) TableMapReduceUtil.initTableMapperJob( "retail_order", scan, Map1.class, Text.class, IntWritable.class, job); // initTableReducerJob(String table, Class<? extends TableReducer> reducer, // org.apache.hadoop.mapreduce.Job job) TableMapReduceUtil.initTableReducerJob("chiTable", Reduce1.class, job); // boolean waitForCompletion(boolean verbose), verbose - print the progress to the user job.waitForCompletion(true); // Submit the job to the cluster and wait for it to finish /** extract value from chiTable */ int totalY = 0; int totalN = 0; ArrayList<CellOfHTable> chiTable = new ArrayList<CellOfHTable>(); Table table_h = con_h.getTable(tableName_chi); Scan s = new Scan(); s.addFamily(Bytes.toBytes("count")); ResultScanner results = table_h.getScanner(s); for (Result r : results) { CellOfHTable c = new CellOfHTable( r.getRow(), r.getValue(Bytes.toBytes("count"), Bytes.toBytes("Y")) == null ? Bytes.toBytes(0) : r.getValue(Bytes.toBytes("count"), Bytes.toBytes("Y")), r.getValue(Bytes.toBytes("count"), Bytes.toBytes("N")) == null ? Bytes.toBytes(0) : r.getValue( Bytes.toBytes("count"), Bytes.toBytes("N"))); // (id, count_Y, count_N) chiTable.add(c); totalY = totalY + c.countY; totalN = totalN + c.countN; } results.close(); table_h.close(); admin.disableTable(tableName_chi); admin.deleteTable(tableName_chi); double chisquare = 0.0; for (int i = 0; i < chiTable.size(); i++) { CellOfHTable c = chiTable.get(i); double expectY = (double) (c.countY + c.countN) * (double) totalY / (double) (totalY + totalN); chisquare = chisquare + (((double) c.countY - expectY) * ((double) c.countY - expectY) / expectY); double expectN = (double) (c.countY + c.countN) * (double) totalN / (double) (totalY + totalN); chisquare = chisquare + (((double) c.countN - expectN) * ((double) c.countN - expectN) / expectN); } System.out.println(chisquare); ChiSquareDist csd = new ChiSquareDist((chiTable.size() - 1)); if (chisquare > csd.inverseF(1.0 - alpha)) { return true; } return false; }
/** * This test is to test the scenario happened in HBASE-6901. All files are bulk loaded and * excluded from minor compaction. Without the fix of HBASE-6901, an * ArrayIndexOutOfBoundsException will be thrown. */ @Ignore("Flakey: See HBASE-9051") @Test public void testExcludeAllFromMinorCompaction() throws Exception { Configuration conf = util.getConfiguration(); conf.setInt("hbase.hstore.compaction.min", 2); generateRandomStartKeys(5); util.startMiniCluster(); try (Connection conn = ConnectionFactory.createConnection(); Admin admin = conn.getAdmin(); Table table = util.createTable(TABLE_NAME, FAMILIES); RegionLocator locator = conn.getRegionLocator(TABLE_NAME)) { final FileSystem fs = util.getDFSCluster().getFileSystem(); assertEquals("Should start with empty table", 0, util.countRows(table)); // deep inspection: get the StoreFile dir final Path storePath = new Path( FSUtils.getTableDir(FSUtils.getRootDir(conf), TABLE_NAME), new Path( admin.getTableRegions(TABLE_NAME).get(0).getEncodedName(), Bytes.toString(FAMILIES[0]))); assertEquals(0, fs.listStatus(storePath).length); // Generate two bulk load files conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude", true); for (int i = 0; i < 2; i++) { Path testDir = util.getDataTestDirOnTestFS("testExcludeAllFromMinorCompaction_" + i); runIncrementalPELoad( conf, table.getTableDescriptor(), conn.getRegionLocator(TABLE_NAME), testDir); // Perform the actual load new LoadIncrementalHFiles(conf).doBulkLoad(testDir, admin, table, locator); } // Ensure data shows up int expectedRows = 2 * NMapInputFormat.getNumMapTasks(conf) * ROWSPERSPLIT; assertEquals( "LoadIncrementalHFiles should put expected data in table", expectedRows, util.countRows(table)); // should have a second StoreFile now assertEquals(2, fs.listStatus(storePath).length); // minor compactions shouldn't get rid of the file admin.compact(TABLE_NAME); try { quickPoll( new Callable<Boolean>() { @Override public Boolean call() throws Exception { return fs.listStatus(storePath).length == 1; } }, 5000); throw new IOException("SF# = " + fs.listStatus(storePath).length); } catch (AssertionError ae) { // this is expected behavior } // a major compaction should work though admin.majorCompact(TABLE_NAME); quickPoll( new Callable<Boolean>() { @Override public Boolean call() throws Exception { return fs.listStatus(storePath).length == 1; } }, 5000); } finally { util.shutdownMiniCluster(); } }
public RSGroupAdminClient(Connection conn) throws IOException { proxy = RSGroupAdminProtos.RSGroupAdminService.newBlockingStub( conn.getAdmin().coprocessorService()); }
public static void main(String[] args) throws IOException { Logger logger = new Logger(CheckConfig.class); GenericOptionsParser optionsParser = new GenericOptionsParser(HBaseConfiguration.create(), args); Configuration fullConfiguration = optionsParser.getConfiguration(); BigtableOptions options; try { options = BigtableOptionsFactory.fromConfiguration(fullConfiguration); } catch (IOException | RuntimeException exc) { logger.warn("Encountered errors attempting to parse configuration.", exc); return; } System.out.println(String.format("User Agent: %s", options.getChannelOptions().getUserAgent())); System.out.println(String.format("Project ID: %s", options.getProjectId())); System.out.println(String.format("Cluster Name: %s", options.getCluster())); System.out.println(String.format("Zone: %s", options.getZone())); System.out.println(String.format("Cluster admin host: %s", options.getClusterAdminHost())); System.out.println(String.format("Table admin host: %s", options.getTableAdminHost())); System.out.println(String.format("Data host: %s", options.getDataHost())); Credentials credentials = options.getChannelOptions().getCredential(); try { System.out.println("Attempting credential refresh..."); credentials.refresh(); } catch (IOException ioe) { logger.warn("Encountered errors attempting to refresh credentials.", ioe); return; } String configuredConnectionClass = fullConfiguration.get(HConnection.HBASE_CLIENT_CONNECTION_IMPL); boolean isCorrectClassSpecified = false; if (!Strings.isNullOrEmpty(configuredConnectionClass)) { try { Class<?> connectionClass = Class.forName(configuredConnectionClass); isCorrectClassSpecified = AbstractBigtableConnection.class.isAssignableFrom(connectionClass); } catch (Exception e) { // Ignore. Problems will be logged in the println below. } } // We can actually determine if this value is correct (disregarding custom subclasses). System.out.println( String.format( "HBase Connection Class = %s %s", configuredConnectionClass, isCorrectClassSpecified ? "(OK)" : "(Configuration error)")); System.out.println("Opening table admin connection..."); try (Connection conn = ConnectionFactory.createConnection(fullConfiguration)) { try (Admin admin = conn.getAdmin()) { System.out.println(String.format("Tables in cluster %s:", options.getCluster())); TableName[] tableNames = admin.listTableNames(); if (tableNames.length == 0) { System.out.println("No tables found."); } else { for (TableName table : tableNames) { System.out.println(table.getNameAsString()); } } } System.out.println("Closing connection..."); } }
@BeforeClass public static void setUpBeforeClass() throws Exception { conf1.setInt("hfile.format.version", 3); conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); conf1.setInt("replication.source.size.capacity", 10240); conf1.setLong("replication.source.sleepforretries", 100); conf1.setInt("hbase.regionserver.maxlogs", 10); conf1.setLong("hbase.master.logcleaner.ttl", 10); conf1.setInt("zookeeper.recovery.retry", 1); conf1.setInt("zookeeper.recovery.retry.intervalmill", 10); conf1.setBoolean("dfs.support.append", true); conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); conf1.setInt("replication.stats.thread.period.seconds", 5); conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false); conf1.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName()); conf1.setStrings( CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, TestCoprocessorForTagsAtSource.class.getName()); utility1 = new HBaseTestingUtility(conf1); utility1.startMiniZKCluster(); MiniZooKeeperCluster miniZK = utility1.getZkCluster(); // Have to reget conf1 in case zk cluster location different // than default conf1 = utility1.getConfiguration(); replicationAdmin = new ReplicationAdmin(conf1); LOG.info("Setup first Zk"); // Base conf2 on conf1 so it gets the right zk cluster. conf2 = HBaseConfiguration.create(conf1); conf2.setInt("hfile.format.version", 3); conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); conf2.setBoolean("dfs.support.append", true); conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false); conf2.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName()); conf2.setStrings( CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, TestCoprocessorForTagsAtSink.class.getName()); utility2 = new HBaseTestingUtility(conf2); utility2.setZkCluster(miniZK); replicationAdmin.addPeer("2", utility2.getClusterKey()); LOG.info("Setup second Zk"); utility1.startMiniCluster(2); utility2.startMiniCluster(2); HTableDescriptor table = new HTableDescriptor(TABLE_NAME); HColumnDescriptor fam = new HColumnDescriptor(FAMILY); fam.setMaxVersions(3); fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); table.addFamily(fam); try (Connection conn = ConnectionFactory.createConnection(conf1); Admin admin = conn.getAdmin()) { admin.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); } try (Connection conn = ConnectionFactory.createConnection(conf2); Admin admin = conn.getAdmin()) { admin.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); } htable1 = utility1.getConnection().getTable(TABLE_NAME); htable2 = utility2.getConnection().getTable(TABLE_NAME); }
public void printStats() throws IOException { Admin admin = connection.getAdmin(); ClusterStatus status = admin.getClusterStatus(); // co ClusterStatusExample-1-GetStatus Get the cluster status. System.out.println("Cluster Status:\n--------------"); System.out.println("HBase Version: " + status.getHBaseVersion()); System.out.println("Version: " + status.getVersion()); System.out.println("Cluster ID: " + status.getClusterId()); System.out.println("Master: " + status.getMaster()); System.out.println("No. Backup Masters: " + status.getBackupMastersSize()); System.out.println("Backup Masters: " + status.getBackupMasters()); System.out.println("No. Live Servers: " + status.getServersSize()); System.out.println("Servers: " + status.getServers()); System.out.println("No. Dead Servers: " + status.getDeadServers()); System.out.println("Dead Servers: " + status.getDeadServerNames()); System.out.println("No. Regions: " + status.getRegionsCount()); System.out.println("Regions in Transition: " + status.getRegionsInTransition()); System.out.println("No. Requests: " + status.getRequestsCount()); System.out.println("Avg Load: " + status.getAverageLoad()); System.out.println("Balancer On: " + status.getBalancerOn()); System.out.println("Is Balancer On: " + status.isBalancerOn()); System.out.println("Master Coprocessors: " + Arrays.asList(status.getMasterCoprocessors())); System.out.println("\nServer Info:\n--------------"); for (ServerName server : status .getServers()) { // co ClusterStatusExample-2-ServerInfo Iterate over the included // server instances. System.out.println("Hostname: " + server.getHostname()); System.out.println("Host and Port: " + server.getHostAndPort()); System.out.println("Server Name: " + server.getServerName()); System.out.println("RPC Port: " + server.getPort()); System.out.println("Start Code: " + server.getStartcode()); ServerLoad load = status.getLoad( server); // co ClusterStatusExample-3-ServerLoad Retrieve the load details for the // current server. System.out.println("\nServer Load:\n--------------"); System.out.println("Info Port: " + load.getInfoServerPort()); System.out.println("Load: " + load.getLoad()); System.out.println("Max Heap (MB): " + load.getMaxHeapMB()); System.out.println("Used Heap (MB): " + load.getUsedHeapMB()); System.out.println("Memstore Size (MB): " + load.getMemstoreSizeInMB()); System.out.println("No. Regions: " + load.getNumberOfRegions()); System.out.println("No. Requests: " + load.getNumberOfRequests()); System.out.println("Total No. Requests: " + load.getTotalNumberOfRequests()); System.out.println("No. Requests per Sec: " + load.getRequestsPerSecond()); System.out.println("No. Read Requests: " + load.getReadRequestsCount()); System.out.println("No. Write Requests: " + load.getWriteRequestsCount()); System.out.println("No. Stores: " + load.getStores()); System.out.println("Store Size Uncompressed (MB): " + load.getStoreUncompressedSizeMB()); System.out.println("No. Storefiles: " + load.getStorefiles()); System.out.println("Storefile Size (MB): " + load.getStorefileSizeInMB()); System.out.println("Storefile Index Size (MB): " + load.getStorefileIndexSizeInMB()); System.out.println("Root Index Size: " + load.getRootIndexSizeKB()); System.out.println("Total Bloom Size: " + load.getTotalStaticBloomSizeKB()); System.out.println("Total Index Size: " + load.getTotalStaticIndexSizeKB()); System.out.println("Current Compacted Cells: " + load.getCurrentCompactedKVs()); System.out.println("Total Compacting Cells: " + load.getTotalCompactingKVs()); System.out.println("Coprocessors1: " + Arrays.asList(load.getRegionServerCoprocessors())); System.out.println("Coprocessors2: " + Arrays.asList(load.getRsCoprocessors())); System.out.println("Replication Load Sink: " + load.getReplicationLoadSink()); System.out.println("Replication Load Source: " + load.getReplicationLoadSourceList()); System.out.println("\nRegion Load:\n--------------"); for (Map.Entry<byte[], RegionLoad> entry : // co ClusterStatusExample-4-Regions Iterate over the region details of the // current server. load.getRegionsLoad().entrySet()) { System.out.println("Region: " + Bytes.toStringBinary(entry.getKey())); RegionLoad regionLoad = entry .getValue(); // co ClusterStatusExample-5-RegionLoad Get the load details for the // current region. System.out.println("Name: " + Bytes.toStringBinary(regionLoad.getName())); System.out.println("Name (as String): " + regionLoad.getNameAsString()); System.out.println("No. Requests: " + regionLoad.getRequestsCount()); System.out.println("No. Read Requests: " + regionLoad.getReadRequestsCount()); System.out.println("No. Write Requests: " + regionLoad.getWriteRequestsCount()); System.out.println("No. Stores: " + regionLoad.getStores()); System.out.println("No. Storefiles: " + regionLoad.getStorefiles()); System.out.println("Data Locality: " + regionLoad.getDataLocality()); System.out.println("Storefile Size (MB): " + regionLoad.getStorefileSizeMB()); System.out.println("Storefile Index Size (MB): " + regionLoad.getStorefileIndexSizeMB()); System.out.println("Memstore Size (MB): " + regionLoad.getMemStoreSizeMB()); System.out.println("Root Index Size: " + regionLoad.getRootIndexSizeKB()); System.out.println("Total Bloom Size: " + regionLoad.getTotalStaticBloomSizeKB()); System.out.println("Total Index Size: " + regionLoad.getTotalStaticIndexSizeKB()); System.out.println("Current Compacted Cells: " + regionLoad.getCurrentCompactedKVs()); System.out.println("Total Compacting Cells: " + regionLoad.getTotalCompactingKVs()); System.out.println(); } } }