public Map<String, Long> getRegionSizes(String tableName) {
Map<String, Long> regions = new HashMap<>();
try {
final Table table = connection.getTable(TableName.valueOf(tableName));
RegionLocator regionLocator = connection.getRegionLocator(table.getName());
List<HRegionLocation> tableRegionInfos = regionLocator.getAllRegionLocations();
Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
for (HRegionLocation regionInfo : tableRegionInfos) {
tableRegions.add(regionInfo.getRegionInfo().getRegionName());
}
ClusterStatus clusterStatus = connection.getAdmin().getClusterStatus();
Collection<ServerName> servers = clusterStatus.getServers();
final long megaByte = 1024L * 1024L;
for (ServerName serverName : servers) {
ServerLoad serverLoad = clusterStatus.getLoad(serverName);
for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) {
byte[] regionId = regionLoad.getName();
if (tableRegions.contains(regionId)) {
long regionSizeBytes = regionLoad.getStorefileSizeMB() * megaByte;
regions.put(regionLoad.getNameAsString(), regionSizeBytes);
}
}
}
} catch (IOException e) {
e.printStackTrace();
}
return regions;
}
private static void createTable() throws Exception {
try {
Configuration configuration = HBaseConfiguration.create();
HBaseAdmin.checkHBaseAvailable(configuration);
Connection connection = ConnectionFactory.createConnection(configuration);
// Instantiating HbaseAdmin class
Admin admin = connection.getAdmin();
// Instantiating table descriptor class
HTableDescriptor stockTableDesc =
new HTableDescriptor(TableName.valueOf(Constants.STOCK_DATES_TABLE));
// Adding column families to table descriptor
HColumnDescriptor stock_0414 = new HColumnDescriptor(Constants.STOCK_DATES_CF);
stockTableDesc.addFamily(stock_0414);
// Execute the table through admin
if (!admin.tableExists(stockTableDesc.getTableName())) {
admin.createTable(stockTableDesc);
System.out.println("Stock table created !!!");
}
// Load hbase-site.xml
HBaseConfiguration.addHbaseResources(configuration);
} catch (ServiceException e) {
log.error("Error occurred while creating HBase tables", e);
throw new Exception("Error occurred while creating HBase tables", e);
}
}
public boolean tableExists(String tableName) {
try {
Admin admin = connection.getAdmin();
return admin.tableExists(TableName.valueOf(tableName));
} catch (IOException e) {
e.printStackTrace();
}
return false;
}
public void createTable(String tableName, List<String> columnFamilies) {
try {
Admin admin = connection.getAdmin();
HTableDescriptor descriptor = new HTableDescriptor(TableName.valueOf(tableName));
for (String family : columnFamilies) {
descriptor.addFamily(new HColumnDescriptor(family));
}
admin.createTable(descriptor);
} catch (IOException e) {
e.printStackTrace();
}
}
public boolean removeTable(String tableName) {
try {
Admin admin = connection.getAdmin();
TableName t = TableName.valueOf(tableName);
if (admin.tableExists(t)) {
admin.disableTable(t);
admin.deleteTable(t);
return true;
}
} catch (IOException e) {
e.printStackTrace();
}
return false;
}
protected void verifyNamespaces() throws IOException {
Connection connection = getConnection();
Admin admin = connection.getAdmin();
// iterating concurrent map
for (String nsName : namespaceMap.keySet()) {
try {
Assert.assertTrue(
"Namespace: " + nsName + " in namespaceMap does not exist",
admin.getNamespaceDescriptor(nsName) != null);
} catch (NamespaceNotFoundException nsnfe) {
Assert.fail(
"Namespace: " + nsName + " in namespaceMap does not exist: " + nsnfe.getMessage());
}
}
admin.close();
}
protected void verifyTables() throws IOException {
Connection connection = getConnection();
Admin admin = connection.getAdmin();
// iterating concurrent map
for (TableName tableName : enabledTables.keySet()) {
Assert.assertTrue(
"Table: " + tableName + " in enabledTables is not enabled",
admin.isTableEnabled(tableName));
}
for (TableName tableName : disabledTables.keySet()) {
Assert.assertTrue(
"Table: " + tableName + " in disabledTables is not disabled",
admin.isTableDisabled(tableName));
}
for (TableName tableName : deletedTables.keySet()) {
Assert.assertFalse(
"Table: " + tableName + " in deletedTables is not deleted", admin.tableExists(tableName));
}
admin.close();
}
public void printStats() throws IOException {
Admin admin = connection.getAdmin();
ClusterStatus status =
admin.getClusterStatus(); // co ClusterStatusExample-1-GetStatus Get the cluster status.
System.out.println("Cluster Status:\n--------------");
System.out.println("HBase Version: " + status.getHBaseVersion());
System.out.println("Version: " + status.getVersion());
System.out.println("Cluster ID: " + status.getClusterId());
System.out.println("Master: " + status.getMaster());
System.out.println("No. Backup Masters: " + status.getBackupMastersSize());
System.out.println("Backup Masters: " + status.getBackupMasters());
System.out.println("No. Live Servers: " + status.getServersSize());
System.out.println("Servers: " + status.getServers());
System.out.println("No. Dead Servers: " + status.getDeadServers());
System.out.println("Dead Servers: " + status.getDeadServerNames());
System.out.println("No. Regions: " + status.getRegionsCount());
System.out.println("Regions in Transition: " + status.getRegionsInTransition());
System.out.println("No. Requests: " + status.getRequestsCount());
System.out.println("Avg Load: " + status.getAverageLoad());
System.out.println("Balancer On: " + status.getBalancerOn());
System.out.println("Is Balancer On: " + status.isBalancerOn());
System.out.println("Master Coprocessors: " + Arrays.asList(status.getMasterCoprocessors()));
System.out.println("\nServer Info:\n--------------");
for (ServerName server :
status
.getServers()) { // co ClusterStatusExample-2-ServerInfo Iterate over the included
// server instances.
System.out.println("Hostname: " + server.getHostname());
System.out.println("Host and Port: " + server.getHostAndPort());
System.out.println("Server Name: " + server.getServerName());
System.out.println("RPC Port: " + server.getPort());
System.out.println("Start Code: " + server.getStartcode());
ServerLoad load =
status.getLoad(
server); // co ClusterStatusExample-3-ServerLoad Retrieve the load details for the
// current server.
System.out.println("\nServer Load:\n--------------");
System.out.println("Info Port: " + load.getInfoServerPort());
System.out.println("Load: " + load.getLoad());
System.out.println("Max Heap (MB): " + load.getMaxHeapMB());
System.out.println("Used Heap (MB): " + load.getUsedHeapMB());
System.out.println("Memstore Size (MB): " + load.getMemstoreSizeInMB());
System.out.println("No. Regions: " + load.getNumberOfRegions());
System.out.println("No. Requests: " + load.getNumberOfRequests());
System.out.println("Total No. Requests: " + load.getTotalNumberOfRequests());
System.out.println("No. Requests per Sec: " + load.getRequestsPerSecond());
System.out.println("No. Read Requests: " + load.getReadRequestsCount());
System.out.println("No. Write Requests: " + load.getWriteRequestsCount());
System.out.println("No. Stores: " + load.getStores());
System.out.println("Store Size Uncompressed (MB): " + load.getStoreUncompressedSizeMB());
System.out.println("No. Storefiles: " + load.getStorefiles());
System.out.println("Storefile Size (MB): " + load.getStorefileSizeInMB());
System.out.println("Storefile Index Size (MB): " + load.getStorefileIndexSizeInMB());
System.out.println("Root Index Size: " + load.getRootIndexSizeKB());
System.out.println("Total Bloom Size: " + load.getTotalStaticBloomSizeKB());
System.out.println("Total Index Size: " + load.getTotalStaticIndexSizeKB());
System.out.println("Current Compacted Cells: " + load.getCurrentCompactedKVs());
System.out.println("Total Compacting Cells: " + load.getTotalCompactingKVs());
System.out.println("Coprocessors1: " + Arrays.asList(load.getRegionServerCoprocessors()));
System.out.println("Coprocessors2: " + Arrays.asList(load.getRsCoprocessors()));
System.out.println("Replication Load Sink: " + load.getReplicationLoadSink());
System.out.println("Replication Load Source: " + load.getReplicationLoadSourceList());
System.out.println("\nRegion Load:\n--------------");
for (Map.Entry<byte[], RegionLoad>
entry : // co ClusterStatusExample-4-Regions Iterate over the region details of the
// current server.
load.getRegionsLoad().entrySet()) {
System.out.println("Region: " + Bytes.toStringBinary(entry.getKey()));
RegionLoad regionLoad =
entry
.getValue(); // co ClusterStatusExample-5-RegionLoad Get the load details for the
// current region.
System.out.println("Name: " + Bytes.toStringBinary(regionLoad.getName()));
System.out.println("Name (as String): " + regionLoad.getNameAsString());
System.out.println("No. Requests: " + regionLoad.getRequestsCount());
System.out.println("No. Read Requests: " + regionLoad.getReadRequestsCount());
System.out.println("No. Write Requests: " + regionLoad.getWriteRequestsCount());
System.out.println("No. Stores: " + regionLoad.getStores());
System.out.println("No. Storefiles: " + regionLoad.getStorefiles());
System.out.println("Data Locality: " + regionLoad.getDataLocality());
System.out.println("Storefile Size (MB): " + regionLoad.getStorefileSizeMB());
System.out.println("Storefile Index Size (MB): " + regionLoad.getStorefileIndexSizeMB());
System.out.println("Memstore Size (MB): " + regionLoad.getMemStoreSizeMB());
System.out.println("Root Index Size: " + regionLoad.getRootIndexSizeKB());
System.out.println("Total Bloom Size: " + regionLoad.getTotalStaticBloomSizeKB());
System.out.println("Total Index Size: " + regionLoad.getTotalStaticIndexSizeKB());
System.out.println("Current Compacted Cells: " + regionLoad.getCurrentCompactedKVs());
System.out.println("Total Compacting Cells: " + regionLoad.getTotalCompactingKVs());
System.out.println();
}
}
}
public static List<KeyValue> getAllTrainInfo(Configuration config, String date) {
List<KeyValue> result = new ArrayList<>();
String strJson = null;
BufferedWriter writer = null;
Table table = null;
try (Connection connect = ConnectionFactory.createConnection(config);
Admin admin = connect.getAdmin()) {
TableName tablename = TableName.valueOf(TABLE_NAME);
if (!admin.tableExists(tablename)) {
System.out.println("Table does not exist.");
return null;
}
table = connect.getTable(tablename);
Put put = null;
String start = null;
String end = null;
writer = new BufferedWriter(new FileWriter(new File(strConfig), true));
for (KeyValue item : lstAllProcessStation) {
start = (String) item.getKey();
end = (String) item.getValue();
try {
try {
Thread.sleep(200);
} catch (InterruptedException e1) {
e1.printStackTrace();
}
System.out.println("process : " + start + ":" + end);
strJson = getFromAPIX(mapStationCode.get(start), mapStationCode.get(end), date);
writer.write(start + ":" + end);
writer.newLine();
} catch (Exception e) {
System.out.println(start + ":" + end + "error");
e.printStackTrace();
break;
}
JSONObject jo = new JSONObject(strJson);
if (jo.has("httpstatus") && (jo.getInt("httpstatus") == 200)) {
JSONObject joData = jo.getJSONObject("data");
if (joData.has("flag") && joData.getBoolean("flag")) {
result.add(new DefaultKeyValue(start, end));
// 插入到hbase
String rowkey = start + ":" + end;
put = new Put(rowkey.getBytes());
put.addColumn(
CF_JSON.getBytes(), "json".getBytes(), joData.toString().getBytes("utf-8"));
table.put(put);
System.out.println("start " + start + "\t end " + end + "\t has ticket");
}
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (writer != null) {
try {
writer.flush();
writer.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (table != null) {
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return result;
}
