public int run(String[] args) throws Exception { Configuration argConf = getConf(); // JobConf conf = new JobConf(diffdb.class); Configuration config = HBaseConfiguration.create(); HBaseAdmin hbAdmin = new HBaseAdmin(config); dbutil db_util = new dbutil(config); HTable runTable = new HTable(config, "gestore_runs"); Get runGet = new Get(argConf.get("id").getBytes()); Result pipeline = runTable.get(runGet); NavigableMap<byte[], byte[]> pipeMap = pipeline.getFamilyMap("d".getBytes()); Map.Entry<byte[], byte[]> results = pipeMap.pollFirstEntry(); HashMap<String, HashMap<String, String>> resultMap = new HashMap<String, HashMap<String, String>>(); while (results != null) { String resultKey = new String(results.getKey()); String resultValue = new String(results.getValue()); String field = "type"; HashMap<String, String> tempMap = new HashMap<String, String>(); String entry = resultKey; if (resultKey.endsWith("_db_timestamp")) { field = "db_timestamp"; entry = resultKey.substring(0, resultKey.lastIndexOf("_db_timestamp")); } else if (resultKey.endsWith("_filename")) { field = "filename"; entry = resultKey.substring(0, resultKey.lastIndexOf("_filename")); } else if (resultKey.endsWith("_regex")) { field = "regex"; entry = resultKey.substring(0, resultKey.lastIndexOf("_regex")); } if (resultMap.containsKey(entry)) { tempMap = resultMap.get(entry); } tempMap.put(field, resultValue); resultMap.put(entry, tempMap); // System.out.println("Key: " + resultKey + " Value: " + resultValue); results = pipeMap.pollFirstEntry(); } for (String key : resultMap.keySet()) { System.out.println("File ID: " + key); for (String subKey : resultMap.get(key).keySet()) { // System.out.println("\t " + subKey + "\t\t" + resultMap.get(key).get(subKey)); System.out.format(" %1$-20s %2$s\n", subKey, resultMap.get(key).get(subKey)); } } return 0; }
private static String getConfigString(Configuration config) { String output = ""; Iterator<Map.Entry<String, String>> iterConfig = config.iterator(); while (iterConfig.hasNext()) { Map.Entry<String, String> curEntry = iterConfig.next(); output = output + "Key: \t" + curEntry.getKey() + "\nValue: \t" + curEntry.getValue() + "\n"; } return output; }
protected HServerLoad.RegionLoad getRegionLoad(HRegionInfo regionInfo, ServerName serverName) throws IOException { HServerLoad serverLoad = admin.getClusterStatus().getLoad(serverName); Map<byte[], HServerLoad.RegionLoad> regionsLoad = serverLoad.getRegionsLoad(); for (Map.Entry<byte[], HServerLoad.RegionLoad> entry : regionsLoad.entrySet()) { if (Arrays.equals(entry.getKey(), regionInfo.getRegionName())) { return entry.getValue(); } } return null; }
protected void waitForMoving(HRegionInfo hRegionInfo, ServerName serverName) throws Exception { Map<byte[], HServerLoad.RegionLoad> regionsLoad = null; for (int i = 0; i < MAX_WAIT_ITERATION; i++) { HServerLoad load = admin.getClusterStatus().getLoad(serverName); regionsLoad = load.getRegionsLoad(); for (byte[] regionName : regionsLoad.keySet()) { if (Arrays.equals(regionName, hRegionInfo.getRegionName())) return; } admin.move(hRegionInfo.getEncodedNameAsBytes(), serverName.getServerName().getBytes()); Thread.sleep(WAIT_INTERVAL); } System.out.println("hRegionInfo = " + Bytes.toString(hRegionInfo.getRegionName())); for (Map.Entry<byte[], HServerLoad.RegionLoad> entry : regionsLoad.entrySet()) { System.out.println( "regionsLoad = " + Bytes.toString(entry.getKey()) + " - " + entry.getValue()); } Assert.fail(Util.getMethodName() + " failed"); }
protected void waitForSplitting(String tableName, int regionCount) throws IOException, InterruptedException { int regionCountActual = 0; for (int i = 0; i < MAX_WAIT_ITERATION; i++) { try (HTable table = new HTable(conf, tableName)) { regionCountActual = 0; NavigableMap<HRegionInfo, ServerName> regionLocations = table.getRegionLocations(); for (Map.Entry<HRegionInfo, ServerName> entry : regionLocations.entrySet()) { HServerLoad serverLoad = admin.getClusterStatus().getLoad(entry.getValue()); for (HServerLoad.RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) { if (Arrays.equals(entry.getKey().getRegionName(), regionLoad.getName())) regionCountActual++; } } if (regionCountActual == regionCount) { return; } } Thread.sleep(WAIT_INTERVAL); } Assert.assertEquals(getMethodName() + " failed - ", regionCount, regionCountActual); }
public void printStats() throws IOException { Admin admin = connection.getAdmin(); ClusterStatus status = admin.getClusterStatus(); // co ClusterStatusExample-1-GetStatus Get the cluster status. System.out.println("Cluster Status:\n--------------"); System.out.println("HBase Version: " + status.getHBaseVersion()); System.out.println("Version: " + status.getVersion()); System.out.println("Cluster ID: " + status.getClusterId()); System.out.println("Master: " + status.getMaster()); System.out.println("No. Backup Masters: " + status.getBackupMastersSize()); System.out.println("Backup Masters: " + status.getBackupMasters()); System.out.println("No. Live Servers: " + status.getServersSize()); System.out.println("Servers: " + status.getServers()); System.out.println("No. Dead Servers: " + status.getDeadServers()); System.out.println("Dead Servers: " + status.getDeadServerNames()); System.out.println("No. Regions: " + status.getRegionsCount()); System.out.println("Regions in Transition: " + status.getRegionsInTransition()); System.out.println("No. Requests: " + status.getRequestsCount()); System.out.println("Avg Load: " + status.getAverageLoad()); System.out.println("Balancer On: " + status.getBalancerOn()); System.out.println("Is Balancer On: " + status.isBalancerOn()); System.out.println("Master Coprocessors: " + Arrays.asList(status.getMasterCoprocessors())); System.out.println("\nServer Info:\n--------------"); for (ServerName server : status .getServers()) { // co ClusterStatusExample-2-ServerInfo Iterate over the included // server instances. System.out.println("Hostname: " + server.getHostname()); System.out.println("Host and Port: " + server.getHostAndPort()); System.out.println("Server Name: " + server.getServerName()); System.out.println("RPC Port: " + server.getPort()); System.out.println("Start Code: " + server.getStartcode()); ServerLoad load = status.getLoad( server); // co ClusterStatusExample-3-ServerLoad Retrieve the load details for the // current server. System.out.println("\nServer Load:\n--------------"); System.out.println("Info Port: " + load.getInfoServerPort()); System.out.println("Load: " + load.getLoad()); System.out.println("Max Heap (MB): " + load.getMaxHeapMB()); System.out.println("Used Heap (MB): " + load.getUsedHeapMB()); System.out.println("Memstore Size (MB): " + load.getMemstoreSizeInMB()); System.out.println("No. Regions: " + load.getNumberOfRegions()); System.out.println("No. Requests: " + load.getNumberOfRequests()); System.out.println("Total No. Requests: " + load.getTotalNumberOfRequests()); System.out.println("No. Requests per Sec: " + load.getRequestsPerSecond()); System.out.println("No. Read Requests: " + load.getReadRequestsCount()); System.out.println("No. Write Requests: " + load.getWriteRequestsCount()); System.out.println("No. Stores: " + load.getStores()); System.out.println("Store Size Uncompressed (MB): " + load.getStoreUncompressedSizeMB()); System.out.println("No. Storefiles: " + load.getStorefiles()); System.out.println("Storefile Size (MB): " + load.getStorefileSizeInMB()); System.out.println("Storefile Index Size (MB): " + load.getStorefileIndexSizeInMB()); System.out.println("Root Index Size: " + load.getRootIndexSizeKB()); System.out.println("Total Bloom Size: " + load.getTotalStaticBloomSizeKB()); System.out.println("Total Index Size: " + load.getTotalStaticIndexSizeKB()); System.out.println("Current Compacted Cells: " + load.getCurrentCompactedKVs()); System.out.println("Total Compacting Cells: " + load.getTotalCompactingKVs()); System.out.println("Coprocessors1: " + Arrays.asList(load.getRegionServerCoprocessors())); System.out.println("Coprocessors2: " + Arrays.asList(load.getRsCoprocessors())); System.out.println("Replication Load Sink: " + load.getReplicationLoadSink()); System.out.println("Replication Load Source: " + load.getReplicationLoadSourceList()); System.out.println("\nRegion Load:\n--------------"); for (Map.Entry<byte[], RegionLoad> entry : // co ClusterStatusExample-4-Regions Iterate over the region details of the // current server. load.getRegionsLoad().entrySet()) { System.out.println("Region: " + Bytes.toStringBinary(entry.getKey())); RegionLoad regionLoad = entry .getValue(); // co ClusterStatusExample-5-RegionLoad Get the load details for the // current region. System.out.println("Name: " + Bytes.toStringBinary(regionLoad.getName())); System.out.println("Name (as String): " + regionLoad.getNameAsString()); System.out.println("No. Requests: " + regionLoad.getRequestsCount()); System.out.println("No. Read Requests: " + regionLoad.getReadRequestsCount()); System.out.println("No. Write Requests: " + regionLoad.getWriteRequestsCount()); System.out.println("No. Stores: " + regionLoad.getStores()); System.out.println("No. Storefiles: " + regionLoad.getStorefiles()); System.out.println("Data Locality: " + regionLoad.getDataLocality()); System.out.println("Storefile Size (MB): " + regionLoad.getStorefileSizeMB()); System.out.println("Storefile Index Size (MB): " + regionLoad.getStorefileIndexSizeMB()); System.out.println("Memstore Size (MB): " + regionLoad.getMemStoreSizeMB()); System.out.println("Root Index Size: " + regionLoad.getRootIndexSizeKB()); System.out.println("Total Bloom Size: " + regionLoad.getTotalStaticBloomSizeKB()); System.out.println("Total Index Size: " + regionLoad.getTotalStaticIndexSizeKB()); System.out.println("Current Compacted Cells: " + regionLoad.getCurrentCompactedKVs()); System.out.println("Total Compacting Cells: " + regionLoad.getTotalCompactingKVs()); System.out.println(); } } }