public int run(String[] args) throws Exception {
    Configuration argConf = getConf();

    // JobConf conf = new JobConf(diffdb.class);
    Configuration config = HBaseConfiguration.create();
    HBaseAdmin hbAdmin = new HBaseAdmin(config);
    dbutil db_util = new dbutil(config);

    HTable runTable = new HTable(config, "gestore_runs");
    Get runGet = new Get(argConf.get("id").getBytes());
    Result pipeline = runTable.get(runGet);

    NavigableMap<byte[], byte[]> pipeMap = pipeline.getFamilyMap("d".getBytes());

    Map.Entry<byte[], byte[]> results = pipeMap.pollFirstEntry();

    HashMap<String, HashMap<String, String>> resultMap =
        new HashMap<String, HashMap<String, String>>();

    while (results != null) {
      String resultKey = new String(results.getKey());
      String resultValue = new String(results.getValue());
      String field = "type";
      HashMap<String, String> tempMap = new HashMap<String, String>();
      String entry = resultKey;

      if (resultKey.endsWith("_db_timestamp")) {
        field = "db_timestamp";
        entry = resultKey.substring(0, resultKey.lastIndexOf("_db_timestamp"));
      } else if (resultKey.endsWith("_filename")) {
        field = "filename";
        entry = resultKey.substring(0, resultKey.lastIndexOf("_filename"));
      } else if (resultKey.endsWith("_regex")) {
        field = "regex";
        entry = resultKey.substring(0, resultKey.lastIndexOf("_regex"));
      }

      if (resultMap.containsKey(entry)) {
        tempMap = resultMap.get(entry);
      }

      tempMap.put(field, resultValue);
      resultMap.put(entry, tempMap);

      // System.out.println("Key: " + resultKey + " Value: " + resultValue);
      results = pipeMap.pollFirstEntry();
    }

    for (String key : resultMap.keySet()) {
      System.out.println("File ID: " + key);
      for (String subKey : resultMap.get(key).keySet()) {
        // System.out.println("\t " + subKey + "\t\t" + resultMap.get(key).get(subKey));
        System.out.format("  %1$-20s  %2$s\n", subKey, resultMap.get(key).get(subKey));
      }
    }

    return 0;
  }
Beispiel #2
0
 private static String getConfigString(Configuration config) {
   String output = "";
   Iterator<Map.Entry<String, String>> iterConfig = config.iterator();
   while (iterConfig.hasNext()) {
     Map.Entry<String, String> curEntry = iterConfig.next();
     output = output + "Key: \t" + curEntry.getKey() + "\nValue: \t" + curEntry.getValue() + "\n";
   }
   return output;
 }
Beispiel #3
0
 protected HServerLoad.RegionLoad getRegionLoad(HRegionInfo regionInfo, ServerName serverName)
     throws IOException {
   HServerLoad serverLoad = admin.getClusterStatus().getLoad(serverName);
   Map<byte[], HServerLoad.RegionLoad> regionsLoad = serverLoad.getRegionsLoad();
   for (Map.Entry<byte[], HServerLoad.RegionLoad> entry : regionsLoad.entrySet()) {
     if (Arrays.equals(entry.getKey(), regionInfo.getRegionName())) {
       return entry.getValue();
     }
   }
   return null;
 }
Beispiel #4
0
  protected void waitForMoving(HRegionInfo hRegionInfo, ServerName serverName) throws Exception {
    Map<byte[], HServerLoad.RegionLoad> regionsLoad = null;
    for (int i = 0; i < MAX_WAIT_ITERATION; i++) {
      HServerLoad load = admin.getClusterStatus().getLoad(serverName);
      regionsLoad = load.getRegionsLoad();
      for (byte[] regionName : regionsLoad.keySet()) {
        if (Arrays.equals(regionName, hRegionInfo.getRegionName())) return;
      }
      admin.move(hRegionInfo.getEncodedNameAsBytes(), serverName.getServerName().getBytes());
      Thread.sleep(WAIT_INTERVAL);
    }

    System.out.println("hRegionInfo = " + Bytes.toString(hRegionInfo.getRegionName()));
    for (Map.Entry<byte[], HServerLoad.RegionLoad> entry : regionsLoad.entrySet()) {
      System.out.println(
          "regionsLoad = " + Bytes.toString(entry.getKey()) + " - " + entry.getValue());
    }

    Assert.fail(Util.getMethodName() + " failed");
  }
Beispiel #5
0
  protected void waitForSplitting(String tableName, int regionCount)
      throws IOException, InterruptedException {
    int regionCountActual = 0;
    for (int i = 0; i < MAX_WAIT_ITERATION; i++) {
      try (HTable table = new HTable(conf, tableName)) {
        regionCountActual = 0;
        NavigableMap<HRegionInfo, ServerName> regionLocations = table.getRegionLocations();
        for (Map.Entry<HRegionInfo, ServerName> entry : regionLocations.entrySet()) {
          HServerLoad serverLoad = admin.getClusterStatus().getLoad(entry.getValue());
          for (HServerLoad.RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) {
            if (Arrays.equals(entry.getKey().getRegionName(), regionLoad.getName()))
              regionCountActual++;
          }
        }
        if (regionCountActual == regionCount) {
          return;
        }
      }
      Thread.sleep(WAIT_INTERVAL);
    }

    Assert.assertEquals(getMethodName() + " failed - ", regionCount, regionCountActual);
  }
Beispiel #6
0
  public void printStats() throws IOException {
    Admin admin = connection.getAdmin();

    ClusterStatus status =
        admin.getClusterStatus(); // co ClusterStatusExample-1-GetStatus Get the cluster status.
    System.out.println("Cluster Status:\n--------------");
    System.out.println("HBase Version: " + status.getHBaseVersion());
    System.out.println("Version: " + status.getVersion());
    System.out.println("Cluster ID: " + status.getClusterId());
    System.out.println("Master: " + status.getMaster());
    System.out.println("No. Backup Masters: " + status.getBackupMastersSize());
    System.out.println("Backup Masters: " + status.getBackupMasters());
    System.out.println("No. Live Servers: " + status.getServersSize());
    System.out.println("Servers: " + status.getServers());
    System.out.println("No. Dead Servers: " + status.getDeadServers());
    System.out.println("Dead Servers: " + status.getDeadServerNames());
    System.out.println("No. Regions: " + status.getRegionsCount());
    System.out.println("Regions in Transition: " + status.getRegionsInTransition());
    System.out.println("No. Requests: " + status.getRequestsCount());
    System.out.println("Avg Load: " + status.getAverageLoad());
    System.out.println("Balancer On: " + status.getBalancerOn());
    System.out.println("Is Balancer On: " + status.isBalancerOn());
    System.out.println("Master Coprocessors: " + Arrays.asList(status.getMasterCoprocessors()));
    System.out.println("\nServer Info:\n--------------");
    for (ServerName server :
        status
            .getServers()) { // co ClusterStatusExample-2-ServerInfo Iterate over the included
                             // server instances.
      System.out.println("Hostname: " + server.getHostname());
      System.out.println("Host and Port: " + server.getHostAndPort());
      System.out.println("Server Name: " + server.getServerName());
      System.out.println("RPC Port: " + server.getPort());
      System.out.println("Start Code: " + server.getStartcode());
      ServerLoad load =
          status.getLoad(
              server); // co ClusterStatusExample-3-ServerLoad Retrieve the load details for the
                       // current server.
      System.out.println("\nServer Load:\n--------------");
      System.out.println("Info Port: " + load.getInfoServerPort());
      System.out.println("Load: " + load.getLoad());
      System.out.println("Max Heap (MB): " + load.getMaxHeapMB());
      System.out.println("Used Heap (MB): " + load.getUsedHeapMB());
      System.out.println("Memstore Size (MB): " + load.getMemstoreSizeInMB());
      System.out.println("No. Regions: " + load.getNumberOfRegions());
      System.out.println("No. Requests: " + load.getNumberOfRequests());
      System.out.println("Total No. Requests: " + load.getTotalNumberOfRequests());
      System.out.println("No. Requests per Sec: " + load.getRequestsPerSecond());
      System.out.println("No. Read Requests: " + load.getReadRequestsCount());
      System.out.println("No. Write Requests: " + load.getWriteRequestsCount());
      System.out.println("No. Stores: " + load.getStores());
      System.out.println("Store Size Uncompressed (MB): " + load.getStoreUncompressedSizeMB());
      System.out.println("No. Storefiles: " + load.getStorefiles());
      System.out.println("Storefile Size (MB): " + load.getStorefileSizeInMB());
      System.out.println("Storefile Index Size (MB): " + load.getStorefileIndexSizeInMB());
      System.out.println("Root Index Size: " + load.getRootIndexSizeKB());
      System.out.println("Total Bloom Size: " + load.getTotalStaticBloomSizeKB());
      System.out.println("Total Index Size: " + load.getTotalStaticIndexSizeKB());
      System.out.println("Current Compacted Cells: " + load.getCurrentCompactedKVs());
      System.out.println("Total Compacting Cells: " + load.getTotalCompactingKVs());
      System.out.println("Coprocessors1: " + Arrays.asList(load.getRegionServerCoprocessors()));
      System.out.println("Coprocessors2: " + Arrays.asList(load.getRsCoprocessors()));
      System.out.println("Replication Load Sink: " + load.getReplicationLoadSink());
      System.out.println("Replication Load Source: " + load.getReplicationLoadSourceList());
      System.out.println("\nRegion Load:\n--------------");
      for (Map.Entry<byte[], RegionLoad>
          entry : // co ClusterStatusExample-4-Regions Iterate over the region details of the
                  // current server.
          load.getRegionsLoad().entrySet()) {
        System.out.println("Region: " + Bytes.toStringBinary(entry.getKey()));
        RegionLoad regionLoad =
            entry
                .getValue(); // co ClusterStatusExample-5-RegionLoad Get the load details for the
                             // current region.
        System.out.println("Name: " + Bytes.toStringBinary(regionLoad.getName()));
        System.out.println("Name (as String): " + regionLoad.getNameAsString());
        System.out.println("No. Requests: " + regionLoad.getRequestsCount());
        System.out.println("No. Read Requests: " + regionLoad.getReadRequestsCount());
        System.out.println("No. Write Requests: " + regionLoad.getWriteRequestsCount());
        System.out.println("No. Stores: " + regionLoad.getStores());
        System.out.println("No. Storefiles: " + regionLoad.getStorefiles());
        System.out.println("Data Locality: " + regionLoad.getDataLocality());
        System.out.println("Storefile Size (MB): " + regionLoad.getStorefileSizeMB());
        System.out.println("Storefile Index Size (MB): " + regionLoad.getStorefileIndexSizeMB());
        System.out.println("Memstore Size (MB): " + regionLoad.getMemStoreSizeMB());
        System.out.println("Root Index Size: " + regionLoad.getRootIndexSizeKB());
        System.out.println("Total Bloom Size: " + regionLoad.getTotalStaticBloomSizeKB());
        System.out.println("Total Index Size: " + regionLoad.getTotalStaticIndexSizeKB());
        System.out.println("Current Compacted Cells: " + regionLoad.getCurrentCompactedKVs());
        System.out.println("Total Compacting Cells: " + regionLoad.getTotalCompactingKVs());
        System.out.println();
      }
    }
  }