コード例 #1
0
ファイル: HBaseUtils.java プロジェクト: paul-weiss/Utils
 public Map<String, Long> getRegionSizes(String tableName) {
   Map<String, Long> regions = new HashMap<>();
   try {
     final Table table = connection.getTable(TableName.valueOf(tableName));
     RegionLocator regionLocator = connection.getRegionLocator(table.getName());
     List<HRegionLocation> tableRegionInfos = regionLocator.getAllRegionLocations();
     Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
     for (HRegionLocation regionInfo : tableRegionInfos) {
       tableRegions.add(regionInfo.getRegionInfo().getRegionName());
     }
     ClusterStatus clusterStatus = connection.getAdmin().getClusterStatus();
     Collection<ServerName> servers = clusterStatus.getServers();
     final long megaByte = 1024L * 1024L;
     for (ServerName serverName : servers) {
       ServerLoad serverLoad = clusterStatus.getLoad(serverName);
       for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) {
         byte[] regionId = regionLoad.getName();
         if (tableRegions.contains(regionId)) {
           long regionSizeBytes = regionLoad.getStorefileSizeMB() * megaByte;
           regions.put(regionLoad.getNameAsString(), regionSizeBytes);
         }
       }
     }
   } catch (IOException e) {
     e.printStackTrace();
   }
   return regions;
 }
コード例 #2
0
  private static void createTable() throws Exception {
    try {
      Configuration configuration = HBaseConfiguration.create();
      HBaseAdmin.checkHBaseAvailable(configuration);
      Connection connection = ConnectionFactory.createConnection(configuration);

      // Instantiating HbaseAdmin class
      Admin admin = connection.getAdmin();

      // Instantiating table descriptor class
      HTableDescriptor stockTableDesc =
          new HTableDescriptor(TableName.valueOf(Constants.STOCK_DATES_TABLE));

      // Adding column families to table descriptor
      HColumnDescriptor stock_0414 = new HColumnDescriptor(Constants.STOCK_DATES_CF);
      stockTableDesc.addFamily(stock_0414);

      // Execute the table through admin
      if (!admin.tableExists(stockTableDesc.getTableName())) {
        admin.createTable(stockTableDesc);
        System.out.println("Stock table created !!!");
      }

      // Load hbase-site.xml
      HBaseConfiguration.addHbaseResources(configuration);
    } catch (ServiceException e) {
      log.error("Error occurred while creating HBase tables", e);
      throw new Exception("Error occurred while creating HBase tables", e);
    }
  }
コード例 #3
0
ファイル: HBaseUtils.java プロジェクト: paul-weiss/Utils
 public boolean tableExists(String tableName) {
   try {
     Admin admin = connection.getAdmin();
     return admin.tableExists(TableName.valueOf(tableName));
   } catch (IOException e) {
     e.printStackTrace();
   }
   return false;
 }
コード例 #4
0
ファイル: HBaseUtils.java プロジェクト: paul-weiss/Utils
 public void createTable(String tableName, List<String> columnFamilies) {
   try {
     Admin admin = connection.getAdmin();
     HTableDescriptor descriptor = new HTableDescriptor(TableName.valueOf(tableName));
     for (String family : columnFamilies) {
       descriptor.addFamily(new HColumnDescriptor(family));
     }
     admin.createTable(descriptor);
   } catch (IOException e) {
     e.printStackTrace();
   }
 }
コード例 #5
0
ファイル: HBaseUtils.java プロジェクト: paul-weiss/Utils
 public boolean removeTable(String tableName) {
   try {
     Admin admin = connection.getAdmin();
     TableName t = TableName.valueOf(tableName);
     if (admin.tableExists(t)) {
       admin.disableTable(t);
       admin.deleteTable(t);
       return true;
     }
   } catch (IOException e) {
     e.printStackTrace();
   }
   return false;
 }
コード例 #6
0
 protected void verifyNamespaces() throws IOException {
   Connection connection = getConnection();
   Admin admin = connection.getAdmin();
   // iterating concurrent map
   for (String nsName : namespaceMap.keySet()) {
     try {
       Assert.assertTrue(
           "Namespace: " + nsName + " in namespaceMap does not exist",
           admin.getNamespaceDescriptor(nsName) != null);
     } catch (NamespaceNotFoundException nsnfe) {
       Assert.fail(
           "Namespace: " + nsName + " in namespaceMap does not exist: " + nsnfe.getMessage());
     }
   }
   admin.close();
 }
コード例 #7
0
 protected void verifyTables() throws IOException {
   Connection connection = getConnection();
   Admin admin = connection.getAdmin();
   // iterating concurrent map
   for (TableName tableName : enabledTables.keySet()) {
     Assert.assertTrue(
         "Table: " + tableName + " in enabledTables is not enabled",
         admin.isTableEnabled(tableName));
   }
   for (TableName tableName : disabledTables.keySet()) {
     Assert.assertTrue(
         "Table: " + tableName + " in disabledTables is not disabled",
         admin.isTableDisabled(tableName));
   }
   for (TableName tableName : deletedTables.keySet()) {
     Assert.assertFalse(
         "Table: " + tableName + " in deletedTables is not deleted", admin.tableExists(tableName));
   }
   admin.close();
 }
コード例 #8
0
ファイル: HBaseUtils.java プロジェクト: paul-weiss/Utils
  public void printStats() throws IOException {
    Admin admin = connection.getAdmin();

    ClusterStatus status =
        admin.getClusterStatus(); // co ClusterStatusExample-1-GetStatus Get the cluster status.
    System.out.println("Cluster Status:\n--------------");
    System.out.println("HBase Version: " + status.getHBaseVersion());
    System.out.println("Version: " + status.getVersion());
    System.out.println("Cluster ID: " + status.getClusterId());
    System.out.println("Master: " + status.getMaster());
    System.out.println("No. Backup Masters: " + status.getBackupMastersSize());
    System.out.println("Backup Masters: " + status.getBackupMasters());
    System.out.println("No. Live Servers: " + status.getServersSize());
    System.out.println("Servers: " + status.getServers());
    System.out.println("No. Dead Servers: " + status.getDeadServers());
    System.out.println("Dead Servers: " + status.getDeadServerNames());
    System.out.println("No. Regions: " + status.getRegionsCount());
    System.out.println("Regions in Transition: " + status.getRegionsInTransition());
    System.out.println("No. Requests: " + status.getRequestsCount());
    System.out.println("Avg Load: " + status.getAverageLoad());
    System.out.println("Balancer On: " + status.getBalancerOn());
    System.out.println("Is Balancer On: " + status.isBalancerOn());
    System.out.println("Master Coprocessors: " + Arrays.asList(status.getMasterCoprocessors()));
    System.out.println("\nServer Info:\n--------------");
    for (ServerName server :
        status
            .getServers()) { // co ClusterStatusExample-2-ServerInfo Iterate over the included
                             // server instances.
      System.out.println("Hostname: " + server.getHostname());
      System.out.println("Host and Port: " + server.getHostAndPort());
      System.out.println("Server Name: " + server.getServerName());
      System.out.println("RPC Port: " + server.getPort());
      System.out.println("Start Code: " + server.getStartcode());
      ServerLoad load =
          status.getLoad(
              server); // co ClusterStatusExample-3-ServerLoad Retrieve the load details for the
                       // current server.
      System.out.println("\nServer Load:\n--------------");
      System.out.println("Info Port: " + load.getInfoServerPort());
      System.out.println("Load: " + load.getLoad());
      System.out.println("Max Heap (MB): " + load.getMaxHeapMB());
      System.out.println("Used Heap (MB): " + load.getUsedHeapMB());
      System.out.println("Memstore Size (MB): " + load.getMemstoreSizeInMB());
      System.out.println("No. Regions: " + load.getNumberOfRegions());
      System.out.println("No. Requests: " + load.getNumberOfRequests());
      System.out.println("Total No. Requests: " + load.getTotalNumberOfRequests());
      System.out.println("No. Requests per Sec: " + load.getRequestsPerSecond());
      System.out.println("No. Read Requests: " + load.getReadRequestsCount());
      System.out.println("No. Write Requests: " + load.getWriteRequestsCount());
      System.out.println("No. Stores: " + load.getStores());
      System.out.println("Store Size Uncompressed (MB): " + load.getStoreUncompressedSizeMB());
      System.out.println("No. Storefiles: " + load.getStorefiles());
      System.out.println("Storefile Size (MB): " + load.getStorefileSizeInMB());
      System.out.println("Storefile Index Size (MB): " + load.getStorefileIndexSizeInMB());
      System.out.println("Root Index Size: " + load.getRootIndexSizeKB());
      System.out.println("Total Bloom Size: " + load.getTotalStaticBloomSizeKB());
      System.out.println("Total Index Size: " + load.getTotalStaticIndexSizeKB());
      System.out.println("Current Compacted Cells: " + load.getCurrentCompactedKVs());
      System.out.println("Total Compacting Cells: " + load.getTotalCompactingKVs());
      System.out.println("Coprocessors1: " + Arrays.asList(load.getRegionServerCoprocessors()));
      System.out.println("Coprocessors2: " + Arrays.asList(load.getRsCoprocessors()));
      System.out.println("Replication Load Sink: " + load.getReplicationLoadSink());
      System.out.println("Replication Load Source: " + load.getReplicationLoadSourceList());
      System.out.println("\nRegion Load:\n--------------");
      for (Map.Entry<byte[], RegionLoad>
          entry : // co ClusterStatusExample-4-Regions Iterate over the region details of the
                  // current server.
          load.getRegionsLoad().entrySet()) {
        System.out.println("Region: " + Bytes.toStringBinary(entry.getKey()));
        RegionLoad regionLoad =
            entry
                .getValue(); // co ClusterStatusExample-5-RegionLoad Get the load details for the
                             // current region.
        System.out.println("Name: " + Bytes.toStringBinary(regionLoad.getName()));
        System.out.println("Name (as String): " + regionLoad.getNameAsString());
        System.out.println("No. Requests: " + regionLoad.getRequestsCount());
        System.out.println("No. Read Requests: " + regionLoad.getReadRequestsCount());
        System.out.println("No. Write Requests: " + regionLoad.getWriteRequestsCount());
        System.out.println("No. Stores: " + regionLoad.getStores());
        System.out.println("No. Storefiles: " + regionLoad.getStorefiles());
        System.out.println("Data Locality: " + regionLoad.getDataLocality());
        System.out.println("Storefile Size (MB): " + regionLoad.getStorefileSizeMB());
        System.out.println("Storefile Index Size (MB): " + regionLoad.getStorefileIndexSizeMB());
        System.out.println("Memstore Size (MB): " + regionLoad.getMemStoreSizeMB());
        System.out.println("Root Index Size: " + regionLoad.getRootIndexSizeKB());
        System.out.println("Total Bloom Size: " + regionLoad.getTotalStaticBloomSizeKB());
        System.out.println("Total Index Size: " + regionLoad.getTotalStaticIndexSizeKB());
        System.out.println("Current Compacted Cells: " + regionLoad.getCurrentCompactedKVs());
        System.out.println("Total Compacting Cells: " + regionLoad.getTotalCompactingKVs());
        System.out.println();
      }
    }
  }
コード例 #9
0
ファイル: TrainRun.java プロジェクト: super-sponge/hbase-demo
  public static List<KeyValue> getAllTrainInfo(Configuration config, String date) {
    List<KeyValue> result = new ArrayList<>();
    String strJson = null;
    BufferedWriter writer = null;
    Table table = null;
    try (Connection connect = ConnectionFactory.createConnection(config);
        Admin admin = connect.getAdmin()) {
      TableName tablename = TableName.valueOf(TABLE_NAME);
      if (!admin.tableExists(tablename)) {
        System.out.println("Table does not exist.");
        return null;
      }

      table = connect.getTable(tablename);
      Put put = null;
      String start = null;
      String end = null;
      writer = new BufferedWriter(new FileWriter(new File(strConfig), true));
      for (KeyValue item : lstAllProcessStation) {
        start = (String) item.getKey();
        end = (String) item.getValue();
        try {
          try {
            Thread.sleep(200);
          } catch (InterruptedException e1) {
            e1.printStackTrace();
          }
          System.out.println("process : " + start + ":" + end);
          strJson = getFromAPIX(mapStationCode.get(start), mapStationCode.get(end), date);
          writer.write(start + ":" + end);
          writer.newLine();

        } catch (Exception e) {
          System.out.println(start + ":" + end + "error");
          e.printStackTrace();
          break;
        }
        JSONObject jo = new JSONObject(strJson);

        if (jo.has("httpstatus") && (jo.getInt("httpstatus") == 200)) {
          JSONObject joData = jo.getJSONObject("data");
          if (joData.has("flag") && joData.getBoolean("flag")) {
            result.add(new DefaultKeyValue(start, end));
            // 插入到hbase
            String rowkey = start + ":" + end;
            put = new Put(rowkey.getBytes());
            put.addColumn(
                CF_JSON.getBytes(), "json".getBytes(), joData.toString().getBytes("utf-8"));
            table.put(put);

            System.out.println("start " + start + "\t end " + end + "\t has ticket");
          }
        }
      }
    } catch (IOException e) {
      e.printStackTrace();
    } finally {
      if (writer != null) {
        try {
          writer.flush();
          writer.close();
        } catch (IOException e) {
          e.printStackTrace();
        }
      }
      if (table != null) {
        try {
          table.close();
        } catch (IOException e) {
          e.printStackTrace();
        }
      }
    }

    return result;
  }