Example #1
0
 public Optional<byte[]> get(
     Optional<String> table,
     Optional<String> family,
     Optional<String> qualifier,
     Optional<String> key) {
   if (!valid) {
     Logger.error("CANNOT GET! NO VALID CONNECTION");
     return Optional.empty();
   }
   if (table.isPresent()
       && family.isPresent()
       && qualifier.isPresent()
       && key.isPresent()
       && !key.get().isEmpty()) {
     try {
       final Table htable = connection.getTable(TableName.valueOf(table.get()));
       Result result = htable.get(new Get(key.get().getBytes("UTF8")));
       return Optional.ofNullable(
           result.getValue(family.get().getBytes("UTF8"), qualifier.get().getBytes("UTF8")));
     } catch (IOException e) {
       e.printStackTrace();
     }
   }
   return Optional.empty();
 }
Example #2
0
 public Map<String, Long> getRegionSizes(String tableName) {
   Map<String, Long> regions = new HashMap<>();
   try {
     final Table table = connection.getTable(TableName.valueOf(tableName));
     RegionLocator regionLocator = connection.getRegionLocator(table.getName());
     List<HRegionLocation> tableRegionInfos = regionLocator.getAllRegionLocations();
     Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
     for (HRegionLocation regionInfo : tableRegionInfos) {
       tableRegions.add(regionInfo.getRegionInfo().getRegionName());
     }
     ClusterStatus clusterStatus = connection.getAdmin().getClusterStatus();
     Collection<ServerName> servers = clusterStatus.getServers();
     final long megaByte = 1024L * 1024L;
     for (ServerName serverName : servers) {
       ServerLoad serverLoad = clusterStatus.getLoad(serverName);
       for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) {
         byte[] regionId = regionLoad.getName();
         if (tableRegions.contains(regionId)) {
           long regionSizeBytes = regionLoad.getStorefileSizeMB() * megaByte;
           regions.put(regionLoad.getNameAsString(), regionSizeBytes);
         }
       }
     }
   } catch (IOException e) {
     e.printStackTrace();
   }
   return regions;
 }
Example #3
0
 public void putBatch(Optional<List<Request>> putRequests, boolean optimize) {
   if (!valid) {
     Logger.error("CANNOT PUT! NO VALID CONNECTION");
     return;
   }
   List<Put> puts = new ArrayList<>();
   if (putRequests.isPresent() && !putRequests.get().isEmpty()) {
     String tableName = putRequests.get().get(0).table;
     putRequests
         .get()
         .forEach(
             pr ->
                 pr.getPut()
                     .ifPresent(
                         p -> {
                           if (optimize) {
                             p.setDurability(Durability.SKIP_WAL);
                           }
                           puts.add(p);
                         }));
     try {
       final Table table = connection.getTable(TableName.valueOf(tableName));
       if (optimize && table instanceof HTable) {
         ((HTable) table).setAutoFlush(false, true);
       }
       table.put(puts);
       table.close();
     } catch (IOException e) {
       e.printStackTrace();
     }
   }
 }
Example #4
0
 public Optional<Response> get(Optional<Request> request) {
   if (!valid) {
     Logger.error("CANNOT GET! NO VALID CONNECTION");
     return Optional.empty();
   }
   Response response = new Response();
   if (request.isPresent()) {
     Request r = request.get();
     response.key = r.key;
     response.table = r.table;
     try {
       final Table htable = connection.getTable(TableName.valueOf(r.table));
       Result result = htable.get(new Get(r.key));
       if (result == null || result.isEmpty()) {
         return Optional.empty();
       }
       r.columns.forEach(
           c ->
               response.columns.add(
                   new Request.Column(
                       c.family,
                       c.qualifier,
                       result.getValue(c.family.getBytes(), c.qualifier.getBytes()))));
     } catch (IOException e) {
       e.printStackTrace();
     }
   }
   return Optional.of(response);
 }
Example #5
0
 public void put(String tablename, Put p) {
   try {
     final Table table = connection.getTable(TableName.valueOf(tablename));
     table.put(p);
     table.close();
   } catch (IOException e) {
     e.printStackTrace();
   }
 }
Example #6
0
 public Result get(String table, String family, byte[] key) {
   final Table htable;
   try {
     htable = connection.getTable(TableName.valueOf(table));
     return htable.get(new Get(key));
   } catch (IOException e) {
     e.printStackTrace();
   }
   return null;
 }
 public void doScan() throws IOException {
   Table tableRef = connection.getTable(TableName.valueOf(table));
   Scan scan = new Scan();
   ResultScanner scanner = tableRef.getScanner(scan);
   long now = System.currentTimeMillis();
   if (verbose) System.out.println("Starting scan");
   for (Result res : scanner) {
     if (verbose) System.out.println(res);
   }
   if (verbose) System.out.printf("Scan finished: %d ms\n\n", System.currentTimeMillis() - now);
   tableRef.close();
 }
Example #8
0
  @Test
  public void testMetaScanner() throws Exception {
    LOG.info("Starting testMetaScanner");

    setUp();
    final TableName TABLENAME = TableName.valueOf("testMetaScanner");
    final byte[] FAMILY = Bytes.toBytes("family");
    TEST_UTIL.createTable(TABLENAME, FAMILY);
    Configuration conf = TEST_UTIL.getConfiguration();
    HTable table = (HTable) connection.getTable(TABLENAME);
    TEST_UTIL.createMultiRegions(
        conf,
        table,
        FAMILY,
        new byte[][] {
          HConstants.EMPTY_START_ROW, Bytes.toBytes("region_a"), Bytes.toBytes("region_b")
        });
    // Make sure all the regions are deployed
    TEST_UTIL.countRows(table);

    MetaScanner.MetaScannerVisitor visitor = mock(MetaScanner.MetaScannerVisitor.class);
    doReturn(true).when(visitor).processRow((Result) anyObject());

    // Scanning the entire table should give us three rows
    MetaScanner.metaScan(connection, visitor, TABLENAME);
    verify(visitor, times(3)).processRow((Result) anyObject());

    // Scanning the table with a specified empty start row should also
    // give us three hbase:meta rows
    reset(visitor);
    doReturn(true).when(visitor).processRow((Result) anyObject());
    MetaScanner.metaScan(connection, visitor, TABLENAME, HConstants.EMPTY_BYTE_ARRAY, 1000);
    verify(visitor, times(3)).processRow((Result) anyObject());

    // Scanning the table starting in the middle should give us two rows:
    // region_a and region_b
    reset(visitor);
    doReturn(true).when(visitor).processRow((Result) anyObject());
    MetaScanner.metaScan(connection, visitor, TABLENAME, Bytes.toBytes("region_ac"), 1000);
    verify(visitor, times(2)).processRow((Result) anyObject());

    // Scanning with a limit of 1 should only give us one row
    reset(visitor);
    doReturn(true).when(visitor).processRow((Result) anyObject());
    MetaScanner.metaScan(connection, visitor, TABLENAME, Bytes.toBytes("region_ac"), 1);
    verify(visitor, times(1)).processRow((Result) anyObject());
    table.close();
  }
Example #9
0
 public void scan(
     Consumer<Result> callback, String tableName, String columnFamily, String... qualifiers) {
   if (callback != null && tableName != null && columnFamily != null && qualifiers != null) {
     Scan s = new Scan();
     byte[] family = Bytes.toBytes(columnFamily);
     for (String qualifier : qualifiers) {
       s.addColumn(family, Bytes.toBytes(qualifier));
     }
     try {
       final Table table = connection.getTable(TableName.valueOf(tableName));
       ResultScanner scanner = table.getScanner(s);
       for (Result r = scanner.next(); r != null; r = scanner.next()) {
         callback.accept(r);
       }
     } catch (IOException e) {
       e.printStackTrace();
     }
   }
 }
Example #10
0
 public void put(Optional<Request> putRequest) {
   if (!valid) {
     Logger.error("CANNOT PUT! NO VALID CONNECTION");
     return;
   }
   putRequest.ifPresent(
       pr ->
           pr.getPut()
               .ifPresent(
                   p -> {
                     try {
                       final Table table = connection.getTable(TableName.valueOf(pr.table));
                       table.put(p);
                       table.close();
                     } catch (IOException e) {
                       e.printStackTrace();
                     }
                   }));
 }
Example #11
0
  public static List<KeyValue> getAllTrainInfo(Configuration config, String date) {
    List<KeyValue> result = new ArrayList<>();
    String strJson = null;
    BufferedWriter writer = null;
    Table table = null;
    try (Connection connect = ConnectionFactory.createConnection(config);
        Admin admin = connect.getAdmin()) {
      TableName tablename = TableName.valueOf(TABLE_NAME);
      if (!admin.tableExists(tablename)) {
        System.out.println("Table does not exist.");
        return null;
      }

      table = connect.getTable(tablename);
      Put put = null;
      String start = null;
      String end = null;
      writer = new BufferedWriter(new FileWriter(new File(strConfig), true));
      for (KeyValue item : lstAllProcessStation) {
        start = (String) item.getKey();
        end = (String) item.getValue();
        try {
          try {
            Thread.sleep(200);
          } catch (InterruptedException e1) {
            e1.printStackTrace();
          }
          System.out.println("process : " + start + ":" + end);
          strJson = getFromAPIX(mapStationCode.get(start), mapStationCode.get(end), date);
          writer.write(start + ":" + end);
          writer.newLine();

        } catch (Exception e) {
          System.out.println(start + ":" + end + "error");
          e.printStackTrace();
          break;
        }
        JSONObject jo = new JSONObject(strJson);

        if (jo.has("httpstatus") && (jo.getInt("httpstatus") == 200)) {
          JSONObject joData = jo.getJSONObject("data");
          if (joData.has("flag") && joData.getBoolean("flag")) {
            result.add(new DefaultKeyValue(start, end));
            // 插入到hbase
            String rowkey = start + ":" + end;
            put = new Put(rowkey.getBytes());
            put.addColumn(
                CF_JSON.getBytes(), "json".getBytes(), joData.toString().getBytes("utf-8"));
            table.put(put);

            System.out.println("start " + start + "\t end " + end + "\t has ticket");
          }
        }
      }
    } catch (IOException e) {
      e.printStackTrace();
    } finally {
      if (writer != null) {
        try {
          writer.flush();
          writer.close();
        } catch (IOException e) {
          e.printStackTrace();
        }
      }
      if (table != null) {
        try {
          table.close();
        } catch (IOException e) {
          e.printStackTrace();
        }
      }
    }

    return result;
  }