/* * 为表添加数据(适合知道有多少列族的固定表) * * @rowKey rowKey * * @tableName 表名 * * @column1 第一个列族列表 * * @value1 第一个列的值的列表 * * @column2 第二个列族列表 * * @value2 第二个列的值的列表 */ public static void addData( String rowKey, String tableName, String[] column1, String[] value1, String[] column2, String[] value2) throws IOException { Put put = new Put(Bytes.toBytes(rowKey)); // 设置rowkey HTableInterface htable = conn.getTable(tableName); HColumnDescriptor[] columnFamilies = htable .getTableDescriptor() // 获取所有的列族 .getColumnFamilies(); for (int i = 0; i < columnFamilies.length; i++) { String familyName = columnFamilies[i].getNameAsString(); // 获取列族名 if (familyName.equals("article")) { // article列族put数据 for (int j = 0; j < column1.length; j++) { put.add(Bytes.toBytes(familyName), Bytes.toBytes(column1[j]), Bytes.toBytes(value1[j])); } } if (familyName.equals("author")) { // author列族put数据 for (int j = 0; j < column2.length; j++) { put.add(Bytes.toBytes(familyName), Bytes.toBytes(column2[j]), Bytes.toBytes(value2[j])); } } } htable.put(put); // System.out.println("add data Success!"); }
public static void deleteTest(String tableStr) { try { Configuration conf = HBaseConfiguration.create(); byte[] tableName = Bytes.toBytes(tableStr); HConnection hConnection = HConnectionManager.createConnection(conf); HTableInterface table = hConnection.getTable(tableName); byte[] startRow = Bytes.toBytes("rowKey_1"); byte[] stopRow = Bytes.toBytes("rowKey_3"); byte[] family = f0; Scan scan = new Scan(); scan.addFamily(family); scan.setMaxVersions(1); // scan.setStartRow(startRow); // scan.setStopRow(stopRow); ResultScanner scanner = table.getScanner(scan); Result result = scanner.next(); List<Delete> delete = new ArrayList<Delete>(); while (result != null) { Delete del = new Delete(result.getRow()); delete.add(del); result = scanner.next(); } table.delete(delete); System.out.println("delete done"); table.close(); // very important } catch (IOException e) { e.printStackTrace(); } }
public static void writeTest(String tableStr) { try { Configuration conf = HBaseConfiguration.create(); byte[] tableName = Bytes.toBytes(tableStr); HConnection hConnection = HConnectionManager.createConnection(conf); HTableInterface table = hConnection.getTable(tableName); byte[] family = f0; List<Put> puts = new ArrayList<Put>(); for (int k = 0; k < 10; k++) // 写10行数据 { byte[] rowkey = Bytes.toBytes("rowKey_" + k); Put p = new Put(rowkey); byte[] value_id = Bytes.toBytes("123456"); byte[] value_user = Bytes.toBytes("mengqinghao" + k); p.add(family, qualifier_id, value_id); p.add(family, qualifier_user, value_user); puts.add(p); } table.put(puts); System.out.println("Puts done: " + puts.size()); table.close(); // very important } catch (IOException e) { e.printStackTrace(); } }
public static void readTest(String tableStr, String row) { try { Configuration conf = HBaseConfiguration.create(); byte[] tableName = Bytes.toBytes(tableStr); HConnection hConnection = HConnectionManager.createConnection(conf); HTableInterface table = hConnection.getTable(tableName); byte[] rowkey = Bytes.toBytes(row); Get get = new Get(rowkey); get.addFamily(f0); Result result = table.get(get); NavigableMap<byte[], byte[]> m = result.getFamilyMap(f0); if (m == null || m.isEmpty()) { System.err.println("Empty." + m); return; } for (Map.Entry<byte[], byte[]> entry : m.entrySet()) { String qualifier = Bytes.toString(entry.getKey()); String value = Bytes.toString(entry.getValue()); System.out.println(qualifier + ":" + value); } table.close(); // very important } catch (IOException e) { e.printStackTrace(); } }
public HTableInterface getTable(String tablename) { try { if (connection == null) { connection = HConnectionManager.createConnection(config); } return connection.getTable(tablename); } catch (IOException e) { LOG.error(ExceptionUtils.getFullStackTrace(e)); LOG.error("exception accurs when getting table from hconnection, will be retried"); try { connection = HConnectionManager.createConnection(config); return connection.getTable(tablename); } catch (Throwable throwable) { throw new RuntimeException(throwable); } } }
public static void createTable(String tableName) { long start = System.currentTimeMillis(); try { HTableInterface htable = conn.getTable(tableName); List<Put> putList = new ArrayList<Put>(); HColumnDescriptor[] columnFamilies = htable.getTableDescriptor().getColumnFamilies(); String[] family = {"article", "author"}; creatTable(tableName, family); String[] column1 = {"title", "content", "tag"}; String[] column2 = {"name", "nickname"}; for (int i = 1; i <= 1000000; i++) { DecimalFormat format = new DecimalFormat("00000000"); String rowKey = format.format(i); // System.out.println("==| insert"+rowKey); String[] valueOfArticle = {"title" + i, "content" + i, "tag" + i}; String[] valueOfAuthor = {"name" + i, "nickname" + i}; // addData(rowKey, tableName, column1, valueOfArticle, column2, // valueOfAuthor); Put put = new Put(Bytes.toBytes(rowKey)); // 设置rowkey for (int colIndex = 0; colIndex < columnFamilies.length; colIndex++) { String familyName = columnFamilies[colIndex].getNameAsString(); // 获取列族名 if (familyName.equals("article")) { // article列族put数据 for (int k = 0; k < column1.length; k++) { put.add( Bytes.toBytes(familyName), Bytes.toBytes(column1[k]), Bytes.toBytes(valueOfArticle[k])); } } if (familyName.equals("author")) { // author列族put数据 for (int k = 0; k < column2.length; k++) { put.add( Bytes.toBytes(familyName), Bytes.toBytes(column2[k]), Bytes.toBytes(valueOfAuthor[k])); } } } putList.add(put); if (i % 10000 == 0) { htable.put(putList); putList = new ArrayList<Put>(); } } long end = System.currentTimeMillis(); long used = end - start; System.out.println("data insert finished used " + used + " ms"); } catch (Exception e) { e.printStackTrace(); } }
public static HTableInterface getHBaseInstance(String tableName) { try { return hConnection.getTable(tableName); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); return null; } }
/** Re-connect to HBase */ private void startClient() { if (this.htable == null) { try { hconnection = HConnectionManager.createConnection(hconf); htable = hconnection.getTable(table); } catch (IOException e) { LOG.warn("Failed to create HBase connection. " + e.getMessage()); } } }
@Override protected void setup(Context context) throws IOException, InterruptedException { super.setup(context); configuration = context.getConfiguration(); connection = HConnectionManager.createConnection(configuration); table = connection.getTable(""); table.setAutoFlush(false, true); table.setWriteBufferSize(12 * 1024 * 1024); wal = true; }
/** * 向表中插入一条新数据 * * @param tableName 表名 * @param row 行键key * @param columnFamily 列族 * @param column 列名 * @param data 要插入的数据 * @throws IOException */ public static void putData( String tableName, String row, String columnFamily, String column, byte[] data) throws IOException { HTableInterface table = conn.getTable(tableName); Put put = new Put(Bytes.toBytes(row)); put.add(Bytes.toBytes(columnFamily), Bytes.toBytes(column), data); table.put(put); System.out.println( "PUT:" + row + "','" + columnFamily + ":" + column + "','" + Bytes.toString(data) + "'"); }
/* * 遍历查询hbase表 * * @tableName 表名 */ public static void getResultScann(String tableName) throws IOException { Scan scan = new Scan(); ResultScanner rs = null; HTableInterface table = conn.getTable(tableName); try { // String split = StringUtils.S001; // QualifierFilter ff = new QualifierFilter(CompareOp.EQUAL, new BinaryComparator( // Bytes.toBytes("A"))); // scan.setFilter(ff); rs = table.getScanner(scan); int count = 0; for (Result r : rs) { count++; for (KeyValue kv : r.list()) { System.out.println("row:" + Bytes.toString(kv.getRow())); // System.out.println("family:" + Bytes.toString(kv.getFamily())); // System.out.println("qualifier:" + Bytes.toString(kv.getQualifier())); System.out.println("value:" + Bytes.toString(kv.getValue())); // // System.out.println("timestamp:" + kv.getTimestamp()); // StringBuilder sb = new StringBuilder(); // sb.append(Bytes.toString(r.getRow())); // sb.append(split); // sb.append(Bytes.toString(kv.getValue())); // EntBaseinfo baseInfo = new EntBaseinfo(); // baseInfo.parseFromString(sb.toString()); // System.out.println(baseInfo.getENTNAME()); // if(baseInfo.getNAME()!=null&&baseInfo.getNAME().isEmpty()){ // System.out.println(baseInfo.getENTNAME()); // } // // // if(baseInfo.getDOM()!=null&&baseInfo.getNAME().isEmpty()){ // System.out.println(baseInfo.getENTNAME()); // } } if (count > 1000) { return; } } } finally { rs.close(); } }
private void loadIPs() { dns = new HashMap(100000000); // ���貢�� unknownHosts = new HashMap(1000000); querying = new HashMap(100000); try { int statsCommit = 500000; HConnection connection = HConnectionManager.createConnection(HBaseConfiguration.create()); HTableInterface fetchFailTable = connection.getTable("fetchFail"); Scan scan = new Scan(); scan.setCaching(statsCommit); List<Filter> filters = new ArrayList<Filter>(); Filter filter = new ColumnPrefixFilter(Bytes.toBytes("ip")); filters.add(filter); FilterList filterList = new FilterList(filters); scan.setFilter(filterList); ResultScanner rs = fetchFailTable.getScanner(scan); long cnt = 0; for (Result r : rs) { NavigableMap<byte[], byte[]> map = r.getFamilyMap(Bytes.toBytes("cf")); String ip = Bytes.toString(map.get(Bytes.toBytes("ip"))); String host = Bytes.toString(r.getRow()).split("��")[0]; if (host != null && ip != null) { dns.put(host, ip); } if (++cnt % statsCommit == 0) { LOG.info("loadIPs url=" + Bytes.toString(r.getRow()) + " cnt=" + cnt); } } rs.close(); fetchFailTable.close(); LOG.info("load hostip cache=" + dns.size()); connection.close(); } catch (Exception e) { e.printStackTrace(); } finally { // } }
public static void scanTest(String tableStr) { try { Configuration conf = HBaseConfiguration.create(); byte[] tableName = Bytes.toBytes(tableStr); HConnection hConnection = HConnectionManager.createConnection(conf); HTableInterface table = hConnection.getTable(tableName); byte[] startRow = Bytes.toBytes("rowKey_0"); byte[] stopRow = Bytes.toBytes("rowKey_6"); byte[] family = f0; Scan scan = new Scan(); scan.addFamily(family); scan.setMaxVersions(1); // scan.setStartRow(startRow); // scan.setStopRow(stopRow); int count = 0; ResultScanner scanner = table.getScanner(scan); Result result = scanner.next(); while (result != null) { String rowKey = Bytes.toString(result.getRow()); NavigableMap<byte[], byte[]> m = result.getFamilyMap(family); if (m == null || m.isEmpty()) { System.err.println("Empty." + m); return; } for (Map.Entry<byte[], byte[]> entry : m.entrySet()) { String qualifier = Bytes.toString(entry.getKey()); String value = Bytes.toString(entry.getValue()); System.out.println(rowKey + ":" + qualifier + ":" + value); } result = scanner.next(); count++; System.out.println("-----------------------------"); } table.close(); // very important System.out.println("count:" + count); } catch (IOException e) { e.printStackTrace(); } }
public void testPool() throws IOException { PoolConfig config = new PoolConfig(); config.setMaxTotal(20); config.setMaxIdle(5); config.setMaxWaitMillis(1000); config.setTestOnBorrow(true); /* properties */ Properties props = new Properties(); props.setProperty("hbase.zookeeper.quorum", "host1,host2,host3"); props.setProperty("hbase.zookeeper.property.clientPort", "2181"); props.setProperty("hbase.master", "host1:60000"); props.setProperty("hbase.rootdir", "hdfs://host1:9000/hbase"); /* connection pool */ HbaseConnectionPool pool = new HbaseConnectionPool(config, props); HTableInterface table = null; HConnection conn = pool.getConnection(); table = conn.getTable(TableName.valueOf("relation")); Get get = new Get(Bytes.toBytes("rowKey")); Result r = table.get(get); for (Cell cell : r.rawCells()) { System.out.println( "Rowkey : " + Bytes.toString(r.getRow()) + " Familiy:Quilifier : " + Bytes.toString(CellUtil.cloneQualifier(cell)) + " Value : " + Bytes.toString(CellUtil.cloneValue(cell))); } table.close(); System.out.println(table); pool.returnConnection(conn); pool.close(); }