/*
* 为表添加数据（适合知道有多少列族的固定表）
*
* @rowKey rowKey
*
* @tableName 表名
*
* @column1 第一个列族列表
*
* @value1 第一个列的值的列表
*
* @column2 第二个列族列表
*
* @value2 第二个列的值的列表
*/
public static void addData(
String rowKey,
String tableName,
String[] column1,
String[] value1,
String[] column2,
String[] value2)
throws IOException {
Put put = new Put(Bytes.toBytes(rowKey)); // 设置rowkey
HTableInterface htable = conn.getTable(tableName);
HColumnDescriptor[] columnFamilies =
htable
.getTableDescriptor() // 获取所有的列族
.getColumnFamilies();
for (int i = 0; i < columnFamilies.length; i++) {
String familyName = columnFamilies[i].getNameAsString(); // 获取列族名
if (familyName.equals("article")) { // article列族put数据
for (int j = 0; j < column1.length; j++) {
put.add(Bytes.toBytes(familyName), Bytes.toBytes(column1[j]), Bytes.toBytes(value1[j]));
}
}
if (familyName.equals("author")) { // author列族put数据
for (int j = 0; j < column2.length; j++) {
put.add(Bytes.toBytes(familyName), Bytes.toBytes(column2[j]), Bytes.toBytes(value2[j]));
}
}
}
htable.put(put);
// System.out.println("add data Success!");
}
public static void deleteTest(String tableStr) {
try {
Configuration conf = HBaseConfiguration.create();
byte[] tableName = Bytes.toBytes(tableStr);
HConnection hConnection = HConnectionManager.createConnection(conf);
HTableInterface table = hConnection.getTable(tableName);
byte[] startRow = Bytes.toBytes("rowKey_1");
byte[] stopRow = Bytes.toBytes("rowKey_3");
byte[] family = f0;
Scan scan = new Scan();
scan.addFamily(family);
scan.setMaxVersions(1);
// scan.setStartRow(startRow);
// scan.setStopRow(stopRow);
ResultScanner scanner = table.getScanner(scan);
Result result = scanner.next();
List<Delete> delete = new ArrayList<Delete>();
while (result != null) {
Delete del = new Delete(result.getRow());
delete.add(del);
result = scanner.next();
}
table.delete(delete);
System.out.println("delete done");
table.close(); // very important
} catch (IOException e) {
e.printStackTrace();
}
}
public static void writeTest(String tableStr) {
try {
Configuration conf = HBaseConfiguration.create();
byte[] tableName = Bytes.toBytes(tableStr);
HConnection hConnection = HConnectionManager.createConnection(conf);
HTableInterface table = hConnection.getTable(tableName);
byte[] family = f0;
List<Put> puts = new ArrayList<Put>();
for (int k = 0; k < 10; k++) // 写10行数据
{
byte[] rowkey = Bytes.toBytes("rowKey_" + k);
Put p = new Put(rowkey);
byte[] value_id = Bytes.toBytes("123456");
byte[] value_user = Bytes.toBytes("mengqinghao" + k);
p.add(family, qualifier_id, value_id);
p.add(family, qualifier_user, value_user);
puts.add(p);
}
table.put(puts);
System.out.println("Puts done: " + puts.size());
table.close(); // very important
} catch (IOException e) {
e.printStackTrace();
}
}
public static void readTest(String tableStr, String row) {
try {
Configuration conf = HBaseConfiguration.create();
byte[] tableName = Bytes.toBytes(tableStr);
HConnection hConnection = HConnectionManager.createConnection(conf);
HTableInterface table = hConnection.getTable(tableName);
byte[] rowkey = Bytes.toBytes(row);
Get get = new Get(rowkey);
get.addFamily(f0);
Result result = table.get(get);
NavigableMap<byte[], byte[]> m = result.getFamilyMap(f0);
if (m == null || m.isEmpty()) {
System.err.println("Empty." + m);
return;
}
for (Map.Entry<byte[], byte[]> entry : m.entrySet()) {
String qualifier = Bytes.toString(entry.getKey());
String value = Bytes.toString(entry.getValue());
System.out.println(qualifier + ":" + value);
}
table.close(); // very important
} catch (IOException e) {
e.printStackTrace();
}
}
public HTableInterface getTable(String tablename) {
try {
if (connection == null) {
connection = HConnectionManager.createConnection(config);
}
return connection.getTable(tablename);
} catch (IOException e) {
LOG.error(ExceptionUtils.getFullStackTrace(e));
LOG.error("exception accurs when getting table from hconnection, will be retried");
try {
connection = HConnectionManager.createConnection(config);
return connection.getTable(tablename);
} catch (Throwable throwable) {
throw new RuntimeException(throwable);
}
}
}
public static void createTable(String tableName) {
long start = System.currentTimeMillis();
try {
HTableInterface htable = conn.getTable(tableName);
List<Put> putList = new ArrayList<Put>();
HColumnDescriptor[] columnFamilies = htable.getTableDescriptor().getColumnFamilies();
String[] family = {"article", "author"};
creatTable(tableName, family);
String[] column1 = {"title", "content", "tag"};
String[] column2 = {"name", "nickname"};
for (int i = 1; i <= 1000000; i++) {
DecimalFormat format = new DecimalFormat("00000000");
String rowKey = format.format(i);
// System.out.println("==| insert"+rowKey);
String[] valueOfArticle = {"title" + i, "content" + i, "tag" + i};
String[] valueOfAuthor = {"name" + i, "nickname" + i};
// addData(rowKey, tableName, column1, valueOfArticle, column2,
// valueOfAuthor);
Put put = new Put(Bytes.toBytes(rowKey)); // 设置rowkey
for (int colIndex = 0; colIndex < columnFamilies.length; colIndex++) {
String familyName = columnFamilies[colIndex].getNameAsString(); // 获取列族名
if (familyName.equals("article")) { // article列族put数据
for (int k = 0; k < column1.length; k++) {
put.add(
Bytes.toBytes(familyName),
Bytes.toBytes(column1[k]),
Bytes.toBytes(valueOfArticle[k]));
}
}
if (familyName.equals("author")) { // author列族put数据
for (int k = 0; k < column2.length; k++) {
put.add(
Bytes.toBytes(familyName),
Bytes.toBytes(column2[k]),
Bytes.toBytes(valueOfAuthor[k]));
}
}
}
putList.add(put);
if (i % 10000 == 0) {
htable.put(putList);
putList = new ArrayList<Put>();
}
}
long end = System.currentTimeMillis();
long used = end - start;
System.out.println("data insert finished used " + used + " ms");
} catch (Exception e) {
e.printStackTrace();
}
}
public static HTableInterface getHBaseInstance(String tableName) {
try {
return hConnection.getTable(tableName);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return null;
}
}
/** Re-connect to HBase */
private void startClient() {
if (this.htable == null) {
try {
hconnection = HConnectionManager.createConnection(hconf);
htable = hconnection.getTable(table);
} catch (IOException e) {
LOG.warn("Failed to create HBase connection. " + e.getMessage());
}
}
}
@Override
protected void setup(Context context) throws IOException, InterruptedException {
super.setup(context);
configuration = context.getConfiguration();
connection = HConnectionManager.createConnection(configuration);
table = connection.getTable("");
table.setAutoFlush(false, true);
table.setWriteBufferSize(12 * 1024 * 1024);
wal = true;
}
/**
* 向表中插入一条新数据
*
* @param tableName 表名
* @param row 行键key
* @param columnFamily 列族
* @param column 列名
* @param data 要插入的数据
* @throws IOException
*/
public static void putData(
String tableName, String row, String columnFamily, String column, byte[] data)
throws IOException {
HTableInterface table = conn.getTable(tableName);
Put put = new Put(Bytes.toBytes(row));
put.add(Bytes.toBytes(columnFamily), Bytes.toBytes(column), data);
table.put(put);
System.out.println(
"PUT:" + row + "','" + columnFamily + ":" + column + "','" + Bytes.toString(data) + "'");
}
/*
* 遍历查询hbase表
*
* @tableName 表名
*/
public static void getResultScann(String tableName) throws IOException {
Scan scan = new Scan();
ResultScanner rs = null;
HTableInterface table = conn.getTable(tableName);
try {
// String split = StringUtils.S001;
// QualifierFilter ff = new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(
// Bytes.toBytes("A")));
// scan.setFilter(ff);
rs = table.getScanner(scan);
int count = 0;
for (Result r : rs) {
count++;
for (KeyValue kv : r.list()) {
System.out.println("row:" + Bytes.toString(kv.getRow()));
// System.out.println("family:" + Bytes.toString(kv.getFamily()));
// System.out.println("qualifier:" + Bytes.toString(kv.getQualifier()));
System.out.println("value:" + Bytes.toString(kv.getValue()));
//
// System.out.println("timestamp:" + kv.getTimestamp());
// StringBuilder sb = new StringBuilder();
// sb.append(Bytes.toString(r.getRow()));
// sb.append(split);
// sb.append(Bytes.toString(kv.getValue()));
// EntBaseinfo baseInfo = new EntBaseinfo();
// baseInfo.parseFromString(sb.toString());
// System.out.println(baseInfo.getENTNAME());
// if(baseInfo.getNAME()!=null&&baseInfo.getNAME().isEmpty()){
// System.out.println(baseInfo.getENTNAME());
// }
//
//
// if(baseInfo.getDOM()!=null&&baseInfo.getNAME().isEmpty()){
// System.out.println(baseInfo.getENTNAME());
// }
}
if (count > 1000) {
return;
}
}
} finally {
rs.close();
}
}
private void loadIPs() {
dns = new HashMap(100000000); // ���貢��
unknownHosts = new HashMap(1000000);
querying = new HashMap(100000);
try {
int statsCommit = 500000;
HConnection connection = HConnectionManager.createConnection(HBaseConfiguration.create());
HTableInterface fetchFailTable = connection.getTable("fetchFail");
Scan scan = new Scan();
scan.setCaching(statsCommit);
List<Filter> filters = new ArrayList<Filter>();
Filter filter = new ColumnPrefixFilter(Bytes.toBytes("ip"));
filters.add(filter);
FilterList filterList = new FilterList(filters);
scan.setFilter(filterList);
ResultScanner rs = fetchFailTable.getScanner(scan);
long cnt = 0;
for (Result r : rs) {
NavigableMap<byte[], byte[]> map = r.getFamilyMap(Bytes.toBytes("cf"));
String ip = Bytes.toString(map.get(Bytes.toBytes("ip")));
String host = Bytes.toString(r.getRow()).split("��")[0];
if (host != null && ip != null) {
dns.put(host, ip);
}
if (++cnt % statsCommit == 0) {
LOG.info("loadIPs url=" + Bytes.toString(r.getRow()) + " cnt=" + cnt);
}
}
rs.close();
fetchFailTable.close();
LOG.info("load hostip cache=" + dns.size());
connection.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
//
}
}
public static void scanTest(String tableStr) {
try {
Configuration conf = HBaseConfiguration.create();
byte[] tableName = Bytes.toBytes(tableStr);
HConnection hConnection = HConnectionManager.createConnection(conf);
HTableInterface table = hConnection.getTable(tableName);
byte[] startRow = Bytes.toBytes("rowKey_0");
byte[] stopRow = Bytes.toBytes("rowKey_6");
byte[] family = f0;
Scan scan = new Scan();
scan.addFamily(family);
scan.setMaxVersions(1);
// scan.setStartRow(startRow);
// scan.setStopRow(stopRow);
int count = 0;
ResultScanner scanner = table.getScanner(scan);
Result result = scanner.next();
while (result != null) {
String rowKey = Bytes.toString(result.getRow());
NavigableMap<byte[], byte[]> m = result.getFamilyMap(family);
if (m == null || m.isEmpty()) {
System.err.println("Empty." + m);
return;
}
for (Map.Entry<byte[], byte[]> entry : m.entrySet()) {
String qualifier = Bytes.toString(entry.getKey());
String value = Bytes.toString(entry.getValue());
System.out.println(rowKey + ":" + qualifier + ":" + value);
}
result = scanner.next();
count++;
System.out.println("-----------------------------");
}
table.close(); // very important
System.out.println("count:" + count);
} catch (IOException e) {
e.printStackTrace();
}
}
public void testPool() throws IOException {
PoolConfig config = new PoolConfig();
config.setMaxTotal(20);
config.setMaxIdle(5);
config.setMaxWaitMillis(1000);
config.setTestOnBorrow(true);
/* properties */
Properties props = new Properties();
props.setProperty("hbase.zookeeper.quorum", "host1,host2,host3");
props.setProperty("hbase.zookeeper.property.clientPort", "2181");
props.setProperty("hbase.master", "host1:60000");
props.setProperty("hbase.rootdir", "hdfs://host1:9000/hbase");
/* connection pool */
HbaseConnectionPool pool = new HbaseConnectionPool(config, props);
HTableInterface table = null;
HConnection conn = pool.getConnection();
table = conn.getTable(TableName.valueOf("relation"));
Get get = new Get(Bytes.toBytes("rowKey"));
Result r = table.get(get);
for (Cell cell : r.rawCells()) {
System.out.println(
"Rowkey : "
+ Bytes.toString(r.getRow())
+ " Familiy:Quilifier : "
+ Bytes.toString(CellUtil.cloneQualifier(cell))
+ " Value : "
+ Bytes.toString(CellUtil.cloneValue(cell)));
}
table.close();
System.out.println(table);
pool.returnConnection(conn);
pool.close();
}