/**
* rowFilter的使用
*
* @param tableName
* @param reg
* @throws Exception
*/
public void getRowFilter(String tableName, String reg) throws Exception {
Connection conn = null;
HTable table = null;
try {
conn = ConnectionFactory.createConnection(conf);
table = (HTable) conn.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
// Filter
RowFilter rowFilter = new RowFilter(CompareOp.NOT_EQUAL, new RegexStringComparator(reg));
scan.setFilter(rowFilter);
ResultScanner scanner = table.getScanner(scan);
for (Result result : scanner) {
System.out.println(new String(result.getRow()));
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (null != table) {
try {
table.close();
} catch (IOException e) {
logger.error("HTable close exception, errMsg:{}", e.getMessage());
}
}
if (null != conn) {
try {
conn.close();
} catch (Exception e) {
logger.error("Connection close exception, errMsg:{}", e.getMessage());
}
}
}
}
/** Close the open connections and shutdown the batchpool */
public void close() {
synchronized (connectionsLock) {
if (connections != null) {
for (Connection conn : connections) {
if (conn != null) {
try {
conn.close();
} catch (IOException e) {
LOG.info("Got exception in closing connection", e);
} finally {
conn = null;
}
}
}
connections = null;
}
}
if (this.batchPool != null && !this.batchPool.isShutdown()) {
this.batchPool.shutdown();
try {
if (!this.batchPool.awaitTermination(10, TimeUnit.SECONDS)) {
this.batchPool.shutdownNow();
}
} catch (InterruptedException e) {
this.batchPool.shutdownNow();
}
}
}
/**
* 删除指定表名
*
* @param rowKey
*/
public void deleteTable(byte[] rowKey) {
Connection conn = null;
HBaseAdmin admin = null;
try {
conn = ConnectionFactory.createConnection(conf);
admin = (HBaseAdmin) conn.getAdmin();
// 在删除一张表前，要使其失效
admin.disableTable(rowKey);
admin.deleteTable(rowKey);
admin.enableTable(rowKey);
} catch (Exception e) {
logger.error("HBaseAdmin deleteTable exception, errMsg:{}", e.getMessage());
} finally {
try {
if (null != admin) {
admin.close();
}
} catch (IOException e) {
logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage());
}
try {
if (null != conn) {
conn.close();
}
} catch (Exception e) {
logger.error("Connection close exception, errMsg:{}", e.getMessage());
}
}
}
/**
* 查询所有表
*
* @return 所以表 / null
* @throws Exception
*/
public static List<HTableDescriptor> queryALLTable() throws Exception {
Connection conn = null;
HBaseAdmin admin = null;
try {
conn = ConnectionFactory.createConnection(conf);
admin = (HBaseAdmin) conn.getAdmin();
if (admin != null) {
HTableDescriptor[] listTables = admin.listTables();
if (null != listTables && listTables.length > 0) {
return Arrays.asList(listTables);
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (null != admin) {
admin.close();
}
} catch (IOException e) {
logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage());
}
try {
if (null != conn) {
conn.close();
}
} catch (Exception e) {
logger.error("Connection close exception, errMsg:{}", e.getMessage());
}
}
return null;
}
/**
* 删除指定名称的列簇
*
* @param tableName 表名
* @param columnFamilyName 列族
*/
public static void deleteFamily(byte[] tableName, String columnFamilyName) {
Connection conn = null;
HBaseAdmin admin = null;
try {
conn = ConnectionFactory.createConnection(conf);
admin = (HBaseAdmin) conn.getAdmin();
admin.deleteColumn(tableName, columnFamilyName);
} catch (Exception e) {
logger.error("HBaseAdmin deleteColumn exception, errMsg:{}", e.getMessage());
} finally {
try {
if (null != admin) {
admin.close();
}
} catch (IOException e) {
logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage());
}
try {
if (null != conn) {
conn.close();
}
} catch (Exception e) {
logger.error("Connection close exception, errMsg:{}", e.getMessage());
}
}
}
/** 删除指定行 */
public static void deleteRow(byte[] rowKey) {
Connection conn = null;
HTable table = null;
try {
conn = ConnectionFactory.createConnection(conf);
table = (HTable) conn.getTable(TableName.valueOf("test"));
table.delete(new Delete(rowKey));
} catch (Exception e) {
logger.error("HBaseAdmin deleteRow exception, errMsg:{}", e.getMessage());
} finally {
if (null != table) {
try {
table.close();
} catch (IOException e) {
logger.error("HTable close exception, errMsg:{}", e.getMessage());
}
}
if (null != conn) {
try {
conn.close();
} catch (Exception e) {
logger.error("Connection close exception, errMsg:{}", e.getMessage());
}
}
}
}
@Override
protected int doWork() throws Exception {
Connection connection = null;
Admin admin = null;
try {
connection = ConnectionFactory.createConnection(getConf());
admin = connection.getAdmin();
HBaseProtos.SnapshotDescription.Type type = HBaseProtos.SnapshotDescription.Type.FLUSH;
if (snapshotType != null) {
type = ProtobufUtil.createProtosSnapShotDescType(snapshotName);
}
admin.snapshot(
new SnapshotDescription(snapshotName, tableName, ProtobufUtil.createSnapshotType(type)));
} catch (Exception e) {
return -1;
} finally {
if (admin != null) {
admin.close();
}
if (connection != null) {
connection.close();
}
}
return 0;
}
/**
* 根据RowKey查询单行
*
* @param rowKey
* @return
*/
public static Result queryByRowKey(byte[] rowKey) {
Result result = null;
if (null == rowKey || rowKey.length < 0) {
return result;
}
Connection conn = null;
HTable table = null;
try {
conn = ConnectionFactory.createConnection(conf);
table = (HTable) conn.getTable(TableName.valueOf("test"));
result = table.get(new Get(rowKey));
} catch (Exception e) {
logger.error("queryByRowKey exception, rowKey:{}, errMsg:{}", new String(rowKey), e);
} finally {
if (null != table) {
try {
table.close();
} catch (IOException e) {
logger.error("HTable close exception, errMsg:{}", e.getMessage());
}
}
if (null != conn) {
try {
conn.close();
} catch (IOException e) {
logger.error("Connection close exception, errMsg:{}", e.getMessage());
}
}
}
return result;
}
/**
* Constructor that creates a connection to the local ZooKeeper ensemble.
*
* @param conf Configuration to use
* @throws IOException if an internal replication error occurs
* @throws RuntimeException if replication isn't enabled.
*/
public ReplicationAdmin(Configuration conf) throws IOException {
if (!conf.getBoolean(
HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT)) {
throw new RuntimeException(
"hbase.replication isn't true, please " + "enable it in order to use replication");
}
this.connection = ConnectionFactory.createConnection(conf);
try {
zkw = createZooKeeperWatcher();
try {
this.replicationPeers = ReplicationFactory.getReplicationPeers(zkw, conf, this.connection);
this.replicationPeers.init();
this.replicationQueuesClient =
ReplicationFactory.getReplicationQueuesClient(zkw, conf, this.connection);
this.replicationQueuesClient.init();
} catch (Exception exception) {
if (zkw != null) {
zkw.close();
}
throw exception;
}
} catch (Exception exception) {
if (connection != null) {
connection.close();
}
if (exception instanceof IOException) {
throw (IOException) exception;
} else if (exception instanceof RuntimeException) {
throw (RuntimeException) exception;
} else {
throw new IOException("Error initializing the replication admin client.", exception);
}
}
}
/**
* Sweeps the mob files on one column family. It deletes the unused mob files and merges the small
* mob files into bigger ones.
*
* @param tableName The current table name in string format.
* @param familyName The column family name.
* @return 0 if success, 2 if job aborted with an exception, 3 if unable to start due to other
* compaction,4 if mr job was unsuccessful
* @throws IOException
* @throws InterruptedException
* @throws ClassNotFoundException
* @throws KeeperException
* @throws ServiceException
*/
int sweepFamily(String tableName, String familyName)
throws IOException, InterruptedException, ClassNotFoundException, KeeperException,
ServiceException {
Configuration conf = getConf();
// make sure the target HBase exists.
HBaseAdmin.checkHBaseAvailable(conf);
Connection connection = ConnectionFactory.createConnection(getConf());
Admin admin = connection.getAdmin();
try {
FileSystem fs = FileSystem.get(conf);
TableName tn = TableName.valueOf(tableName);
HTableDescriptor htd = admin.getTableDescriptor(tn);
HColumnDescriptor family = htd.getFamily(Bytes.toBytes(familyName));
if (family == null || !family.isMobEnabled()) {
throw new IOException("Column family " + familyName + " is not a MOB column family");
}
SweepJob job = new SweepJob(conf, fs);
// Run the sweeping
return job.sweep(tn, family);
} catch (Exception e) {
System.err.println("Job aborted due to exception " + e);
return 2; // job failed
} finally {
try {
admin.close();
} catch (IOException e) {
System.out.println("Failed to close the HBaseAdmin: " + e.getMessage());
}
try {
connection.close();
} catch (IOException e) {
System.out.println("Failed to close the connection: " + e.getMessage());
}
}
}
public static boolean createTableOrOverwrite(String tableName, String[] columnFamily) {
if (tableName == null && columnFamily == null) {
return false;
}
Connection connection = null;
try {
connection = ConnectionFactory.createConnection(config);
Admin admin = connection.getAdmin();
if (admin.tableExists(TableName.valueOf(tableName))) {
admin.disableTable(TableName.valueOf(tableName));
admin.deleteTable(TableName.valueOf(tableName));
}
HTableDescriptor table = new HTableDescriptor(TableName.valueOf(tableName));
for (String cf : columnFamily) {
table.addFamily(new HColumnDescriptor(cf));
}
admin.createTable(table);
System.out.println("create table successfully.");
return true;
} catch (IOException e) {
e.printStackTrace();
return false;
} finally {
try {
connection.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
public static void main(String[] args) throws Exception {
conf.set("hbase.zookeeper.quorum", "hadoop271.itversity.com");
conf.set("hbase.zookeeper.property.clientPort", "2181");
Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(TableName.valueOf("demo"));
Scan scan1 = new Scan();
ResultScanner scanner1 = table.getScanner(scan1);
for (Result res : scanner1) {
System.out.println(Bytes.toString(res.getRow()));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
}
scanner1.close();
Put put = new Put("3".getBytes());
put.addColumn("cf1".getBytes(), "column1".getBytes(), "value1".getBytes());
put.addColumn("cf1".getBytes(), "column2".getBytes(), "value2".getBytes());
table.put(put);
Get get = new Get("3".getBytes());
Result getResult = table.get(get);
System.out.println("Printing colunns for rowkey 3");
System.out.println(Bytes.toString(getResult.getValue("cf1".getBytes(), "column1".getBytes())));
System.out.println(Bytes.toString(getResult.getValue("cf1".getBytes(), "column2".getBytes())));
scanner1 = table.getScanner(scan1);
System.out.println("Before Delete");
for (Result res : scanner1) {
System.out.println(Bytes.toString(res.getRow()));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
}
scanner1.close();
Delete del = new Delete("3".getBytes());
table.delete(del);
System.out.println("After Delete");
scanner1 = table.getScanner(scan1);
for (Result res : scanner1) {
System.out.println(Bytes.toString(res.getRow()));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
}
scanner1.close();
table.close();
connection.close();
}
private static void cleanup() {
try {
table.close();
connection.close();
} catch (IOException e) {
System.out.println("Error: while clean up database connection " + e);
System.exit(-1);
}
}
@Override
protected void cleanup(
Reducer<Key_IMOAndRecordTime, TextArrayWritable, NullWritable, NullWritable>.Context
context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
connection.close();
}
public void contextDestroyed(ServletContextEvent event) {
// App Engine does not currently invoke this method.
try {
connection.close();
} catch (IOException io) {
sc.log("contextDestroyed ", io);
}
connection = null;
}
/**
* Confirm ImportTsv via data in online table.
*
* @param dataAvailable
*/
private static void validateTable(
Configuration conf,
TableName tableName,
String family,
int valueMultiplier,
boolean dataAvailable)
throws IOException {
LOG.debug("Validating table.");
Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName);
boolean verified = false;
long pause = conf.getLong("hbase.client.pause", 5 * 1000);
int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
for (int i = 0; i < numRetries; i++) {
try {
Scan scan = new Scan();
// Scan entire family.
scan.addFamily(Bytes.toBytes(family));
if (dataAvailable) {
ResultScanner resScanner = table.getScanner(scan);
for (Result res : resScanner) {
LOG.debug("Getting results " + res.size());
assertTrue(res.size() == 2);
List<Cell> kvs = res.listCells();
assertTrue(CellUtil.matchingRow(kvs.get(0), Bytes.toBytes("KEY")));
assertTrue(CellUtil.matchingRow(kvs.get(1), Bytes.toBytes("KEY")));
assertTrue(
CellUtil.matchingValue(kvs.get(0), Bytes.toBytes("VALUE" + valueMultiplier)));
assertTrue(
CellUtil.matchingValue(kvs.get(1), Bytes.toBytes("VALUE" + 2 * valueMultiplier)));
// Only one result set is expected, so let it loop.
verified = true;
}
} else {
ResultScanner resScanner = table.getScanner(scan);
Result[] next = resScanner.next(2);
assertEquals(0, next.length);
verified = true;
}
break;
} catch (NullPointerException e) {
// If here, a cell was empty. Presume its because updates came in
// after the scanner had been opened. Wait a while and retry.
}
try {
Thread.sleep(pause);
} catch (InterruptedException e) {
// continue
}
}
table.close();
connection.close();
assertTrue(verified);
}
public static void main(String[] args) throws IOException {
Configuration conf = HBaseClientHelper.loadDefaultConfiguration();
Connection connection = ConnectionFactory.createConnection(conf);
try {
Table table = connection.getTable(TableName.valueOf("testtable"));
try {
// 1 Put
Put p = new Put(Bytes.toBytes("row1"));
p.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val1"));
table.put(p);
// 2 Get
Get g = new Get(Bytes.toBytes("row1"));
Result r = table.get(g);
byte[] value = r.getValue(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"));
String valueStr = Bytes.toString(value);
System.out.println("GET: " + valueStr);
// 3 Scan
Scan s = new Scan();
s.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"));
ResultScanner scanner = table.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
System.out.println("Found row: " + rr);
}
// The other approach is to use a foreach loop. Scanners are
// iterable!
// for (Result rr : scanner) {
// System.out.println("Found row: " + rr);
// }
} finally {
scanner.close();
}
// Close your table and cluster connection.
} finally {
if (table != null) table.close();
}
} finally {
connection.close();
}
}
public static void main(String[] args) throws IOException {
Configuration conf = HBaseConfiguration.create();
HBaseHelper helper = HBaseHelper.getHelper(conf);
helper.dropTable("testtable");
helper.createTable("testtable", "colfam1");
Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(TableName.valueOf("testtable"));
List<Put> puts = new ArrayList<Put>();
// vv PutListErrorExample2
Put put1 = new Put(Bytes.toBytes("row1"));
put1.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val1"));
puts.add(put1);
Put put2 = new Put(Bytes.toBytes("row2"));
put2.addColumn(Bytes.toBytes("BOGUS"), Bytes.toBytes("qual1"), Bytes.toBytes("val2"));
puts.add(put2);
Put put3 = new Put(Bytes.toBytes("row2"));
put3.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual2"), Bytes.toBytes("val3"));
puts.add(put3);
/*[*/ Put put4 = new Put(Bytes.toBytes("row2"));
puts.add(
put4); /*]*/ // co PutListErrorExample2-1-AddErrorPut Add put with no content at all to
// list.
/*[*/ try {
/*]*/
table.put(puts);
/*[*/ } catch (Exception e) {
System.err.println("Error: " + e);
// table.flushCommits();
// todo: FIX!
/*]*/
// co PutListErrorExample2-2-Catch Catch local exception and commit queued updates.
/*[*/ } /*]*/
// ^^ PutListErrorExample2
table.close();
connection.close();
helper.close();
}
/**
* 添加多条记录到表中
*
* @param tableName
* @param puts
* @throws ParamIsNullException 参数为空
* @throws TableNotFoundException 表不存在/获取表连接失败
*/
public static void insertRecords(String tableName, List<Put> puts)
throws ParamIsNullException, TableNotFoundException {
if (null == tableName) {
throw new ParamIsNullException("tableName不能为空");
}
if (null == puts) {
throw new ParamIsNullException("Put不能为空");
}
logger.info(
"insertRecords begin......, tableName:{}, List<Put> size:{}", tableName, puts.size());
Connection conn = null;
HTable hTable = null;
try {
conn = ConnectionFactory.createConnection(conf);
hTable = (HTable) conn.getTable(TableName.valueOf(tableName));
if (null != hTable) {
hTable.put(puts);
logger.info("hTable.put(puts) ......");
} else {
throw new TableNotFoundException(tableName);
}
} catch (Exception e) {
logger.error("获取Hbase连接发生异常, errMsg:{}", e);
} finally {
if (null != hTable) {
try {
hTable.close();
} catch (IOException e) {
logger.error("HTable close exception, errMsg:{}", e);
}
}
if (null != conn) {
try {
conn.close();
} catch (IOException e) {
logger.error("Connection close exception, errMsg:{}", e);
}
}
}
logger.info("insertRecords end ......");
}
/**
* 创建一张表
*
* @param table
* @throws ParamIsNullException 参数为空
* @throws TableExistsException 表已存在
*/
public static void createTable(HTableDescriptor table)
throws ParamIsNullException, TableExistsException {
if (null == table) {
throw new ParamIsNullException("参数不能为空");
}
logger.info("create table begin... , table:{}", table.toString());
Connection conn = null;
HBaseAdmin admin = null;
String tableName = table.getNameAsString();
try {
logger.info("获取connection");
conn = ConnectionFactory.createConnection(conf);
logger.info("获取admin");
admin = (HBaseAdmin) conn.getAdmin();
/** 表已存在 */
if (admin.tableExists(Bytes.toBytes(tableName))) {
throw new TableExistsException(tableName);
}
logger.info("create...");
admin.createTable(table);
logger.info("table create success, tableName:{}", tableName);
} catch (IOException e) {
logger.error("table create fail, tableName:{}, errMsg:{}", tableName, e);
} finally {
if (null != admin) {
try {
admin.close();
} catch (IOException e) {
logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage());
}
}
if (null != conn) {
try {
conn.close();
} catch (IOException e) {
logger.error("Connection close exception, errMsg:{}", e.getMessage());
}
}
}
}
public static int getTable(Configuration config) throws IOException {
/** Connection to the cluster. A single connection shared by all application threads. */
Connection connection = null;
/** A lightweight handle to a specific table. Used from a single thread. */
Table table = null;
try {
// establish the connection to the cluster.
connection = ConnectionFactory.createConnection(config);
// retrieve a handle to the target table.
table = connection.getTable(TABLE_NAME1);
// describe the data we want to write.
Get g = new Get(Bytes.toBytes("row4"));
// send the data.
table.get(g);
} finally {
// close everything down
if (table != null) table.close();
if (connection != null) connection.close();
}
return 0;
}
public static void main(String[] args) throws Exception {
Configuration conf = HBaseConfiguration.create();
IntegrationTestingUtility.setUseDistributedCluster(conf);
IntegrationTestDDLMasterFailover masterFailover = new IntegrationTestDDLMasterFailover();
Connection connection = null;
int ret = 1;
try {
// Initialize connection once, then pass to Actions
LOG.debug("Setting up connection ...");
connection = ConnectionFactory.createConnection(conf);
masterFailover.setConnection(connection);
ret = ToolRunner.run(conf, masterFailover, args);
} catch (IOException e) {
LOG.fatal("Failed to establish connection. Aborting test ...", e);
} finally {
connection = masterFailover.getConnection();
if (connection != null) {
connection.close();
}
System.exit(ret);
}
}
/**
* Cleanup any state for this DB. Called once per DB instance; there is one DB instance per client
* thread.
*/
@Override
public void cleanup() throws DBException {
// Get the measurements instance as this is the only client that should
// count clean up time like an update if client-side buffering is
// enabled.
Measurements measurements = Measurements.getMeasurements();
try {
long st = System.nanoTime();
if (bufferedMutator != null) {
bufferedMutator.close();
}
if (currentTable != null) {
currentTable.close();
}
long en = System.nanoTime();
final String type = clientSideBuffering ? "UPDATE" : "CLEANUP";
measurements.measure(type, (int) ((en - st) / 1000));
connection.close();
} catch (IOException e) {
throw new DBException(e);
}
}
/**
* 添加一个列族
*
* @param tableName
* @param family 列族
* @throws ParamIsNullException 参数为空
* @throws TableNotFoundException 表不存在/获取表连接失败
*/
public static void addColumnFamily(String tableName, HColumnDescriptor family)
throws ParamIsNullException, TableNotFoundException {
if (null == tableName) {
throw new ParamIsNullException("tableName不能为空");
}
if (null == family) {
throw new ParamIsNullException("HColumnDescriptor不能为空");
}
Connection conn = null;
HBaseAdmin admin = null;
try {
conn = ConnectionFactory.createConnection(conf);
admin = (HBaseAdmin) conn.getAdmin();
/** 表不存在 */
if (!admin.tableExists(Bytes.toBytes(tableName))) {
throw new TableNotFoundException(tableName);
}
HTableDescriptor table = admin.getTableDescriptor(Bytes.toBytes(tableName));
table.addFamily(family);
} catch (IOException e) {
logger.error("获取Hbase连接发生异常, errMsg:{}", e.getMessage());
} finally {
if (null != admin) {
try {
admin.close();
} catch (IOException e) {
logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage());
}
}
if (null != conn) {
try {
conn.close();
} catch (IOException e) {
logger.error("Connection close exception, errMsg:{}", e.getMessage());
}
}
}
}
@Override
public void cleanUpCluster() throws Exception {
if (!keepObjectsAtTheEnd) {
Admin admin = util.getAdmin();
admin.disableTables("ittable-\\d+");
admin.deleteTables("ittable-\\d+");
NamespaceDescriptor[] nsds = admin.listNamespaceDescriptors();
for (NamespaceDescriptor nsd : nsds) {
if (nsd.getName().matches("itnamespace\\d+")) {
LOG.info("Removing namespace=" + nsd.getName());
admin.deleteNamespace(nsd.getName());
}
}
}
enabledTables.clear();
disabledTables.clear();
deletedTables.clear();
namespaceMap.clear();
Connection connection = getConnection();
connection.close();
super.cleanUpCluster();
}
/**
* 添加一条记录到表中
*
* @param tableName
* @param put
* @throws ParamIsNullException 参数为空
* @throws IOException
*/
public static void insertOneRecord(String tableName, Put put)
throws ParamIsNullException, IOException {
if (null == tableName) {
throw new ParamIsNullException("tableName不能为空");
}
if (null == put) {
throw new ParamIsNullException("Put不能为空");
}
logger.info("insertOneRecord ...");
Connection conn = null;
HTable hTable = null;
try {
conn = ConnectionFactory.createConnection(conf);
hTable = (HTable) conn.getTable(TableName.valueOf(tableName));
if (null != hTable) {
hTable.put(put);
} else {
throw new TableNotFoundException(tableName);
}
} finally {
if (null != hTable) {
try {
hTable.close();
} catch (IOException e) {
logger.error("HTable close exception, errMsg:{}", e.getMessage());
}
}
if (null != conn) {
try {
conn.close();
} catch (IOException e) {
logger.error("Connection close exception, errMsg:{}", e.getMessage());
}
}
}
}
@SuppressWarnings("deprecation")
public static int putTable(Configuration config) throws IOException {
/** Connection to the cluster. A single connection shared by all application threads. */
Connection connection = null;
/** A lightweight handle to a specific table. Used from a single thread. */
Table table = null;
try {
// establish the connection to the cluster.
connection = ConnectionFactory.createConnection(config);
// retrieve a handle to the target table.
table = connection.getTable(TABLE_NAME1);
// describe the data we want to write.
Put p = new Put(Bytes.toBytes("row4"));
p.add(CF, Bytes.toBytes("b"), Bytes.toBytes("datos agregados desde java"));
p.add(CF, Bytes.toBytes("qual"), Bytes.toBytes("segunda columna"));
// send the data.
table.put(p);
} finally {
// close everything down
if (table != null) table.close();
if (connection != null) connection.close();
}
return 0;
}
public void convertH()
throws SQLException, IOException, ClassNotFoundException, IllegalAccessException,
InstantiationException {
/** create HBase connection */
conf_h = HBaseConfiguration.create();
conf_h.set("hbase.zookeeper.quorum", "localhost");
conf_h.set("hbase.zookeeper.property.clientPort", "2181");
try {
conn_h = ConnectionFactory.createConnection(conf_h);
} catch (IOException e) {
e.printStackTrace();
}
/** create Mysql connection */
try {
Class.forName("com.mysql.jdbc.Driver").newInstance();
// con_sql = DriverManager.getConnection("jdbc:mysql:///retail_db", "root", "cloudera");
conn_sql = DriverManager.getConnection("jdbc:mysql:///retail_db", "root", "cloudera");
if (!conn_sql.isClosed()) {
System.out.println("Successfully connected to MySQL server...");
}
} catch (Exception e) {
System.err.println(e.getMessage());
}
/** create table and family */
Admin admin = conn_h.getAdmin();
HTableDescriptor tableDesc = new HTableDescriptor(tableName_retail);
// create Family Name
String[] familyName = {"order_items", "orders", "products"};
for (String name : familyName) {
HColumnDescriptor colDesc = new HColumnDescriptor(name);
tableDesc.addFamily(colDesc);
}
admin.createTable(tableDesc);
/** query from mysql */
if (conn_sql != null) {
Statement stat = conn_sql.createStatement();
stat.executeQuery(
"select * from products "
+ "inner join order_items on product_id = order_item_product_id "
+ "inner join orders on order_item_order_id = order_id ");
ResultSet rs = stat.getResultSet();
while (rs.next()) {
// order_items
String rowKey = rs.getString("order_item_id");
Put put = new Put(Bytes.toBytes(rowKey)); // new Put(rowKey)
put.addColumn(
Bytes.toBytes("order_items"), Bytes.toBytes("order_item_id"), Bytes.toBytes(rowKey));
String order_item_order_id = rs.getString("order_item_order_id");
put.addColumn(
Bytes.toBytes("order_items"),
Bytes.toBytes("order_item_order_id"),
Bytes.toBytes(order_item_order_id));
String order_item_product_id = rs.getString("order_item_product_id");
put.addColumn(
Bytes.toBytes("order_items"),
Bytes.toBytes("order_item_product_id"),
Bytes.toBytes(order_item_product_id));
String order_item_quantity = rs.getString("order_item_quantity");
put.addColumn(
Bytes.toBytes("order_items"),
Bytes.toBytes("order_item_quantity"),
Bytes.toBytes(order_item_quantity));
String order_item_subtotal = rs.getString("order_item_subtotal");
put.addColumn(
Bytes.toBytes("order_items"),
Bytes.toBytes("order_item_subtotal"),
Bytes.toBytes(order_item_subtotal));
String order_item_product_price = rs.getString("order_item_product_price");
put.addColumn(
Bytes.toBytes("order_items"),
Bytes.toBytes("order_item_product_price"),
Bytes.toBytes(order_item_product_price));
// orders
String order_id = rs.getString("order_id");
put.addColumn(Bytes.toBytes("orders"), Bytes.toBytes("order_id"), Bytes.toBytes(order_id));
String order_date = rs.getString("order_date");
put.addColumn(
Bytes.toBytes("orders"), Bytes.toBytes("order_date"), Bytes.toBytes(order_date));
String order_customer_id = rs.getString("order_customer_id");
put.addColumn(
Bytes.toBytes("orders"),
Bytes.toBytes("order_customer_id"),
Bytes.toBytes(order_customer_id));
String order_status = rs.getString("order_status");
put.addColumn(
Bytes.toBytes("orders"), Bytes.toBytes("order_status"), Bytes.toBytes(order_status));
// products
String product_id = rs.getString("product_id");
put.addColumn(
Bytes.toBytes("products"), Bytes.toBytes("product_id"), Bytes.toBytes(product_id));
String product_category_id = rs.getString("product_category_id");
put.addColumn(
Bytes.toBytes("products"),
Bytes.toBytes("product_category_id"),
Bytes.toBytes(product_category_id));
String product_name = rs.getString("product_name");
put.addColumn(
Bytes.toBytes("products"), Bytes.toBytes("product_name"), Bytes.toBytes(product_name));
String product_description = rs.getString("product_description");
put.addColumn(
Bytes.toBytes("products"),
Bytes.toBytes("product_description"),
Bytes.toBytes(product_description));
String product_price = rs.getString("product_price");
put.addColumn(
Bytes.toBytes("products"),
Bytes.toBytes("product_price"),
Bytes.toBytes(product_price));
String product_image = rs.getString("product_image");
put.addColumn(
Bytes.toBytes("products"),
Bytes.toBytes("product_image"),
Bytes.toBytes(product_image));
// put in table
Table table = conn_h.getTable(tableName_retail);
table.put(put);
table.close();
}
rs.close();
stat.close();
}
conn_h.close();
conn_sql.close();
}
@After
public void close() throws IOException {
deleteTable();
admin.close();
conn.close();
}
public static boolean scanTable(Configuration config) throws IOException, URISyntaxException {
/** Connection to the cluster. A single connection shared by all application threads. */
Connection connection = null;
/** A lightweight handle to a specific table. Used from a single thread. */
Table table = null;
Scan scan = null;
ResultScanner scanner = null;
// String[] columns = null;
try {
// establish the connection to the cluster.
connection = ConnectionFactory.createConnection(config);
// retrieve a handle to the target table.
table = connection.getTable(TABLE_NAME1);
// Instantiating the Scan class
scan = new Scan();
// Recommended tuning Ch12 HBase Performance Tuning
scan.setCaching(5000); // Defined rows returned from server during next()
scan.setCacheBlocks(false); // Disable server side cache
// Scanning the required columns
scan.addFamily(CF);
// Getting the scan result
scanner = table.getScanner(scan);
System.out.println("Rows founded");
StringBuffer line = new StringBuffer();
int counter = 0;
// Reading values from scan result
for (Result result = scanner.next(); result != null; result = scanner.next()) {
columns = getColumnsInColumnFamily(result, CF);
// clean line
// line.delete(0, line.length());
line.append(Bytes.toString(result.getRow()));
for (String column : columns) {
line.append("^" + Bytes.toString(result.getValue(CF, Bytes.toBytes(column))));
}
line.append("\n");
counter++;
System.out.println("Linea: " + counter);
// line.append(""
// + Bytes.toString(result.getRow()) + ","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("account"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("amount"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("aut"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("balance"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("card"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("commerce"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("commerceData"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("date"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("f"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("f2"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("f3"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("f4"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("hour"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("m"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("money"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("ms"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("r"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("ref"))) +","
// + Bytes.toString(result.getValue(CF, Bytes.toBytes("trxcd"))) +"\n"
// );
// System.out.println(Bytes.toString(result.getRow()) +","+
// Bytes.toString(result.getValue(CF, qualifier)));
}
DistributedFileSystem hdfs = new DistributedFileSystem();
hdfs.initialize(new URI("hdfs://quickstart.cloudera:8020"), config);
Path homeDir = hdfs.getHomeDirectory();
// Print the home directory
System.out.println("Home folder HDFS-" + homeDir);
Path newFilePath = new Path(homeDir + "/datatest/" + TABLE_NAME + "/file.csv");
byte[] byt = line.toString().getBytes();
FSDataOutputStream fsOutStream = hdfs.create(newFilePath);
fsOutStream.write(byt);
fsOutStream.close();
hdfs.close();
System.out.println("File exported to " + homeDir + "/datatest/" + TABLE_NAME + "/file.csv");
} finally {
// close everything down
if (table != null) table.close();
if (connection != null) connection.close();
if (scanner != null) scanner.close();
}
return true;
}
