@Override
void perform() throws IOException {
Admin admin = connection.getAdmin();
try {
HTableDescriptor htd = createTableDesc();
TableName tableName = htd.getTableName();
if (admin.tableExists(tableName)) {
return;
}
String numRegionKey = String.format(NUM_REGIONS_KEY, this.getClass().getSimpleName());
numRegions = getConf().getInt(numRegionKey, DEFAULT_NUM_REGIONS);
byte[] startKey = Bytes.toBytes("row-0000000000");
byte[] endKey = Bytes.toBytes("row-" + Integer.MAX_VALUE);
LOG.info("Creating table:" + htd);
admin.createTable(htd, startKey, endKey, numRegions);
Assert.assertTrue("Table: " + htd + " was not created", admin.tableExists(tableName));
HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
enabledTables.put(tableName, freshTableDesc);
LOG.info("Created table:" + freshTableDesc);
} catch (Exception e) {
LOG.warn("Caught exception in action: " + this.getClass());
throw e;
} finally {
admin.close();
}
verifyTables();
}
@Test
public void testTableCreateAndDeletePB() throws IOException, JAXBException {
String schemaPath = "/" + TABLE2 + "/schema";
TableSchemaModel model;
Response response;
Admin admin = TEST_UTIL.getHBaseAdmin();
assertFalse(admin.tableExists(TableName.valueOf(TABLE2)));
// create the table
model = testTableSchemaModel.buildTestModel(TABLE2);
testTableSchemaModel.checkModel(model, TABLE2);
response = client.put(schemaPath, Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 201);
// recall the same put operation but in read-only mode
conf.set("hbase.rest.readonly", "true");
response = client.put(schemaPath, Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 403);
// retrieve the schema and validate it
response = client.get(schemaPath, Constants.MIMETYPE_PROTOBUF);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
model = new TableSchemaModel();
model.getObjectFromMessage(response.getBody());
testTableSchemaModel.checkModel(model, TABLE2);
// retrieve the schema and validate it with alternate pbuf type
response = client.get(schemaPath, Constants.MIMETYPE_PROTOBUF_IETF);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type"));
model = new TableSchemaModel();
model.getObjectFromMessage(response.getBody());
testTableSchemaModel.checkModel(model, TABLE2);
// test delete schema operation is forbidden in read-only mode
response = client.delete(schemaPath);
assertEquals(response.getCode(), 403);
// return read-only setting back to default
conf.set("hbase.rest.readonly", "false");
// delete the table and make sure HBase concurs
response = client.delete(schemaPath);
assertEquals(response.getCode(), 200);
assertFalse(admin.tableExists(TableName.valueOf(TABLE2)));
}
private static void createTable() throws Exception {
try {
Configuration configuration = HBaseConfiguration.create();
HBaseAdmin.checkHBaseAvailable(configuration);
Connection connection = ConnectionFactory.createConnection(configuration);
// Instantiating HbaseAdmin class
Admin admin = connection.getAdmin();
// Instantiating table descriptor class
HTableDescriptor stockTableDesc =
new HTableDescriptor(TableName.valueOf(Constants.STOCK_DATES_TABLE));
// Adding column families to table descriptor
HColumnDescriptor stock_0414 = new HColumnDescriptor(Constants.STOCK_DATES_CF);
stockTableDesc.addFamily(stock_0414);
// Execute the table through admin
if (!admin.tableExists(stockTableDesc.getTableName())) {
admin.createTable(stockTableDesc);
System.out.println("Stock table created !!!");
}
// Load hbase-site.xml
HBaseConfiguration.addHbaseResources(configuration);
} catch (ServiceException e) {
log.error("Error occurred while creating HBase tables", e);
throw new Exception("Error occurred while creating HBase tables", e);
}
}
private void deleteTable() throws IOException {
admin.disableTable(tn);
admin.deleteTable(tn);
if (!admin.tableExists(tn)) {
System.out.println(tableName + " is delete !");
}
}
public static boolean createTableOrOverwrite(String tableName, String[] columnFamily) {
if (tableName == null && columnFamily == null) {
return false;
}
Connection connection = null;
try {
connection = ConnectionFactory.createConnection(config);
Admin admin = connection.getAdmin();
if (admin.tableExists(TableName.valueOf(tableName))) {
admin.disableTable(TableName.valueOf(tableName));
admin.deleteTable(TableName.valueOf(tableName));
}
HTableDescriptor table = new HTableDescriptor(TableName.valueOf(tableName));
for (String cf : columnFamily) {
table.addFamily(new HColumnDescriptor(cf));
}
admin.createTable(table);
System.out.println("create table successfully.");
return true;
} catch (IOException e) {
e.printStackTrace();
return false;
} finally {
try {
connection.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
@Test
public void testTableCreateAndDeleteXML() throws IOException, JAXBException {
String schemaPath = "/" + TABLE1 + "/schema";
TableSchemaModel model;
Response response;
Admin admin = TEST_UTIL.getHBaseAdmin();
assertFalse(admin.tableExists(TableName.valueOf(TABLE1)));
// create the table
model = testTableSchemaModel.buildTestModel(TABLE1);
testTableSchemaModel.checkModel(model, TABLE1);
response = client.put(schemaPath, Constants.MIMETYPE_XML, toXML(model));
assertEquals(response.getCode(), 201);
// recall the same put operation but in read-only mode
conf.set("hbase.rest.readonly", "true");
response = client.put(schemaPath, Constants.MIMETYPE_XML, toXML(model));
assertEquals(response.getCode(), 403);
// retrieve the schema and validate it
response = client.get(schemaPath, Constants.MIMETYPE_XML);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
model = fromXML(response.getBody());
testTableSchemaModel.checkModel(model, TABLE1);
// with json retrieve the schema and validate it
response = client.get(schemaPath, Constants.MIMETYPE_JSON);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_JSON, response.getHeader("content-type"));
model = testTableSchemaModel.fromJSON(Bytes.toString(response.getBody()));
testTableSchemaModel.checkModel(model, TABLE1);
// test delete schema operation is forbidden in read-only mode
response = client.delete(schemaPath);
assertEquals(response.getCode(), 403);
// return read-only setting back to default
conf.set("hbase.rest.readonly", "false");
// delete the table and make sure HBase concurs
response = client.delete(schemaPath);
assertEquals(response.getCode(), 200);
assertFalse(admin.tableExists(TableName.valueOf(TABLE1)));
}
public boolean tableExists(String tableName) {
try {
Admin admin = connection.getAdmin();
return admin.tableExists(TableName.valueOf(tableName));
} catch (IOException e) {
e.printStackTrace();
}
return false;
}
/**
* Disable a table's replication switch.
*
* @param tableName name of the table
* @throws IOException if a remote or network exception occurs
*/
public void disableTableRep(final TableName tableName) throws IOException {
if (tableName == null) {
throw new IllegalArgumentException("Table name is null");
}
try (Admin admin = this.connection.getAdmin()) {
if (!admin.tableExists(tableName)) {
throw new TableNotFoundException(
"Table '" + tableName.getNamespaceAsString() + "' does not exists.");
}
}
setTableRep(tableName, false);
}
@After
public void tearDown() throws IOException {
LOG.info("Cleaning up after test.");
Admin admin = util.getHBaseAdmin();
if (admin.tableExists(TABLE_NAME)) {
if (admin.isTableEnabled(TABLE_NAME)) admin.disableTable(TABLE_NAME);
admin.deleteTable(TABLE_NAME);
}
LOG.info("Restoring cluster.");
util.restoreCluster();
LOG.info("Cluster restored.");
}
@Test
public void createTableInSystemNamespace() throws Exception {
TableName tableName = TableName.valueOf("hbase:createTableInSystemNamespace");
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor colDesc = new HColumnDescriptor("cf1");
desc.addFamily(colDesc);
admin.createTable(desc);
assertEquals(0, admin.listTables().length);
assertTrue(admin.tableExists(tableName));
admin.disableTable(desc.getTableName());
admin.deleteTable(desc.getTableName());
}
/**
* Enable a table's replication switch.
*
* @param tableName name of the table
* @throws IOException if a remote or network exception occurs
*/
public void enableTableRep(final TableName tableName) throws IOException {
if (tableName == null) {
throw new IllegalArgumentException("Table name cannot be null");
}
try (Admin admin = this.connection.getAdmin()) {
if (!admin.tableExists(tableName)) {
throw new TableNotFoundException(
"Table '" + tableName.getNameAsString() + "' does not exists.");
}
}
byte[][] splits = getTableSplitRowKeys(tableName);
checkAndSyncTableDescToPeers(tableName, splits);
setTableRep(tableName, true);
}
public boolean removeTable(String tableName) {
try {
Admin admin = connection.getAdmin();
TableName t = TableName.valueOf(tableName);
if (admin.tableExists(t)) {
admin.disableTable(t);
admin.deleteTable(t);
return true;
}
} catch (IOException e) {
e.printStackTrace();
}
return false;
}
@Before
public void setUp() throws Exception {
LOG.info(String.format("Initializing cluster with %d region servers.", REGION_SERVER_COUNT));
util.initializeCluster(REGION_SERVER_COUNT);
LOG.info("Cluster initialized");
Admin admin = util.getHBaseAdmin();
if (admin.tableExists(TABLE_NAME)) {
LOG.info(String.format("Deleting existing table %s.", TABLE_NAME));
if (admin.isTableEnabled(TABLE_NAME)) admin.disableTable(TABLE_NAME);
admin.deleteTable(TABLE_NAME);
LOG.info(String.format("Existing table %s deleted.", TABLE_NAME));
}
LOG.info("Cluster ready");
}
/** 初始化数据 */
public void initData() {
try {
if (admin.tableExists(tn)) {
System.out.println(tableName + " exists !");
} else {
HTableDescriptor desc = new HTableDescriptor(tn);
desc.addFamily(new HColumnDescriptor(columnFamily1));
desc.addFamily(new HColumnDescriptor(columnFamily2));
admin.createTable(desc);
System.out.println("create " + tableName + " success !");
addData();
}
} catch (Exception e) {
e.printStackTrace();
}
}
protected void verifyTables() throws IOException {
Connection connection = getConnection();
Admin admin = connection.getAdmin();
// iterating concurrent map
for (TableName tableName : enabledTables.keySet()) {
Assert.assertTrue(
"Table: " + tableName + " in enabledTables is not enabled",
admin.isTableEnabled(tableName));
}
for (TableName tableName : disabledTables.keySet()) {
Assert.assertTrue(
"Table: " + tableName + " in disabledTables is not disabled",
admin.isTableDisabled(tableName));
}
for (TableName tableName : deletedTables.keySet()) {
Assert.assertFalse(
"Table: " + tableName + " in deletedTables is not deleted", admin.tableExists(tableName));
}
admin.close();
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
conf = TEST_UTIL.getConfiguration();
TEST_UTIL.startMiniCluster();
REST_TEST_UTIL.startServletContainer(conf);
client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort()));
context =
JAXBContext.newInstance(
CellModel.class, CellSetModel.class, RowModel.class, ScannerModel.class);
marshaller = context.createMarshaller();
unmarshaller = context.createUnmarshaller();
Admin admin = TEST_UTIL.getHBaseAdmin();
if (admin.tableExists(TABLE)) {
return;
}
HTableDescriptor htd = new HTableDescriptor(TABLE);
htd.addFamily(new HColumnDescriptor(CFA));
htd.addFamily(new HColumnDescriptor(CFB));
admin.createTable(htd);
expectedRows1 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_1, 1.0);
expectedRows2 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_2, 0.5);
}
/**
* Connect to peer and check the table descriptor on peer:
*
* <ol>
* <li>Create the same table on peer when not exist.
* <li>Throw exception if the table exists on peer cluster but descriptors are not same.
* </ol>
*
* @param tableName name of the table to sync to the peer
* @param splits table split keys
* @throws IOException
*/
private void checkAndSyncTableDescToPeers(final TableName tableName, final byte[][] splits)
throws IOException {
List<ReplicationPeer> repPeers = listValidReplicationPeers();
if (repPeers == null || repPeers.size() <= 0) {
throw new IllegalArgumentException("Found no peer cluster for replication.");
}
for (ReplicationPeer repPeer : repPeers) {
Configuration peerConf = repPeer.getConfiguration();
HTableDescriptor htd = null;
try (Connection conn = ConnectionFactory.createConnection(peerConf);
Admin admin = this.connection.getAdmin();
Admin repHBaseAdmin = conn.getAdmin()) {
htd = admin.getTableDescriptor(tableName);
HTableDescriptor peerHtd = null;
if (!repHBaseAdmin.tableExists(tableName)) {
repHBaseAdmin.createTable(htd, splits);
} else {
peerHtd = repHBaseAdmin.getTableDescriptor(tableName);
if (peerHtd == null) {
throw new IllegalArgumentException(
"Failed to get table descriptor for table "
+ tableName.getNameAsString()
+ " from peer cluster "
+ repPeer.getId());
} else if (!peerHtd.equals(htd)) {
throw new IllegalArgumentException(
"Table "
+ tableName.getNameAsString()
+ " exists in peer cluster "
+ repPeer.getId()
+ ", but the table descriptors are not same when comapred with source cluster."
+ " Thus can not enable the table's replication switch.");
}
}
}
}
}
@Override
void perform() throws IOException {
HTableDescriptor selected = selectTable(disabledTables);
if (selected == null) {
return;
}
Admin admin = connection.getAdmin();
try {
TableName tableName = selected.getTableName();
LOG.info("Deleting table :" + selected);
admin.deleteTable(tableName);
Assert.assertFalse("Table: " + selected + " was not deleted", admin.tableExists(tableName));
deletedTables.put(tableName, selected);
LOG.info("Deleted table :" + selected);
} catch (Exception e) {
LOG.warn("Caught exception in action: " + this.getClass());
throw e;
} finally {
admin.close();
}
verifyTables();
}
public static List<KeyValue> getAllTrainInfo(Configuration config, String date) {
List<KeyValue> result = new ArrayList<>();
String strJson = null;
BufferedWriter writer = null;
Table table = null;
try (Connection connect = ConnectionFactory.createConnection(config);
Admin admin = connect.getAdmin()) {
TableName tablename = TableName.valueOf(TABLE_NAME);
if (!admin.tableExists(tablename)) {
System.out.println("Table does not exist.");
return null;
}
table = connect.getTable(tablename);
Put put = null;
String start = null;
String end = null;
writer = new BufferedWriter(new FileWriter(new File(strConfig), true));
for (KeyValue item : lstAllProcessStation) {
start = (String) item.getKey();
end = (String) item.getValue();
try {
try {
Thread.sleep(200);
} catch (InterruptedException e1) {
e1.printStackTrace();
}
System.out.println("process : " + start + ":" + end);
strJson = getFromAPIX(mapStationCode.get(start), mapStationCode.get(end), date);
writer.write(start + ":" + end);
writer.newLine();
} catch (Exception e) {
System.out.println(start + ":" + end + "error");
e.printStackTrace();
break;
}
JSONObject jo = new JSONObject(strJson);
if (jo.has("httpstatus") && (jo.getInt("httpstatus") == 200)) {
JSONObject joData = jo.getJSONObject("data");
if (joData.has("flag") && joData.getBoolean("flag")) {
result.add(new DefaultKeyValue(start, end));
// 插入到hbase
String rowkey = start + ":" + end;
put = new Put(rowkey.getBytes());
put.addColumn(
CF_JSON.getBytes(), "json".getBytes(), joData.toString().getBytes("utf-8"));
table.put(put);
System.out.println("start " + start + "\t end " + end + "\t has ticket");
}
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (writer != null) {
try {
writer.flush();
writer.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (table != null) {
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return result;
}
