@Override
void perform() throws IOException {
Admin admin = connection.getAdmin();
try {
HTableDescriptor htd = createTableDesc();
TableName tableName = htd.getTableName();
if (admin.tableExists(tableName)) {
return;
}
String numRegionKey = String.format(NUM_REGIONS_KEY, this.getClass().getSimpleName());
numRegions = getConf().getInt(numRegionKey, DEFAULT_NUM_REGIONS);
byte[] startKey = Bytes.toBytes("row-0000000000");
byte[] endKey = Bytes.toBytes("row-" + Integer.MAX_VALUE);
LOG.info("Creating table:" + htd);
admin.createTable(htd, startKey, endKey, numRegions);
Assert.assertTrue("Table: " + htd + " was not created", admin.tableExists(tableName));
HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
enabledTables.put(tableName, freshTableDesc);
LOG.info("Created table:" + freshTableDesc);
} catch (Exception e) {
LOG.warn("Caught exception in action: " + this.getClass());
throw e;
} finally {
admin.close();
}
verifyTables();
}
private static void createTable() throws Exception {
try {
Configuration configuration = HBaseConfiguration.create();
HBaseAdmin.checkHBaseAvailable(configuration);
Connection connection = ConnectionFactory.createConnection(configuration);
// Instantiating HbaseAdmin class
Admin admin = connection.getAdmin();
// Instantiating table descriptor class
HTableDescriptor stockTableDesc =
new HTableDescriptor(TableName.valueOf(Constants.STOCK_DATES_TABLE));
// Adding column families to table descriptor
HColumnDescriptor stock_0414 = new HColumnDescriptor(Constants.STOCK_DATES_CF);
stockTableDesc.addFamily(stock_0414);
// Execute the table through admin
if (!admin.tableExists(stockTableDesc.getTableName())) {
admin.createTable(stockTableDesc);
System.out.println("Stock table created !!!");
}
// Load hbase-site.xml
HBaseConfiguration.addHbaseResources(configuration);
} catch (ServiceException e) {
log.error("Error occurred while creating HBase tables", e);
throw new Exception("Error occurred while creating HBase tables", e);
}
}
public boolean tableExists(String tableName) {
try {
Admin admin = connection.getAdmin();
return admin.tableExists(TableName.valueOf(tableName));
} catch (IOException e) {
e.printStackTrace();
}
return false;
}
public boolean removeTable(String tableName) {
try {
Admin admin = connection.getAdmin();
TableName t = TableName.valueOf(tableName);
if (admin.tableExists(t)) {
admin.disableTable(t);
admin.deleteTable(t);
return true;
}
} catch (IOException e) {
e.printStackTrace();
}
return false;
}
protected void verifyTables() throws IOException {
Connection connection = getConnection();
Admin admin = connection.getAdmin();
// iterating concurrent map
for (TableName tableName : enabledTables.keySet()) {
Assert.assertTrue(
"Table: " + tableName + " in enabledTables is not enabled",
admin.isTableEnabled(tableName));
}
for (TableName tableName : disabledTables.keySet()) {
Assert.assertTrue(
"Table: " + tableName + " in disabledTables is not disabled",
admin.isTableDisabled(tableName));
}
for (TableName tableName : deletedTables.keySet()) {
Assert.assertFalse(
"Table: " + tableName + " in deletedTables is not deleted", admin.tableExists(tableName));
}
admin.close();
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
conf = HBaseConfiguration.create();
conf.setStrings(
CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, TestCoprocessor.class.getName());
util = new HBaseTestingUtility(conf);
util.startMiniCluster();
Admin admin = util.getAdmin();
if (admin.tableExists(tableName)) {
if (admin.isTableEnabled(tableName)) {
admin.disableTable(tableName);
}
admin.deleteTable(tableName);
}
Table ht = util.createTable(tableName, new byte[][] {dummy, test});
Put p = new Put(row1);
p.addColumn(dummy, dummy, dummy);
ht.put(p);
}
@Override
void perform() throws IOException {
HTableDescriptor selected = selectTable(disabledTables);
if (selected == null) {
return;
}
Admin admin = connection.getAdmin();
try {
TableName tableName = selected.getTableName();
LOG.info("Deleting table :" + selected);
admin.deleteTable(tableName);
Assert.assertFalse("Table: " + selected + " was not deleted", admin.tableExists(tableName));
deletedTables.put(tableName, selected);
LOG.info("Deleted table :" + selected);
} catch (Exception e) {
LOG.warn("Caught exception in action: " + this.getClass());
throw e;
} finally {
admin.close();
}
verifyTables();
}
void splitTest(
byte[] splitPoint, byte[][] familyNames, int[] rowCounts, int numVersions, int blockSize)
throws Exception {
TableName tableName = TableName.valueOf("testForceSplit");
StringBuilder sb = new StringBuilder();
// Add tail to String so can see better in logs where a test is running.
for (int i = 0; i < rowCounts.length; i++) {
sb.append("_").append(Integer.toString(rowCounts[i]));
}
assertFalse(admin.tableExists(tableName));
final HTable table = TEST_UTIL.createTable(tableName, familyNames, numVersions, blockSize);
int rowCount = 0;
byte[] q = new byte[0];
// insert rows into column families. The number of rows that have values
// in a specific column family is decided by rowCounts[familyIndex]
for (int index = 0; index < familyNames.length; index++) {
ArrayList<Put> puts = new ArrayList<Put>(rowCounts[index]);
for (int i = 0; i < rowCounts[index]; i++) {
byte[] k = Bytes.toBytes(i);
Put put = new Put(k);
put.add(familyNames[index], q, k);
puts.add(put);
}
table.put(puts);
if (rowCount < rowCounts[index]) {
rowCount = rowCounts[index];
}
}
// get the initial layout (should just be one region)
Map<HRegionInfo, ServerName> m = table.getRegionLocations();
LOG.info("Initial regions (" + m.size() + "): " + m);
assertTrue(m.size() == 1);
// Verify row count
Scan scan = new Scan();
ResultScanner scanner = table.getScanner(scan);
int rows = 0;
for (@SuppressWarnings("unused") Result result : scanner) {
rows++;
}
scanner.close();
assertEquals(rowCount, rows);
// Have an outstanding scan going on to make sure we can scan over splits.
scan = new Scan();
scanner = table.getScanner(scan);
// Scan first row so we are into first region before split happens.
scanner.next();
// Split the table
this.admin.split(tableName, splitPoint);
final AtomicInteger count = new AtomicInteger(0);
Thread t =
new Thread("CheckForSplit") {
@Override
public void run() {
for (int i = 0; i < 45; i++) {
try {
sleep(1000);
} catch (InterruptedException e) {
continue;
}
// check again table = new HTable(conf, tableName);
Map<HRegionInfo, ServerName> regions = null;
try {
regions = table.getRegionLocations();
} catch (IOException e) {
e.printStackTrace();
}
if (regions == null) continue;
count.set(regions.size());
if (count.get() >= 2) {
LOG.info("Found: " + regions);
break;
}
LOG.debug("Cycle waiting on split");
}
LOG.debug("CheckForSplit thread exited, current region count: " + count.get());
}
};
t.setPriority(Thread.NORM_PRIORITY - 2);
t.start();
t.join();
// Verify row count
rows = 1; // We counted one row above.
for (@SuppressWarnings("unused") Result result : scanner) {
rows++;
if (rows > rowCount) {
scanner.close();
assertTrue("Scanned more than expected (" + rowCount + ")", false);
}
}
scanner.close();
assertEquals(rowCount, rows);
Map<HRegionInfo, ServerName> regions = null;
try {
regions = table.getRegionLocations();
} catch (IOException e) {
e.printStackTrace();
}
assertEquals(2, regions.size());
Set<HRegionInfo> hRegionInfos = regions.keySet();
HRegionInfo[] r = hRegionInfos.toArray(new HRegionInfo[hRegionInfos.size()]);
if (splitPoint != null) {
// make sure the split point matches our explicit configuration
assertEquals(Bytes.toString(splitPoint), Bytes.toString(r[0].getEndKey()));
assertEquals(Bytes.toString(splitPoint), Bytes.toString(r[1].getStartKey()));
LOG.debug("Properly split on " + Bytes.toString(splitPoint));
} else {
if (familyNames.length > 1) {
int splitKey = Bytes.toInt(r[0].getEndKey());
// check if splitKey is based on the largest column family
// in terms of it store size
int deltaForLargestFamily = Math.abs(rowCount / 2 - splitKey);
LOG.debug(
"SplitKey="
+ splitKey
+ "&deltaForLargestFamily="
+ deltaForLargestFamily
+ ", r="
+ r[0]);
for (int index = 0; index < familyNames.length; index++) {
int delta = Math.abs(rowCounts[index] / 2 - splitKey);
if (delta < deltaForLargestFamily) {
assertTrue(
"Delta "
+ delta
+ " for family "
+ index
+ " should be at least deltaForLargestFamily "
+ deltaForLargestFamily,
false);
}
}
}
}
TEST_UTIL.deleteTable(tableName);
table.close();
}
public static List<KeyValue> getAllTrainInfo(Configuration config, String date) {
List<KeyValue> result = new ArrayList<>();
String strJson = null;
BufferedWriter writer = null;
Table table = null;
try (Connection connect = ConnectionFactory.createConnection(config);
Admin admin = connect.getAdmin()) {
TableName tablename = TableName.valueOf(TABLE_NAME);
if (!admin.tableExists(tablename)) {
System.out.println("Table does not exist.");
return null;
}
table = connect.getTable(tablename);
Put put = null;
String start = null;
String end = null;
writer = new BufferedWriter(new FileWriter(new File(strConfig), true));
for (KeyValue item : lstAllProcessStation) {
start = (String) item.getKey();
end = (String) item.getValue();
try {
try {
Thread.sleep(200);
} catch (InterruptedException e1) {
e1.printStackTrace();
}
System.out.println("process : " + start + ":" + end);
strJson = getFromAPIX(mapStationCode.get(start), mapStationCode.get(end), date);
writer.write(start + ":" + end);
writer.newLine();
} catch (Exception e) {
System.out.println(start + ":" + end + "error");
e.printStackTrace();
break;
}
JSONObject jo = new JSONObject(strJson);
if (jo.has("httpstatus") && (jo.getInt("httpstatus") == 200)) {
JSONObject joData = jo.getJSONObject("data");
if (joData.has("flag") && joData.getBoolean("flag")) {
result.add(new DefaultKeyValue(start, end));
// 插入到hbase
String rowkey = start + ":" + end;
put = new Put(rowkey.getBytes());
put.addColumn(
CF_JSON.getBytes(), "json".getBytes(), joData.toString().getBytes("utf-8"));
table.put(put);
System.out.println("start " + start + "\t end " + end + "\t has ticket");
}
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (writer != null) {
try {
writer.flush();
writer.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (table != null) {
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return result;
}
