private void ensureColumnFamilyExists(String tableName, String columnFamily)
throws StorageException {
HBaseAdmin adm = getAdminInterface();
HTableDescriptor desc = ensureTableExists(tableName);
Preconditions.checkNotNull(desc);
HColumnDescriptor cf = desc.getFamily(columnFamily.getBytes());
// Create our column family, if necessary
if (cf == null) {
try {
adm.disableTable(tableName);
desc.addFamily(
new HColumnDescriptor(columnFamily).setCompressionType(Compression.Algorithm.GZ));
adm.modifyTable(tableName.getBytes(), desc);
try {
logger.debug(
"Added HBase ColumnFamily {}, waiting for 1 sec. to propogate.", columnFamily);
Thread.sleep(1000L);
} catch (InterruptedException ie) {
throw new TemporaryStorageException(ie);
}
adm.enableTable(tableName);
} catch (TableNotFoundException ee) {
logger.error("TableNotFoundException", ee);
throw new PermanentStorageException(ee);
} catch (org.apache.hadoop.hbase.TableExistsException ee) {
logger.debug("Swallowing exception {}", ee);
} catch (IOException ee) {
throw new TemporaryStorageException(ee);
}
} else { // check if compression was enabled, if not - enable it
if (cf.getCompressionType() == null
|| cf.getCompressionType() == Compression.Algorithm.NONE) {
try {
adm.disableTable(tableName);
adm.modifyColumn(tableName, cf.setCompressionType(Compression.Algorithm.GZ));
adm.enableTable(tableName);
} catch (IOException e) {
throw new TemporaryStorageException(e);
}
}
}
}
@Test(timeout = 180000)
public void testRestoreSnapshot() throws Exception {
String nsp = prefix + "_testRestoreSnapshot";
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(nsp)
.addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
.build();
ADMIN.createNamespace(nspDesc);
assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
String snapshot = "snapshot_testRestoreSnapshot";
ADMIN.snapshot(snapshot, tableName1);
List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
Collections.sort(regions);
ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
Thread.sleep(2000);
assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
ADMIN.disableTable(tableName1);
ADMIN.restoreSnapshot(snapshot);
assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
ADMIN.enableTable(tableName1);
ADMIN.deleteSnapshot(snapshot);
}
/** Apply column family options such as Bloom filters, compression, and data block encoding. */
protected void applyColumnFamilyOptions(byte[] tableName, byte[][] columnFamilies)
throws IOException {
HBaseAdmin admin = new HBaseAdmin(conf);
HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
LOG.info("Disabling table " + Bytes.toString(tableName));
admin.disableTable(tableName);
for (byte[] cf : columnFamilies) {
HColumnDescriptor columnDesc = tableDesc.getFamily(cf);
boolean isNewCf = columnDesc == null;
if (isNewCf) {
columnDesc = new HColumnDescriptor(cf);
}
if (bloomType != null) {
columnDesc.setBloomFilterType(bloomType);
}
if (compressAlgo != null) {
columnDesc.setCompressionType(compressAlgo);
}
if (dataBlockEncodingAlgo != null) {
columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo);
columnDesc.setEncodeOnDisk(!encodeInCacheOnly);
}
if (inMemoryCF) {
columnDesc.setInMemory(inMemoryCF);
}
if (isNewCf) {
admin.addColumn(tableName, columnDesc);
} else {
admin.modifyColumn(tableName, columnDesc);
}
}
LOG.info("Enabling table " + Bytes.toString(tableName));
admin.enableTable(tableName);
}
public void fixInconsistent() throws IOException {
if (ifFix == true) {
for (String segFullName : inconsistentHTables) {
String[] sepNameList = segFullName.split(",");
HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
logger.info(
"Change the host of htable "
+ sepNameList[0]
+ "belonging to cube "
+ sepNameList[1]
+ " from "
+ desc.getValue(IRealizationConstants.HTableTag)
+ " to "
+ dstCfg.getMetadataUrlPrefix());
hbaseAdmin.disableTable(sepNameList[0]);
desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
hbaseAdmin.modifyTable(sepNameList[0], desc);
hbaseAdmin.enableTable(sepNameList[0]);
}
} else {
logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
for (String hTable : inconsistentHTables) {
String[] sepNameList = hTable.split(",");
logger.info(sepNameList[0] + " belonging to cube " + sepNameList[1]);
}
logger.info("----------------------------------------------------");
}
}
/**
* @param rawAttributeName is the attribute name viewed by applications, it allows multiple
* values. For example, secondaryIndex in secondaryIndex$1 and coprocessor in corpcessor$2
* @param indexOfAttribute is of the same raw attribute name, for example 2 in secondary$2
*/
static void updateTableAttribute(
Configuration conf,
byte[] tableName,
String rawAttributeName,
int indexOfAttribute,
boolean ifUpdateorRemove,
String value)
throws IOException {
HBaseAdmin admin = new HBaseAdmin(conf);
HTableDescriptor desc = admin.getTableDescriptor(tableName);
admin.disableTable(tableName);
// System.out.println("TTDEBUG: disable table " + Bytes.toString(tableName));
String coprocessorKey = rawAttributeName + indexOfAttribute;
if (!ifUpdateorRemove) {
desc.remove(Bytes.toBytes(coprocessorKey));
} else {
desc.setValue(coprocessorKey, value);
}
admin.modifyTable(tableName, desc);
// System.out.println("TTDEBUG: modify table " + Bytes.toString(tableName));
admin.enableTable(tableName);
// System.out.println("TTDEBUG: enable table " + Bytes.toString(tableName));
HTableDescriptor descNew = admin.getTableDescriptor(tableName);
// modify table is asynchronous, has to loop over to check
while (!desc.equals(descNew)) {
System.err.println(
"TTDEBUG: waiting for descriptor to change: from " + descNew + " to " + desc);
try {
Thread.sleep(500);
} catch (InterruptedException ex) {
}
descNew = admin.getTableDescriptor(tableName);
}
}
/**
* 删除指定表名
*
* @param rowKey
*/
public void deleteTable(byte[] rowKey) {
Connection conn = null;
HBaseAdmin admin = null;
try {
conn = ConnectionFactory.createConnection(conf);
admin = (HBaseAdmin) conn.getAdmin();
// 在删除一张表前，要使其失效
admin.disableTable(rowKey);
admin.deleteTable(rowKey);
admin.enableTable(rowKey);
} catch (Exception e) {
logger.error("HBaseAdmin deleteTable exception, errMsg:{}", e.getMessage());
} finally {
try {
if (null != admin) {
admin.close();
}
} catch (IOException e) {
logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage());
}
try {
if (null != conn) {
conn.close();
}
} catch (Exception e) {
logger.error("Connection close exception, errMsg:{}", e.getMessage());
}
}
}
public static void createTable(Configuration hbaseConf, String tableName) throws IOException {
HBaseAdmin admin = new HBaseAdmin(hbaseConf);
if (!admin.tableExists(tableName)) {
HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
HColumnDescriptor datafam = new HColumnDescriptor(HBaseTimestampStorage.TSO_FAMILY);
datafam.setMaxVersions(3);
desc.addFamily(datafam);
admin.createTable(desc);
}
if (admin.isTableDisabled(tableName)) {
admin.enableTable(tableName);
}
admin.close();
LOG.info("Table {} created successfully", tableName);
}
public static void main(String args[]) throws MasterNotRunningException, IOException {
// Instantiating configuration class
Configuration conf = HBaseConfiguration.create();
// Instantiating HBaseAdmin class
HBaseAdmin admin = new HBaseAdmin(conf);
// Verifying weather the table is disabled
Boolean bool = admin.isTableEnabled("emp");
System.out.println(bool);
// Disabling the table using HBaseAdmin object
if (!bool) {
admin.enableTable("emp");
System.out.println("Table Enabled");
}
}
@Override
public void setConfigurationProperty(final String key, final String value)
throws StorageException {
byte[] name = tableName.getBytes();
HTableDescriptor desc = ensureTableExists(tableName);
try {
HBaseAdmin adm = getAdminInterface();
adm.disableTable(tableName);
desc.setValue(key, value);
adm.modifyTable(name, desc);
adm.enableTable(tableName);
} catch (IOException e) {
throw new PermanentStorageException(e);
}
}
@Test(timeout = 180000)
public void testRestoreSnapshotQuotaExceed() throws Exception {
String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(nsp)
.addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
.build();
ADMIN.createNamespace(nspDesc);
NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
assertNotNull("Namespace descriptor found null.", ndesc);
TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
ADMIN.snapshot(snapshot, tableName1);
List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
Collections.sort(regions);
ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
Thread.sleep(2000);
assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2");
ADMIN.modifyNamespace(ndesc);
ADMIN.disableTable(tableName1);
try {
ADMIN.restoreSnapshot(snapshot);
fail(
"Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
+ " wraps IOException into RestoreSnapshotException");
} catch (RestoreSnapshotException ignore) {
}
ADMIN.enableTable(tableName1);
ADMIN.deleteSnapshot(snapshot);
}
private void doIncrementalLoadTest(boolean shouldChangeRegions) throws Exception {
util = new HBaseTestingUtility();
Configuration conf = util.getConfiguration();
byte[][] splitKeys = generateRandomSplitKeys(4);
HBaseAdmin admin = null;
try {
util.startMiniCluster();
Path testDir = util.getDataTestDirOnTestFS("testLocalMRIncrementalLoad");
admin = util.getHBaseAdmin();
HTable table = util.createTable(TABLE_NAME, FAMILIES, splitKeys);
assertEquals("Should start with empty table", 0, util.countRows(table));
int numRegions = -1;
try (RegionLocator r = table.getRegionLocator()) {
numRegions = r.getStartKeys().length;
}
assertEquals("Should make 5 regions", numRegions, 5);
// Generate the bulk load files
util.startMiniMapReduceCluster();
runIncrementalPELoad(conf, table, testDir);
// This doesn't write into the table, just makes files
assertEquals("HFOF should not touch actual table", 0, util.countRows(table));
// Make sure that a directory was created for every CF
int dir = 0;
for (FileStatus f : testDir.getFileSystem(conf).listStatus(testDir)) {
for (byte[] family : FAMILIES) {
if (Bytes.toString(family).equals(f.getPath().getName())) {
++dir;
}
}
}
assertEquals("Column family not found in FS.", FAMILIES.length, dir);
// handle the split case
if (shouldChangeRegions) {
LOG.info("Changing regions in table");
admin.disableTable(table.getName());
while (util.getMiniHBaseCluster()
.getMaster()
.getAssignmentManager()
.getRegionStates()
.isRegionsInTransition()) {
Threads.sleep(200);
LOG.info("Waiting on table to finish disabling");
}
util.deleteTable(table.getName());
byte[][] newSplitKeys = generateRandomSplitKeys(14);
table = util.createTable(TABLE_NAME, FAMILIES, newSplitKeys);
while (table.getRegionLocations().size() != 15
|| !admin.isTableAvailable(table.getName())) {
Thread.sleep(200);
LOG.info("Waiting for new region assignment to happen");
}
}
// Perform the actual load
new LoadIncrementalHFiles(conf).doBulkLoad(testDir, table);
// Ensure data shows up
int expectedRows = NMapInputFormat.getNumMapTasks(conf) * ROWSPERSPLIT;
assertEquals(
"LoadIncrementalHFiles should put expected data in table",
expectedRows,
util.countRows(table));
Scan scan = new Scan();
ResultScanner results = table.getScanner(scan);
for (Result res : results) {
assertEquals(FAMILIES.length, res.rawCells().length);
Cell first = res.rawCells()[0];
for (Cell kv : res.rawCells()) {
assertTrue(CellUtil.matchingRow(first, kv));
assertTrue(Bytes.equals(CellUtil.cloneValue(first), CellUtil.cloneValue(kv)));
}
}
results.close();
String tableDigestBefore = util.checksumRows(table);
// Cause regions to reopen
admin.disableTable(TABLE_NAME);
while (!admin.isTableDisabled(TABLE_NAME)) {
Thread.sleep(200);
LOG.info("Waiting for table to disable");
}
admin.enableTable(TABLE_NAME);
util.waitTableAvailable(TABLE_NAME);
assertEquals(
"Data should remain after reopening of regions",
tableDigestBefore,
util.checksumRows(table));
} finally {
if (admin != null) admin.close();
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@Ignore
@Test
public void testNonTxToTxTableFailure() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
// Put table in SYSTEM schema to prevent attempts to update the cache after we disable
// SYSTEM.CATALOG
conn.createStatement()
.execute("CREATE TABLE SYSTEM.NON_TX_TABLE(k INTEGER PRIMARY KEY, v VARCHAR)");
conn.createStatement().execute("UPSERT INTO SYSTEM.NON_TX_TABLE VALUES (1)");
conn.commit();
// Reset empty column value to an empty value like it is pre-transactions
HTableInterface htable =
conn.unwrap(PhoenixConnection.class)
.getQueryServices()
.getTable(Bytes.toBytes("SYSTEM.NON_TX_TABLE"));
Put put = new Put(PInteger.INSTANCE.toBytes(1));
put.add(
QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES,
QueryConstants.EMPTY_COLUMN_BYTES,
ByteUtil.EMPTY_BYTE_ARRAY);
htable.put(put);
HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
admin.disableTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME);
try {
// This will succeed initially in updating the HBase metadata, but then will fail when
// the SYSTEM.CATALOG table is attempted to be updated, exercising the code to restore
// the coprocessors back to the non transactional ones.
conn.createStatement().execute("ALTER TABLE SYSTEM.NON_TX_TABLE SET TRANSACTIONAL=true");
fail();
} catch (SQLException e) {
assertTrue(
e.getMessage().contains(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME + " is disabled"));
} finally {
admin.enableTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME);
admin.close();
}
ResultSet rs =
conn.createStatement().executeQuery("SELECT k FROM SYSTEM.NON_TX_TABLE WHERE v IS NULL");
assertTrue(rs.next());
assertEquals(1, rs.getInt(1));
assertFalse(rs.next());
htable =
conn.unwrap(PhoenixConnection.class)
.getQueryServices()
.getTable(Bytes.toBytes("SYSTEM.NON_TX_TABLE"));
assertFalse(
htable
.getTableDescriptor()
.getCoprocessors()
.contains(TransactionProcessor.class.getName()));
assertEquals(
1,
conn.unwrap(PhoenixConnection.class)
.getQueryServices()
.getTableDescriptor(Bytes.toBytes("SYSTEM.NON_TX_TABLE"))
.getFamily(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES)
.getMaxVersions());
}