@Test
public void testDeleteTable() throws Exception {
String namespace = prefix + "_dummy";
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(namespace)
.addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100")
.addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3")
.build();
ADMIN.createNamespace(nspDesc);
assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace));
NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName());
assertNotNull("Namespace state found null for " + namespace, stateInfo);
HTableDescriptor tableDescOne =
new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1"));
HTableDescriptor tableDescTwo =
new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2"));
ADMIN.createTable(tableDescOne);
ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
stateInfo = getNamespaceState(nspDesc.getName());
assertNotNull("Namespace state found to be null.", stateInfo);
assertEquals(2, stateInfo.getTables().size());
assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.getTableName()));
assertEquals(6, stateInfo.getRegionCount());
ADMIN.disableTable(tableDescOne.getTableName());
deleteTable(tableDescOne.getTableName());
stateInfo = getNamespaceState(nspDesc.getName());
assertNotNull("Namespace state found to be null.", stateInfo);
assertEquals(5, stateInfo.getRegionCount());
assertEquals(1, stateInfo.getTables().size());
ADMIN.disableTable(tableDescTwo.getTableName());
deleteTable(tableDescTwo.getTableName());
ADMIN.deleteNamespace(namespace);
stateInfo = getNamespaceState(namespace);
assertNull("Namespace state not found to be null.", stateInfo);
}
@Test(timeout = 60000)
public void testRollbackAndDoubleExecution() throws Exception {
final NamespaceDescriptor nsd =
NamespaceDescriptor.create("testRollbackAndDoubleExecution").build();
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
ProcedureTestingUtility.waitNoProcedureRunning(procExec);
ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
// Start the CreateNamespace procedure && kill the executor
long procId =
procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
int numberOfSteps = 0; // failing at pre operation
MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, numberOfSteps);
// Validate the non-existence of namespace
try {
NamespaceDescriptor nsDescriptor = UTIL.getAdmin().getNamespaceDescriptor(nsd.getName());
assertNull(nsDescriptor);
} catch (NamespaceNotFoundException nsnfe) {
// Expected
LOG.info("The namespace " + nsd.getName() + " is not created.");
}
}
@Override
protected boolean acquireLock(final MasterProcedureEnv env) {
if (!env.getMasterServices().isInitialized()) {
// Namespace manager might not be ready if master is not fully initialized,
// return false to reject user namespace creation; return true for default
// and system namespace creation (this is part of master initialization).
if (nsDescriptor.equals(NamespaceDescriptor.DEFAULT_NAMESPACE)
|| nsDescriptor.equals(NamespaceDescriptor.SYSTEM_NAMESPACE)) {
return true;
}
return false;
}
return getTableNamespaceManager(env).acquireExclusiveLock();
}
@After
public void cleanup() throws Exception, KeeperException {
for (HTableDescriptor table : ADMIN.listTables()) {
ADMIN.disableTable(table.getTableName());
deleteTable(table.getTableName());
}
for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
if (ns.getName().startsWith(prefix)) {
ADMIN.deleteNamespace(ns.getName());
}
}
assertTrue(
"Quota manager not enabled",
UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().isQuotaEnabled());
}
@Override
public void toStringClassDetails(StringBuilder sb) {
sb.append(getClass().getSimpleName());
sb.append(" (Namespace=");
sb.append(nsDescriptor.getName());
sb.append(")");
}
@Test(timeout = 60000)
public void testCreateSameNamespaceTwice() throws Exception {
final NamespaceDescriptor nsd =
NamespaceDescriptor.create("testCreateSameNamespaceTwice").build();
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
long procId1 =
procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
// Wait the completion
ProcedureTestingUtility.waitProcedure(procExec, procId1);
ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
// Create the namespace that exists
long procId2 =
procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
// Wait the completion
ProcedureTestingUtility.waitProcedure(procExec, procId2);
// Second create should fail with NamespaceExistException
ProcedureInfo result = procExec.getResult(procId2);
assertTrue(result.isFailed());
LOG.debug("Create namespace failed with exception: " + result.getExceptionFullMessage());
assertTrue(
ProcedureTestingUtility.getExceptionCause(result) instanceof NamespaceExistException);
}
@Before
public void setUp() throws Exception {
TEST_UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(namespace).build());
try (Table table =
TEST_UTIL.createTable(
tableName, new String[] {Bytes.toString(TEST_FAMILY), Bytes.toString(TEST_FAMILY_2)})) {
TEST_UTIL.waitTableEnabled(tableName);
List<Put> puts = new ArrayList<Put>(5);
Put put_1 = new Put(TEST_ROW);
put_1.addColumn(TEST_FAMILY, Q1, value1);
Put put_2 = new Put(TEST_ROW_2);
put_2.addColumn(TEST_FAMILY, Q2, value2);
Put put_3 = new Put(TEST_ROW_3);
put_3.addColumn(TEST_FAMILY_2, Q1, value1);
puts.add(put_1);
puts.add(put_2);
puts.add(put_3);
table.put(puts);
}
assertEquals(1, AccessControlLists.getTablePermissions(conf, tableName).size());
try {
assertEquals(
1, AccessControlClient.getUserPermissions(connection, tableName.toString()).size());
} catch (Throwable e) {
LOG.error("Error during call of AccessControlClient.getUserPermissions. ", e);
}
// setupOperations();
}
@Test(timeout = 180000)
public void testRestoreSnapshot() throws Exception {
String nsp = prefix + "_testRestoreSnapshot";
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(nsp)
.addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
.build();
ADMIN.createNamespace(nspDesc);
assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
String snapshot = "snapshot_testRestoreSnapshot";
ADMIN.snapshot(snapshot, tableName1);
List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
Collections.sort(regions);
ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
Thread.sleep(2000);
assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
ADMIN.disableTable(tableName1);
ADMIN.restoreSnapshot(snapshot);
assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
ADMIN.enableTable(tableName1);
ADMIN.deleteSnapshot(snapshot);
}
/**
* Undo the insert.
*
* @param env MasterProcedureEnv
* @throws IOException
*/
private void rollbackInsertIntoNSTable(final MasterProcedureEnv env) throws IOException {
try {
DeleteNamespaceProcedure.deleteFromNSTable(env, nsDescriptor.getName());
} catch (Exception e) {
// Ignore exception
LOG.debug("Rollback of insertIntoNSTable throws exception: " + e);
}
}
/**
* undo create directory
*
* @param env MasterProcedureEnv
* @throws IOException
*/
private void rollbackCreateDirectory(final MasterProcedureEnv env) throws IOException {
try {
DeleteNamespaceProcedure.deleteDirectory(env, nsDescriptor.getName());
} catch (Exception e) {
// Ignore exception
LOG.debug("Rollback of createDirectory throws exception: " + e);
}
}
/**
* remove quota for the namespace if exists
*
* @param env MasterProcedureEnv
* @throws IOException
*/
private void rollbackSetNamespaceQuota(final MasterProcedureEnv env) throws IOException {
try {
DeleteNamespaceProcedure.removeNamespaceQuota(env, nsDescriptor.getName());
} catch (Exception e) {
// Ignore exception
LOG.debug("Rollback of setNamespaceQuota throws exception: " + e);
}
}
/**
* rollback Zookeeper update.
*
* @param env MasterProcedureEnv
* @throws IOException
*/
private void rollbackZKNamespaceManagerChange(final MasterProcedureEnv env) throws IOException {
try {
DeleteNamespaceProcedure.removeFromZKNamespaceManager(env, nsDescriptor.getName());
} catch (Exception e) {
// Ignore exception
LOG.debug("Rollback of updateZKNamespaceManager throws exception: " + e);
}
}
@Test(timeout = 60000)
public void testCreateNamespaceWithInvalidTableCount() throws Exception {
final NamespaceDescriptor nsd =
NamespaceDescriptor.create("testCreateNamespaceWithInvalidTableCount").build();
final String nsKey = "hbase.namespace.quota.maxtables";
final String nsValue = "-1";
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
nsd.setConfiguration(nsKey, nsValue);
long procId =
procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
// Wait the completion
ProcedureTestingUtility.waitProcedure(procExec, procId);
ProcedureInfo result = procExec.getResult(procId);
assertTrue(result.isFailed());
LOG.debug("Create namespace failed with exception: " + result.getExceptionFullMessage());
assertTrue(ProcedureTestingUtility.getExceptionCause(result) instanceof ConstraintException);
}
@Test(timeout = 180000)
public void testRestoreSnapshotQuotaExceed() throws Exception {
String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(nsp)
.addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
.build();
ADMIN.createNamespace(nspDesc);
NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
assertNotNull("Namespace descriptor found null.", ndesc);
TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
ADMIN.snapshot(snapshot, tableName1);
List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
Collections.sort(regions);
ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
Thread.sleep(2000);
assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2");
ADMIN.modifyNamespace(ndesc);
ADMIN.disableTable(tableName1);
try {
ADMIN.restoreSnapshot(snapshot);
fail(
"Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
+ " wraps IOException into RestoreSnapshotException");
} catch (RestoreSnapshotException ignore) {
}
ADMIN.enableTable(tableName1);
ADMIN.deleteSnapshot(snapshot);
}
/*
* Create a table and make sure that the table creation fails after adding this table entry into
* namespace quota cache. Now correct the failure and recreate the table with same name.
* HBASE-13394
*/
@Test(timeout = 180000)
public void testRecreateTableWithSameNameAfterFirstTimeFailure() throws Exception {
String nsp1 = prefix + "_testRecreateTable";
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(nsp1)
.addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
.addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
.build();
ADMIN.createNamespace(nspDesc);
final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
byte[] columnFamily = Bytes.toBytes("info");
HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
MasterSyncObserver.throwExceptionInPreCreateTableHandler = true;
try {
try {
ADMIN.createTable(tableDescOne);
fail("Table " + tableOne.toString() + "creation should fail.");
} catch (Exception exp) {
LOG.error(exp);
}
assertFalse(ADMIN.tableExists(tableOne));
NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp1);
assertEquals(
"First table creation failed in namespace so number of tables in namespace "
+ "should be 0.",
0,
nstate.getTables().size());
MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
try {
ADMIN.createTable(tableDescOne);
} catch (Exception e) {
fail("Table " + tableOne.toString() + "creation should succeed.");
LOG.error(e);
}
assertTrue(ADMIN.tableExists(tableOne));
nstate = getNamespaceState(nsp1);
assertEquals(
"First table was created successfully so table size in namespace should " + "be one now.",
1,
nstate.getTables().size());
} finally {
MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
if (ADMIN.tableExists(tableOne)) {
ADMIN.disableTable(tableOne);
deleteTable(tableOne);
}
ADMIN.deleteNamespace(nsp1);
}
}
@Test(timeout = 60000)
public void testCreateNamespace() throws Exception {
final NamespaceDescriptor nsd = NamespaceDescriptor.create("testCreateNamespace").build();
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
long procId =
procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
// Wait the completion
ProcedureTestingUtility.waitProcedure(procExec, procId);
ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
validateNamespaceCreated(nsd);
}
@Override
protected Flow executeFromState(final MasterProcedureEnv env, final CreateNamespaceState state)
throws InterruptedException {
if (isTraceEnabled()) {
LOG.trace(this + " execute state=" + state);
}
try {
switch (state) {
case CREATE_NAMESPACE_PREPARE:
prepareCreate(env);
setNextState(CreateNamespaceState.CREATE_NAMESPACE_CREATE_DIRECTORY);
break;
case CREATE_NAMESPACE_CREATE_DIRECTORY:
createDirectory(env, nsDescriptor);
setNextState(CreateNamespaceState.CREATE_NAMESPACE_INSERT_INTO_NS_TABLE);
break;
case CREATE_NAMESPACE_INSERT_INTO_NS_TABLE:
insertIntoNSTable(env, nsDescriptor);
setNextState(CreateNamespaceState.CREATE_NAMESPACE_UPDATE_ZK);
break;
case CREATE_NAMESPACE_UPDATE_ZK:
updateZKNamespaceManager(env, nsDescriptor);
setNextState(CreateNamespaceState.CREATE_NAMESPACE_SET_NAMESPACE_QUOTA);
break;
case CREATE_NAMESPACE_SET_NAMESPACE_QUOTA:
setNamespaceQuota(env, nsDescriptor);
return Flow.NO_MORE_STATE;
default:
throw new UnsupportedOperationException(this + " unhandled state=" + state);
}
} catch (IOException e) {
if (isRollbackSupported(state)) {
setFailure("master-create-namespace", e);
} else {
LOG.warn(
"Retriable error trying to create namespace="
+ nsDescriptor.getName()
+ " (in state="
+ state
+ ")",
e);
}
}
return Flow.HAS_MORE_STATE;
}
@Test(expected = QuotaExceededException.class, timeout = 30000)
public void testExceedTableQuotaInNamespace() throws Exception {
String nsp = prefix + "_testExceedTableQuotaInNamespace";
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(nsp)
.addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
.build();
ADMIN.createNamespace(nspDesc);
assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
HTableDescriptor tableDescOne =
new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
HTableDescriptor tableDescTwo =
new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
ADMIN.createTable(tableDescOne);
ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
}
@Test(expected = QuotaExceededException.class, timeout = 30000)
public void testCloneSnapshotQuotaExceed() throws Exception {
String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(nsp)
.addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
.build();
ADMIN.createNamespace(nspDesc);
assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
ADMIN.createTable(tableDescOne);
String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
ADMIN.snapshot(snapshot, tableName);
ADMIN.cloneSnapshot(snapshot, cloneTableName);
ADMIN.deleteSnapshot(snapshot);
}
@Test(timeout = 60000)
public void testTableOperations() throws Exception {
String nsp = prefix + "_np2";
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(nsp)
.addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
.addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
.build();
ADMIN.createNamespace(nspDesc);
assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
HTableDescriptor tableDescOne =
new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
HTableDescriptor tableDescTwo =
new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
HTableDescriptor tableDescThree =
new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table3"));
ADMIN.createTable(tableDescOne);
boolean constraintViolated = false;
try {
ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
} catch (Exception exp) {
assertTrue(exp instanceof IOException);
constraintViolated = true;
} finally {
assertTrue(
"Constraint not violated for table " + tableDescTwo.getTableName(), constraintViolated);
}
ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
NamespaceTableAndRegionInfo nspState = getQuotaManager().getState(nsp);
assertNotNull(nspState);
assertTrue(nspState.getTables().size() == 2);
assertTrue(nspState.getRegionCount() == 5);
constraintViolated = false;
try {
ADMIN.createTable(tableDescThree);
} catch (Exception exp) {
assertTrue(exp instanceof IOException);
constraintViolated = true;
} finally {
assertTrue(
"Constraint not violated for table " + tableDescThree.getTableName(), constraintViolated);
}
}
@Test
public void test() throws IOException, InterruptedException {
testUtil
.getHBaseAdmin()
.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString()).build());
Table table = testUtil.createTable(tableName, families);
table.put(
new Put(Bytes.toBytes("k")).addColumn(family, Bytes.toBytes("q"), Bytes.toBytes("v")));
MiniHBaseCluster cluster = testUtil.getMiniHBaseCluster();
List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
Region region = null;
for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
HRegionServer hrs = rsts.get(i).getRegionServer();
for (Region r : hrs.getOnlineRegions(tableName)) {
region = r;
break;
}
}
assertNotNull(region);
Thread.sleep(2000);
RegionStoreSequenceIds ids =
testUtil
.getHBaseCluster()
.getMaster()
.getLastSequenceId(region.getRegionInfo().getEncodedNameAsBytes());
assertEquals(HConstants.NO_SEQNUM, ids.getLastFlushedSequenceId());
// This will be the sequenceid just before that of the earliest edit in memstore.
long storeSequenceId = ids.getStoreSequenceId(0).getSequenceId();
assertTrue(storeSequenceId > 0);
testUtil.getHBaseAdmin().flush(tableName);
Thread.sleep(2000);
ids =
testUtil
.getHBaseCluster()
.getMaster()
.getLastSequenceId(region.getRegionInfo().getEncodedNameAsBytes());
assertTrue(
ids.getLastFlushedSequenceId() + " > " + storeSequenceId,
ids.getLastFlushedSequenceId() > storeSequenceId);
assertEquals(ids.getLastFlushedSequenceId(), ids.getStoreSequenceId(0).getSequenceId());
table.close();
}
@Test(timeout = 60000)
public void testRecoveryAndDoubleExecution() throws Exception {
final NamespaceDescriptor nsd =
NamespaceDescriptor.create("testRecoveryAndDoubleExecution").build();
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
ProcedureTestingUtility.waitNoProcedureRunning(procExec);
ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
// Start the CreateNamespace procedure && kill the executor
long procId =
procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
// Restart the executor and execute the step twice
int numberOfSteps = CreateNamespaceState.values().length;
MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps);
// Validate the creation of namespace
ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
validateNamespaceCreated(nsd);
}
public static void main(String[] args) throws IOException, InterruptedException {
Configuration conf = HBaseConfiguration.create();
HBaseHelper helper = HBaseHelper.getHelper(conf);
helper.dropTable("testtable");
Connection connection = ConnectionFactory.createConnection(conf);
Admin admin = connection.getAdmin();
// vv CreateTableWithNamespaceExample
/*[*/ NamespaceDescriptor namespace = NamespaceDescriptor.create("testspace").build();
admin.createNamespace(namespace); /*]*/
TableName tableName = TableName.valueOf("testspace", "testtable");
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor coldef = new HColumnDescriptor(Bytes.toBytes("colfam1"));
desc.addFamily(coldef);
admin.createTable(desc);
// ^^ CreateTableWithNamespaceExample
boolean avail = admin.isTableAvailable(tableName);
System.out.println("Table available: " + avail);
}
@Override
protected void rollbackState(final MasterProcedureEnv env, final CreateNamespaceState state)
throws IOException {
if (isTraceEnabled()) {
LOG.trace(this + " rollback state=" + state);
}
try {
switch (state) {
case CREATE_NAMESPACE_SET_NAMESPACE_QUOTA:
rollbackSetNamespaceQuota(env);
break;
case CREATE_NAMESPACE_UPDATE_ZK:
rollbackZKNamespaceManagerChange(env);
break;
case CREATE_NAMESPACE_INSERT_INTO_NS_TABLE:
rollbackInsertIntoNSTable(env);
break;
case CREATE_NAMESPACE_CREATE_DIRECTORY:
rollbackCreateDirectory(env);
break;
case CREATE_NAMESPACE_PREPARE:
break; // nothing to do
default:
throw new UnsupportedOperationException(this + " unhandled state=" + state);
}
} catch (IOException e) {
// This will be retried. Unless there is a bug in the code,
// this should be just a "temporary error" (e.g. network down)
LOG.warn(
"Failed rollback attempt step "
+ state
+ " for creating the namespace "
+ nsDescriptor.getName(),
e);
throw e;
}
}
@Test
public void testStatePreserve() throws Exception {
final String nsp1 = prefix + "_testStatePreserve";
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(nsp1)
.addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
.addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "10")
.build();
ADMIN.createNamespace(nspDesc);
TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
TableName tableThree = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table3");
HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
HTableDescriptor tableDescTwo = new HTableDescriptor(tableTwo);
HTableDescriptor tableDescThree = new HTableDescriptor(tableThree);
ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
ADMIN.createTable(tableDescTwo, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
ADMIN.createTable(tableDescThree, Bytes.toBytes("1"), Bytes.toBytes("1000"), 4);
ADMIN.disableTable(tableThree);
deleteTable(tableThree);
// wait for chore to complete
UTIL.waitFor(
1000,
new Waiter.Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
return (getNamespaceState(nsp1).getTables().size() == 2);
}
});
NamespaceTableAndRegionInfo before = getNamespaceState(nsp1);
restartMaster();
NamespaceTableAndRegionInfo after = getNamespaceState(nsp1);
assertEquals(
"Expected: " + before.getTables() + " Found: " + after.getTables(),
before.getTables().size(),
after.getTables().size());
}
@Test(timeout = 180000)
public void testCloneSnapshot() throws Exception {
String nsp = prefix + "_testCloneSnapshot";
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(nsp)
.addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
.addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
.build();
ADMIN.createNamespace(nspDesc);
assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
String snapshot = "snapshot_testCloneSnapshot";
ADMIN.snapshot(snapshot, tableName);
ADMIN.cloneSnapshot(snapshot, cloneTableName);
int tableLength;
try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
tableLength = locator.getStartKeys().length;
}
assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
tableLength = locator.getStartKeys().length;
}
assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
ADMIN.deleteSnapshot(snapshot);
}
@Ignore("Hangs on occasion waiting on countdown latch")
@Test
public void testRegionOperations() throws Exception {
String nsp1 = prefix + "_regiontest";
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(nsp1)
.addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2")
.addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
.build();
ADMIN.createNamespace(nspDesc);
boolean constraintViolated = false;
final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
byte[] columnFamily = Bytes.toBytes("info");
HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
NamespaceTableAndRegionInfo stateInfo;
try {
ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 7);
} catch (Exception exp) {
assertTrue(exp instanceof DoNotRetryIOException);
LOG.info(exp);
constraintViolated = true;
} finally {
assertTrue(constraintViolated);
}
assertFalse(ADMIN.tableExists(tableOne));
// This call will pass.
ADMIN.createTable(tableDescOne);
Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
Table htable = connection.getTable(tableOne);
UTIL.loadNumericRows(htable, Bytes.toBytes("info"), 1, 1000);
ADMIN.flush(tableOne);
stateInfo = getNamespaceState(nsp1);
assertEquals(1, stateInfo.getTables().size());
assertEquals(1, stateInfo.getRegionCount());
restartMaster();
ADMIN.split(tableOne, Bytes.toBytes("500"));
HRegion actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
CustomObserver observer =
(CustomObserver)
actualRegion.getCoprocessorHost().findCoprocessor(CustomObserver.class.getName());
assertNotNull(observer);
observer.postSplit.await();
assertEquals(2, ADMIN.getTableRegions(tableOne).size());
actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
observer =
(CustomObserver)
actualRegion.getCoprocessorHost().findCoprocessor(CustomObserver.class.getName());
assertNotNull(observer);
ADMIN.split(
tableOne,
getSplitKey(
actualRegion.getRegionInfo().getStartKey(), actualRegion.getRegionInfo().getEndKey()));
observer.postSplit.await();
// Make sure no regions have been added.
List<HRegionInfo> hris = ADMIN.getTableRegions(tableOne);
assertEquals(2, hris.size());
assertTrue("split completed", observer.preSplitBeforePONR.getCount() == 1);
htable.close();
}
private void validateNamespaceCreated(NamespaceDescriptor nsd) throws IOException {
NamespaceDescriptor createdNsDescriptor = UTIL.getAdmin().getNamespaceDescriptor(nsd.getName());
assertNotNull(createdNsDescriptor);
}
@BeforeClass
public static void beforeClass() throws Exception {
conf = UTIL.getConfiguration();
enableSecurity(conf);
SUPERUSER = User.createUserForTesting(conf, "admin", new String[] {"supergroup"});
// Users with global permissions
USER_GLOBAL_ADMIN = User.createUserForTesting(conf, "global_admin", new String[0]);
USER_GLOBAL_CREATE = User.createUserForTesting(conf, "global_create", new String[0]);
USER_GLOBAL_WRITE = User.createUserForTesting(conf, "global_write", new String[0]);
USER_GLOBAL_READ = User.createUserForTesting(conf, "global_read", new String[0]);
USER_GLOBAL_EXEC = User.createUserForTesting(conf, "global_exec", new String[0]);
USER_NS_ADMIN = User.createUserForTesting(conf, "namespace_admin", new String[0]);
USER_NS_CREATE = User.createUserForTesting(conf, "namespace_create", new String[0]);
USER_NS_WRITE = User.createUserForTesting(conf, "namespace_write", new String[0]);
USER_NS_READ = User.createUserForTesting(conf, "namespace_read", new String[0]);
USER_NS_EXEC = User.createUserForTesting(conf, "namespace_exec", new String[0]);
USER_TABLE_CREATE = User.createUserForTesting(conf, "table_create", new String[0]);
USER_TABLE_WRITE = User.createUserForTesting(conf, "table_write", new String[0]);
USER_GROUP_ADMIN =
User.createUserForTesting(conf, "user_group_admin", new String[] {GROUP_ADMIN});
USER_GROUP_NS_ADMIN =
User.createUserForTesting(conf, "user_group_ns_admin", new String[] {GROUP_NS_ADMIN});
USER_GROUP_CREATE =
User.createUserForTesting(conf, "user_group_create", new String[] {GROUP_CREATE});
USER_GROUP_READ = User.createUserForTesting(conf, "user_group_read", new String[] {GROUP_READ});
USER_GROUP_WRITE =
User.createUserForTesting(conf, "user_group_write", new String[] {GROUP_WRITE});
// TODO: other table perms
UTIL.startMiniCluster();
// Wait for the ACL table to become available
UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME.getName(), 30 * 1000);
ACCESS_CONTROLLER =
(AccessController)
UTIL.getMiniHBaseCluster()
.getMaster()
.getRegionServerCoprocessorHost()
.findCoprocessor(AccessController.class.getName());
UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(TEST_NAMESPACE).build());
UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(TEST_NAMESPACE2).build());
// grants on global
grantGlobal(UTIL, USER_GLOBAL_ADMIN.getShortName(), Permission.Action.ADMIN);
grantGlobal(UTIL, USER_GLOBAL_CREATE.getShortName(), Permission.Action.CREATE);
grantGlobal(UTIL, USER_GLOBAL_WRITE.getShortName(), Permission.Action.WRITE);
grantGlobal(UTIL, USER_GLOBAL_READ.getShortName(), Permission.Action.READ);
grantGlobal(UTIL, USER_GLOBAL_EXEC.getShortName(), Permission.Action.EXEC);
// grants on namespace
grantOnNamespace(UTIL, USER_NS_ADMIN.getShortName(), TEST_NAMESPACE, Permission.Action.ADMIN);
grantOnNamespace(UTIL, USER_NS_CREATE.getShortName(), TEST_NAMESPACE, Permission.Action.CREATE);
grantOnNamespace(UTIL, USER_NS_WRITE.getShortName(), TEST_NAMESPACE, Permission.Action.WRITE);
grantOnNamespace(UTIL, USER_NS_READ.getShortName(), TEST_NAMESPACE, Permission.Action.READ);
grantOnNamespace(UTIL, USER_NS_EXEC.getShortName(), TEST_NAMESPACE, Permission.Action.EXEC);
grantOnNamespace(UTIL, toGroupEntry(GROUP_NS_ADMIN), TEST_NAMESPACE, Permission.Action.ADMIN);
grantOnNamespace(UTIL, USER_NS_ADMIN.getShortName(), TEST_NAMESPACE2, Permission.Action.ADMIN);
grantGlobal(UTIL, toGroupEntry(GROUP_ADMIN), Permission.Action.ADMIN);
grantGlobal(UTIL, toGroupEntry(GROUP_CREATE), Permission.Action.CREATE);
grantGlobal(UTIL, toGroupEntry(GROUP_READ), Permission.Action.READ);
grantGlobal(UTIL, toGroupEntry(GROUP_WRITE), Permission.Action.WRITE);
}
@Test
public void testRegionMerge() throws Exception {
String nsp1 = prefix + "_regiontest";
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(nsp1)
.addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3")
.addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
.build();
ADMIN.createNamespace(nspDesc);
final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
byte[] columnFamily = Bytes.toBytes("info");
HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo);
tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
final int initialRegions = 3;
ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("2000"), initialRegions);
Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
try (Table table = connection.getTable(tableTwo)) {
UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
}
ADMIN.flush(tableTwo);
List<HRegionInfo> hris = ADMIN.getTableRegions(tableTwo);
Collections.sort(hris);
// merge the two regions
final Set<String> encodedRegionNamesToMerge =
Sets.newHashSet(hris.get(0).getEncodedName(), hris.get(1).getEncodedName());
ADMIN.mergeRegions(
hris.get(0).getEncodedNameAsBytes(), hris.get(1).getEncodedNameAsBytes(), false);
UTIL.waitFor(
10000,
100,
new Waiter.ExplainingPredicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
RegionStates regionStates =
UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
return false;
}
if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
return false;
}
}
return true;
}
@Override
public String explainFailure() throws Exception {
RegionStates regionStates =
UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
return hri + " which is expected to be merged is still online";
}
if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
return hri + " is still in not opened";
}
}
return "Unknown";
}
});
hris = ADMIN.getTableRegions(tableTwo);
assertEquals(initialRegions - 1, hris.size());
Collections.sort(hris);
final HRegionInfo hriToSplit = hris.get(1);
ADMIN.split(tableTwo, Bytes.toBytes("500"));
UTIL.waitFor(
10000,
100,
new Waiter.ExplainingPredicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
RegionStates regionStates =
UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
return false;
}
if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
return false;
}
}
return true;
}
@Override
public String explainFailure() throws Exception {
RegionStates regionStates =
UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
return hriToSplit + " which is expected to be split is still online";
}
if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
return hri + " is still in not opened";
}
}
return "Unknown";
}
});
hris = ADMIN.getTableRegions(tableTwo);
assertEquals(initialRegions, hris.size());
Collections.sort(hris);
// fail region merge through Coprocessor hook
MiniHBaseCluster cluster = UTIL.getHBaseCluster();
HRegionServer regionServer = cluster.getRegionServer(0);
RegionServerCoprocessorHost cpHost = regionServer.getRegionServerCoprocessorHost();
Coprocessor coprocessor = cpHost.findCoprocessor(CPRegionServerObserver.class.getName());
CPRegionServerObserver regionServerObserver = (CPRegionServerObserver) coprocessor;
regionServerObserver.failMerge(true);
regionServerObserver.triggered = false;
ADMIN.mergeRegions(
hris.get(1).getEncodedNameAsBytes(), hris.get(2).getEncodedNameAsBytes(), false);
regionServerObserver.waitUtilTriggered();
hris = ADMIN.getTableRegions(tableTwo);
assertEquals(initialRegions, hris.size());
Collections.sort(hris);
// verify that we cannot split
HRegionInfo hriToSplit2 = hris.get(1);
ADMIN.split(
tableTwo,
TableInputFormatBase.getSplitKey(hriToSplit2.getStartKey(), hriToSplit2.getEndKey(), true));
Thread.sleep(2000);
assertEquals(initialRegions, ADMIN.getTableRegions(tableTwo).size());
}