@Test(timeout = 120000)
public void testChangeTable() throws Exception {
HTableDescriptor hdt = HTU.createTableDescriptor("testChangeTable");
hdt.setRegionReplication(NB_SERVERS);
hdt.addCoprocessor(SlowMeCopro.class.getName());
Table table = HTU.createTable(hdt, new byte[][] {f}, HTU.getConfiguration());
// basic test: it should work.
Put p = new Put(row);
p.add(f, row, row);
table.put(p);
Get g = new Get(row);
Result r = table.get(g);
Assert.assertFalse(r.isStale());
// Add a CF, it should work.
HTableDescriptor bHdt = HTU.getHBaseAdmin().getTableDescriptor(hdt.getTableName());
HColumnDescriptor hcd = new HColumnDescriptor(row);
hdt.addFamily(hcd);
HTU.getHBaseAdmin().disableTable(hdt.getTableName());
HTU.getHBaseAdmin().modifyTable(hdt.getTableName(), hdt);
HTU.getHBaseAdmin().enableTable(hdt.getTableName());
HTableDescriptor nHdt = HTU.getHBaseAdmin().getTableDescriptor(hdt.getTableName());
Assert.assertEquals(
"fams=" + Arrays.toString(nHdt.getColumnFamilies()),
bHdt.getColumnFamilies().length + 1,
nHdt.getColumnFamilies().length);
p = new Put(row);
p.add(row, row, row);
table.put(p);
g = new Get(row);
r = table.get(g);
Assert.assertFalse(r.isStale());
try {
SlowMeCopro.cdl.set(new CountDownLatch(1));
g = new Get(row);
g.setConsistency(Consistency.TIMELINE);
r = table.get(g);
Assert.assertTrue(r.isStale());
} finally {
SlowMeCopro.cdl.get().countDown();
SlowMeCopro.sleepTime.set(0);
}
Admin admin = HTU.getHBaseAdmin();
nHdt = admin.getTableDescriptor(hdt.getTableName());
Assert.assertEquals(
"fams=" + Arrays.toString(nHdt.getColumnFamilies()),
bHdt.getColumnFamilies().length + 1,
nHdt.getColumnFamilies().length);
admin.disableTable(hdt.getTableName());
admin.deleteTable(hdt.getTableName());
admin.close();
}
@Test
public void testTTL() throws Exception {
TableName tableName = TableName.valueOf("testTTL");
if (TEST_UTIL.getHBaseAdmin().tableExists(tableName)) {
TEST_UTIL.deleteTable(tableName);
}
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor hcd = new HColumnDescriptor(F).setMaxVersions(10).setTimeToLive(1);
desc.addFamily(hcd);
TEST_UTIL.getHBaseAdmin().createTable(desc);
Table t = new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName);
long now = EnvironmentEdgeManager.currentTime();
ManualEnvironmentEdge me = new ManualEnvironmentEdge();
me.setValue(now);
EnvironmentEdgeManagerTestHelper.injectEdge(me);
// 2s in the past
long ts = now - 2000;
// Set the TTL override to 3s
Put p = new Put(R);
p.setAttribute("ttl", new byte[] {});
p.add(F, tableName.getName(), Bytes.toBytes(3000L));
t.put(p);
p = new Put(R);
p.add(F, Q, ts, Q);
t.put(p);
p = new Put(R);
p.add(F, Q, ts + 1, Q);
t.put(p);
// these two should be expired but for the override
// (their ts was 2s in the past)
Get g = new Get(R);
g.setMaxVersions(10);
Result r = t.get(g);
// still there?
assertEquals(2, r.size());
TEST_UTIL.flush(tableName);
TEST_UTIL.compact(tableName, true);
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
// still there?
assertEquals(2, r.size());
// roll time forward 2s.
me.setValue(now + 2000);
// now verify that data eventually does expire
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
// should be gone now
assertEquals(0, r.size());
t.close();
}
@Before
public void setUp() throws Exception {
TEST_UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(namespace).build());
try (Table table =
TEST_UTIL.createTable(
tableName, new String[] {Bytes.toString(TEST_FAMILY), Bytes.toString(TEST_FAMILY_2)})) {
TEST_UTIL.waitTableEnabled(tableName);
List<Put> puts = new ArrayList<Put>(5);
Put put_1 = new Put(TEST_ROW);
put_1.addColumn(TEST_FAMILY, Q1, value1);
Put put_2 = new Put(TEST_ROW_2);
put_2.addColumn(TEST_FAMILY, Q2, value2);
Put put_3 = new Put(TEST_ROW_3);
put_3.addColumn(TEST_FAMILY_2, Q1, value1);
puts.add(put_1);
puts.add(put_2);
puts.add(put_3);
table.put(puts);
}
assertEquals(1, AccessControlLists.getTablePermissions(conf, tableName).size());
try {
assertEquals(
1, AccessControlClient.getUserPermissions(connection, tableName.toString()).size());
} catch (Throwable e) {
LOG.error("Error during call of AccessControlClient.getUserPermissions. ", e);
}
// setupOperations();
}
@Test(timeout = 60000)
public void testDoubleDeletedTableWithSameNonce() throws Exception {
final TableName tableName = TableName.valueOf("testDoubleDeletedTableWithSameNonce");
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
HRegionInfo[] regions =
MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
UTIL.getHBaseAdmin().disableTable(tableName);
// delete the table (that exists)
long procId1 =
procExec.submitProcedure(
new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);
// delete the table (that will no longer exist)
long procId2 =
procExec.submitProcedure(
new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);
// Wait the completion
ProcedureTestingUtility.waitProcedure(procExec, procId1);
ProcedureTestingUtility.waitProcedure(procExec, procId2);
// First delete should succeed
ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
MasterProcedureTestingUtility.validateTableDeletion(
UTIL.getHBaseCluster().getMaster(), tableName, regions, "f");
// Second delete should not fail, because it is the same delete
ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);
assertTrue(procId1 == procId2);
}
@Test(timeout = 30000)
public void testCreateDeleteTable() throws IOException {
// Create table then get the single region for our new table.
HTableDescriptor hdt = HTU.createTableDescriptor("testCreateDeleteTable");
hdt.setRegionReplication(NB_SERVERS);
hdt.addCoprocessor(SlowMeCopro.class.getName());
Table table = HTU.createTable(hdt, new byte[][] {f}, HTU.getConfiguration());
Put p = new Put(row);
p.add(f, row, row);
table.put(p);
Get g = new Get(row);
Result r = table.get(g);
Assert.assertFalse(r.isStale());
try {
// But if we ask for stale we will get it
SlowMeCopro.cdl.set(new CountDownLatch(1));
g = new Get(row);
g.setConsistency(Consistency.TIMELINE);
r = table.get(g);
Assert.assertTrue(r.isStale());
SlowMeCopro.cdl.get().countDown();
} finally {
SlowMeCopro.cdl.get().countDown();
SlowMeCopro.sleepTime.set(0);
}
HTU.getHBaseAdmin().disableTable(hdt.getTableName());
HTU.deleteTable(hdt.getTableName());
}
@Test
public void testPreWALRestoreSkip() throws Exception {
LOG.info(TestRegionObserverInterface.class.getName() + ".testPreWALRestoreSkip");
TableName tableName = TableName.valueOf(SimpleRegionObserver.TABLE_SKIPPED);
HTable table = util.createTable(tableName, new byte[][] {A, B, C});
JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
ServerName sn2 = rs1.getRegionServer().getServerName();
String regEN = table.getRegionLocations().firstEntry().getKey().getEncodedName();
util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes());
while (!sn2.equals(table.getRegionLocations().firstEntry().getValue())) {
Thread.sleep(100);
}
Put put = new Put(ROW);
put.add(A, A, A);
put.add(B, B, B);
put.add(C, C, C);
table.put(put);
table.flushCommits();
cluster.killRegionServer(rs1.getRegionServer().getServerName());
Threads.sleep(20000); // just to be sure that the kill has fully started.
util.waitUntilAllRegionsAssigned(tableName);
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"getCtPreWALRestore", "getCtPostWALRestore"},
tableName,
new Integer[] {0, 0});
util.deleteTable(tableName);
table.close();
}
@Test(timeout = 60000)
public void testRecoveryAndDoubleExecution() throws Exception {
final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution");
// create the table
byte[][] splitKeys = null;
HRegionInfo[] regions =
MasterProcedureTestingUtility.createTable(
getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
UTIL.getHBaseAdmin().disableTable(tableName);
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
ProcedureTestingUtility.waitNoProcedureRunning(procExec);
ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
// Start the Delete procedure && kill the executor
long procId =
procExec.submitProcedure(
new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);
// Restart the executor and execute the step twice
// NOTE: the 6 (number of DeleteTableState steps) is hardcoded,
// so you have to look at this test at least once when you add a new step.
MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(
procExec, procId, 6, DeleteTableState.values());
MasterProcedureTestingUtility.validateTableDeletion(
UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
}
@Test(timeout = 60000)
public void testDeleteDeletedTable() throws Exception {
final TableName tableName = TableName.valueOf("testDeleteDeletedTable");
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
HRegionInfo[] regions =
MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
UTIL.getHBaseAdmin().disableTable(tableName);
// delete the table (that exists)
long procId1 =
procExec.submitProcedure(
new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);
// delete the table (that will no longer exist)
long procId2 =
procExec.submitProcedure(
new DeleteTableProcedure(procExec.getEnvironment(), tableName),
nonceGroup + 1,
nonce + 1);
// Wait the completion
ProcedureTestingUtility.waitProcedure(procExec, procId1);
ProcedureTestingUtility.waitProcedure(procExec, procId2);
// First delete should succeed
ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
MasterProcedureTestingUtility.validateTableDeletion(
UTIL.getHBaseCluster().getMaster(), tableName, regions, "f");
// Second delete should fail with TableNotFound
ProcedureResult result = procExec.getResult(procId2);
assertTrue(result.isFailed());
LOG.debug("Delete failed with exception: " + result.getException());
assertTrue(result.getException().getCause() instanceof TableNotFoundException);
}
/**
* Setup a clean table before we start mucking with it.
*
* @throws IOException
* @throws InterruptedException
* @throws KeeperException
*/
private Table setupTable(TableName tablename) throws Exception {
HTableDescriptor desc = new HTableDescriptor(tablename);
HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
TEST_UTIL.getHBaseAdmin().createTable(desc, splits);
return this.connection.getTable(tablename);
}
@After
public void tearDown() throws Exception {
ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
for (HTableDescriptor htd : UTIL.getHBaseAdmin().listTables()) {
LOG.info("Tear down, remove table=" + htd.getTableName());
UTIL.deleteTable(htd.getTableName());
}
}
private void runTest(
String testName,
HTableDescriptor htd,
BloomType bloomType,
boolean preCreateTable,
byte[][] tableSplitKeys,
byte[][][] hfileRanges)
throws Exception {
Path dir = util.getDataTestDirOnTestFS(testName);
FileSystem fs = util.getTestFileSystem();
dir = dir.makeQualified(fs);
Path familyDir = new Path(dir, Bytes.toString(FAMILY));
int hfileIdx = 0;
for (byte[][] range : hfileRanges) {
byte[] from = range[0];
byte[] to = range[1];
HFileTestUtil.createHFile(
util.getConfiguration(),
fs,
new Path(familyDir, "hfile_" + hfileIdx++),
FAMILY,
QUALIFIER,
from,
to,
1000);
}
int expectedRows = hfileIdx * 1000;
if (preCreateTable) {
util.getHBaseAdmin().createTable(htd, tableSplitKeys);
}
final TableName tableName = htd.getTableName();
LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
String[] args = {dir.toString(), tableName.toString()};
loader.run(args);
Table table = new HTable(util.getConfiguration(), tableName);
try {
assertEquals(expectedRows, util.countRows(table));
} finally {
table.close();
}
// verify staging folder has been cleaned up
Path stagingBasePath = SecureBulkLoadUtil.getBaseStagingDir(util.getConfiguration());
if (fs.exists(stagingBasePath)) {
FileStatus[] files = fs.listStatus(stagingBasePath);
for (FileStatus file : files) {
assertTrue(
"Folder=" + file.getPath() + " is not cleaned up.",
file.getPath().getName() != "DONOTERASE");
}
}
util.deleteTable(tableName);
}
@BeforeClass
public static void beforeAllTests() throws Exception {
// Start a cluster
TEST_UTIL.startMiniCluster(1, NB_SERVERS, null, MyMaster.class, null);
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
master = cluster.getMaster();
master.balanceSwitch(false);
admin = TEST_UTIL.getHBaseAdmin();
}
@After
public void tearDown() throws Exception {
// Clean the _acl_ table
TEST_UTIL.deleteTable(tableName);
TEST_UTIL.getHBaseAdmin().deleteNamespace(namespace);
// Verify all table/namespace permissions are erased
assertEquals(0, AccessControlLists.getTablePermissions(conf, tableName).size());
assertEquals(0, AccessControlLists.getNamespacePermissions(conf, namespace).size());
}
@Test
public void testRecovery() throws Exception {
LOG.info(TestRegionObserverInterface.class.getName() + ".testRecovery");
TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testRecovery");
HTable table = util.createTable(tableName, new byte[][] {A, B, C});
try {
JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
ServerName sn2 = rs1.getRegionServer().getServerName();
String regEN = table.getRegionLocations().firstEntry().getKey().getEncodedName();
util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes());
while (!sn2.equals(table.getRegionLocations().firstEntry().getValue())) {
Thread.sleep(100);
}
Put put = new Put(ROW);
put.add(A, A, A);
put.add(B, B, B);
put.add(C, C, C);
table.put(put);
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {
"hadPreGet",
"hadPostGet",
"hadPrePut",
"hadPostPut",
"hadPreBatchMutate",
"hadPostBatchMutate",
"hadDelete"
},
tableName,
new Boolean[] {false, false, true, true, true, true, false});
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut", "getCtPostPut"},
tableName,
new Integer[] {0, 0, 1, 1});
cluster.killRegionServer(rs1.getRegionServer().getServerName());
Threads.sleep(1000); // Let the kill soak in.
util.waitUntilAllRegionsAssigned(tableName);
LOG.info("All regions assigned");
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"getCtPrePut", "getCtPostPut"},
tableName,
new Integer[] {0, 0});
} finally {
util.deleteTable(tableName);
table.close();
}
}
@BeforeClass
public static void setupBeforeClass() throws Exception {
testUtil = new HBaseTestingUtility();
testUtil.startMiniCluster();
conf = testUtil.getConfiguration();
hBaseAdmin = testUtil.getHBaseAdmin();
txStateStorage = new InMemoryTransactionStateStorage();
txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector());
txManager.startAndWait();
}
@Test
public void test() throws IOException, InterruptedException {
testUtil
.getHBaseAdmin()
.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString()).build());
Table table = testUtil.createTable(tableName, families);
table.put(
new Put(Bytes.toBytes("k")).addColumn(family, Bytes.toBytes("q"), Bytes.toBytes("v")));
MiniHBaseCluster cluster = testUtil.getMiniHBaseCluster();
List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
Region region = null;
for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
HRegionServer hrs = rsts.get(i).getRegionServer();
for (Region r : hrs.getOnlineRegions(tableName)) {
region = r;
break;
}
}
assertNotNull(region);
Thread.sleep(2000);
RegionStoreSequenceIds ids =
testUtil
.getHBaseCluster()
.getMaster()
.getLastSequenceId(region.getRegionInfo().getEncodedNameAsBytes());
assertEquals(HConstants.NO_SEQNUM, ids.getLastFlushedSequenceId());
// This will be the sequenceid just before that of the earliest edit in memstore.
long storeSequenceId = ids.getStoreSequenceId(0).getSequenceId();
assertTrue(storeSequenceId > 0);
testUtil.getHBaseAdmin().flush(tableName);
Thread.sleep(2000);
ids =
testUtil
.getHBaseCluster()
.getMaster()
.getLastSequenceId(region.getRegionInfo().getEncodedNameAsBytes());
assertTrue(
ids.getLastFlushedSequenceId() + " > " + storeSequenceId,
ids.getLastFlushedSequenceId() > storeSequenceId);
assertEquals(ids.getLastFlushedSequenceId(), ids.getStoreSequenceId(0).getSequenceId());
table.close();
}
/*
* Take a snapshot of a table, add metadata, and verify that this only
* affects one table
* @param online - Whether the table is online or not during the snapshot
*/
private void runTestSnapshotMetadataChangesIndependent(boolean online) throws Exception {
FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
// Create a table
Admin admin = UTIL.getHBaseAdmin();
final long startTime = System.currentTimeMillis();
final TableName localTableName = TableName.valueOf(STRING_TABLE_NAME + startTime);
Table original = UTIL.createTable(localTableName, TEST_FAM);
UTIL.loadTable(original, TEST_FAM);
final String snapshotNameAsString = "snapshot_" + localTableName;
// Create a snapshot
SnapshotTestingUtils.createSnapshotAndValidate(
admin, localTableName, TEST_FAM_STR, snapshotNameAsString, rootDir, fs, online);
if (!online) {
admin.enableTable(localTableName);
}
TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
// Clone the snapshot
byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
admin.cloneSnapshot(snapshotName, cloneTableName);
// Add a new column family to the original table
byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);
admin.disableTable(localTableName);
admin.addColumnFamily(localTableName, hcd);
// Verify that it is not in the snapshot
admin.enableTable(localTableName);
// get a description of the cloned table
// get a list of its families
// assert that the family is there
HTableDescriptor originalTableDescriptor = original.getTableDescriptor();
HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);
Assert.assertTrue(
"The original family was not found. There is something wrong. ",
originalTableDescriptor.hasFamily(TEST_FAM));
Assert.assertTrue(
"The original family was not found in the clone. There is something wrong. ",
clonedTableDescriptor.hasFamily(TEST_FAM));
Assert.assertTrue(
"The new family was not found. ", originalTableDescriptor.hasFamily(TEST_FAM_2));
Assert.assertTrue(
"The new family was not found. ", !clonedTableDescriptor.hasFamily(TEST_FAM_2));
}
@Before
public void setUpBefore() throws Exception {
TEST_UTIL = new HBaseTestingUtility();
TEST_UTIL.getConfiguration().setInt("dfs.datanode.max.xceivers", 9192);
TEST_UTIL.startMiniCluster(3);
conf = TEST_UTIL.getConfiguration();
this.connection = ConnectionFactory.createConnection(conf);
assertEquals(0, TEST_UTIL.getHBaseAdmin().listTables().length);
// setup the table
table = TableName.valueOf(TABLE_BASE + "-" + tableIdx);
tableIdx++;
htbl = setupTable(table);
populateTable(htbl);
assertEquals(5, scanMeta());
LOG.info("Table " + table + " has " + tableRowCount(conf, table) + " entries.");
assertEquals(16, tableRowCount(conf, table));
TEST_UTIL.getHBaseAdmin().disableTable(table);
assertEquals(1, TEST_UTIL.getHBaseAdmin().listTables().length);
}
private PairOfSameType<HRegionInfo> requestMergeRegion(
HMaster master, TableName tablename, int regionAnum, int regionBnum) throws Exception {
List<Pair<HRegionInfo, ServerName>> tableRegions =
MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename);
HRegionInfo regionA = tableRegions.get(regionAnum).getFirst();
HRegionInfo regionB = tableRegions.get(regionBnum).getFirst();
TEST_UTIL
.getHBaseAdmin()
.mergeRegions(regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false);
return new PairOfSameType<HRegionInfo>(regionA, regionB);
}
/** Create the Mob Table. */
public static void createMobTable(
final HBaseTestingUtility util,
final TableName tableName,
int regionReplication,
final byte[]... families)
throws IOException, InterruptedException {
HTableDescriptor htd = new HTableDescriptor(tableName);
htd.setRegionReplication(regionReplication);
for (byte[] family : families) {
HColumnDescriptor hcd = new HColumnDescriptor(family);
hcd.setMobEnabled(true);
hcd.setMobThreshold(0L);
htd.addFamily(hcd);
}
byte[][] splitKeys = SnapshotTestingUtils.getSplitKeys();
util.getHBaseAdmin().createTable(htd, splitKeys);
SnapshotTestingUtils.waitForTableToBeOnline(util, tableName);
assertEquals(
(splitKeys.length + 1) * regionReplication,
util.getHBaseAdmin().getTableRegions(tableName).size());
}
private static void initialize(Configuration conf) {
conf = HBaseConfiguration.create(conf);
conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
try {
admin = TEST_UTIL.getHBaseAdmin();
} catch (MasterNotRunningException e) {
assertNull("Master is not running", e);
} catch (ZooKeeperConnectionException e) {
assertNull("Cannot connect to ZooKeeper", e);
} catch (IOException e) {
assertNull("IOException", e);
}
}
@Test
public void testCreateInstance_success_shouldNotCreateHBaseNamespace() throws Exception {
// given
HBaseAdmin hBaseAdmin = utility.getHBaseAdmin();
int numberOfNamespacesBeforeServiceCreation = hBaseAdmin.listNamespaceDescriptors().length;
// when
ServiceInstance instance = getServiceInstance("instanceId", "fake-bare-plan");
instanceService.createServiceInstance(getCreateInstanceRequest(instance));
// then
int numberOfNamespacesAfterServiceCreation = hBaseAdmin.listNamespaceDescriptors().length;
assertThat(
numberOfNamespacesBeforeServiceCreation, equalTo(numberOfNamespacesAfterServiceCreation));
}
/*
* Take a snapshot of a table, do a split, and verify that this only affects one table
* @param online - Whether the table is online or not during the snapshot
*/
private void runTestRegionOperationsIndependent(boolean online) throws Exception {
FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
// Create a table
Admin admin = UTIL.getHBaseAdmin();
final long startTime = System.currentTimeMillis();
final TableName localTableName = TableName.valueOf(STRING_TABLE_NAME + startTime);
Table original = UTIL.createTable(localTableName, TEST_FAM);
UTIL.loadTable(original, TEST_FAM);
final int loadedTableCount = UTIL.countRows(original);
System.out.println("Original table has: " + loadedTableCount + " rows");
final String snapshotNameAsString = "snapshot_" + localTableName;
// Create a snapshot
SnapshotTestingUtils.createSnapshotAndValidate(
admin, localTableName, TEST_FAM_STR, snapshotNameAsString, rootDir, fs, online);
if (!online) {
admin.enableTable(localTableName);
}
TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
// Clone the snapshot
byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
admin.cloneSnapshot(snapshotName, cloneTableName);
// Verify that region information is the same pre-split
((ClusterConnection) UTIL.getConnection()).clearRegionCache();
List<HRegionInfo> originalTableHRegions = admin.getTableRegions(localTableName);
final int originalRegionCount = originalTableHRegions.size();
final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size();
Assert.assertEquals(
"The number of regions in the cloned table is different than in the original table.",
originalRegionCount,
cloneTableRegionCount);
// Split a region on the parent table
admin.splitRegion(originalTableHRegions.get(0).getRegionName());
waitOnSplit(UTIL.getConnection(), original, originalRegionCount);
// Verify that the cloned table region is not split
final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size();
Assert.assertEquals(
"The number of regions in the cloned table changed though none of its regions were split.",
cloneTableRegionCount,
cloneTableRegionCount2);
}
@Test
public void testTableExists() throws IOException {
final String name = "testTableExists";
final byte[] nameBytes = Bytes.toBytes(name);
assertFalse(MetaReader.tableExists(ct, name));
UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY);
assertTrue(MetaReader.tableExists(ct, name));
HBaseAdmin admin = UTIL.getHBaseAdmin();
admin.disableTable(name);
admin.deleteTable(name);
assertFalse(MetaReader.tableExists(ct, name));
assertTrue(MetaReader.tableExists(ct, Bytes.toString(HConstants.META_TABLE_NAME)));
assertTrue(MetaReader.tableExists(ct, Bytes.toString(HConstants.ROOT_TABLE_NAME)));
}
/**
* Creates a table with given table name and specified number of column families if the table does
* not already exist.
*/
private void setupTable(TableName table, int cfs) throws IOException {
try {
LOG.info("Creating table " + table);
HTableDescriptor htd = new HTableDescriptor(table);
htd.addCoprocessor(MyObserver.class.getName());
MyObserver.sleepDuration = this.sleepDuration;
for (int i = 0; i < 10; i++) {
htd.addFamily(new HColumnDescriptor(family(i)));
}
UTIL.getHBaseAdmin().createTable(htd);
} catch (TableExistsException tee) {
LOG.info("Table " + table + " already exists");
}
}
private void testSimpleDelete(final TableName tableName, byte[][] splitKeys) throws Exception {
HRegionInfo[] regions =
MasterProcedureTestingUtility.createTable(
getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
UTIL.getHBaseAdmin().disableTable(tableName);
// delete the table
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
long procId =
ProcedureTestingUtility.submitAndWait(
procExec, new DeleteTableProcedure(procExec.getEnvironment(), tableName));
ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
MasterProcedureTestingUtility.validateTableDeletion(
UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
}
@Test
public void testBaseCases() throws Exception {
TableName tableName = TableName.valueOf("baseCases");
if (TEST_UTIL.getHBaseAdmin().tableExists(tableName)) {
TEST_UTIL.deleteTable(tableName);
}
Table t = TEST_UTIL.createTable(tableName, F, 1);
// set the version override to 2
Put p = new Put(R);
p.setAttribute("versions", new byte[] {});
p.add(F, tableName.getName(), Bytes.toBytes(2));
t.put(p);
long now = EnvironmentEdgeManager.currentTime();
// insert 2 versions
p = new Put(R);
p.add(F, Q, now, Q);
t.put(p);
p = new Put(R);
p.add(F, Q, now + 1, Q);
t.put(p);
Get g = new Get(R);
g.setMaxVersions(10);
Result r = t.get(g);
assertEquals(2, r.size());
TEST_UTIL.flush(tableName);
TEST_UTIL.compact(tableName, true);
// both version are still visible even after a flush/compaction
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
assertEquals(2, r.size());
// insert a 3rd version
p = new Put(R);
p.add(F, Q, now + 2, Q);
t.put(p);
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
// still only two version visible
assertEquals(2, r.size());
t.close();
}
@BeforeClass
public static void setUpOnce() throws Exception {
miniCluster = System.getProperty("cluster.type").equals("mini");
securedCluster = System.getProperty("cluster.secured").equals("true");
System.out.println("realCluster - " + !miniCluster);
System.out.println("securedCluster - " + securedCluster);
Util.setLoggingThreshold("ERROR");
if (miniCluster) {
if (hbase == null) {
hbase = new HBaseTestingUtility();
conf = hbase.getConfiguration();
conf.set("zookeeper.session.timeout", "3600000");
conf.set("dfs.client.socket-timeout", "3600000");
if (securedCluster) {
hbase.startMiniCluster(RS_COUNT);
hbase.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME, 30000L);
admin = new HBaseAdminWrapper(conf);
} else {
hbase.startMiniCluster(RS_COUNT);
admin = hbase.getHBaseAdmin();
}
}
} else {
if (admin == null) {
final String argsFileName =
securedCluster
? "../../testClusterRealSecured.args"
: "../../testClusterRealNonSecured.args";
if (!Util.isFile(argsFileName)) {
throw new IllegalStateException(
"You have to define args file " + argsFileName + " for tests.");
}
String[] testArgs = {argsFileName};
Args args = new TestArgs(testArgs);
admin = HBaseClient.getAdmin(args);
conf = admin.getConfiguration();
RS_COUNT = getServerNameList().size();
}
}
previousBalancerRunning = admin.setBalancerRunning(false, true);
hConnection = HConnectionManager.createConnection(conf);
USER_RW = User.createUserForTesting(conf, "rwuser", new String[0]);
}
@Test
public void testTableCreateAndDeletePB() throws IOException, JAXBException {
String schemaPath = "/" + TABLE2 + "/schema";
TableSchemaModel model;
Response response;
Admin admin = TEST_UTIL.getHBaseAdmin();
assertFalse(admin.tableExists(TableName.valueOf(TABLE2)));
// create the table
model = testTableSchemaModel.buildTestModel(TABLE2);
testTableSchemaModel.checkModel(model, TABLE2);
response = client.put(schemaPath, Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 201);
// recall the same put operation but in read-only mode
conf.set("hbase.rest.readonly", "true");
response = client.put(schemaPath, Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
assertEquals(response.getCode(), 403);
// retrieve the schema and validate it
response = client.get(schemaPath, Constants.MIMETYPE_PROTOBUF);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
model = new TableSchemaModel();
model.getObjectFromMessage(response.getBody());
testTableSchemaModel.checkModel(model, TABLE2);
// retrieve the schema and validate it with alternate pbuf type
response = client.get(schemaPath, Constants.MIMETYPE_PROTOBUF_IETF);
assertEquals(response.getCode(), 200);
assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type"));
model = new TableSchemaModel();
model.getObjectFromMessage(response.getBody());
testTableSchemaModel.checkModel(model, TABLE2);
// test delete schema operation is forbidden in read-only mode
response = client.delete(schemaPath);
assertEquals(response.getCode(), 403);
// return read-only setting back to default
conf.set("hbase.rest.readonly", "false");
// delete the table and make sure HBase concurs
response = client.delete(schemaPath);
assertEquals(response.getCode(), 200);
assertFalse(admin.tableExists(TableName.valueOf(TABLE2)));
}
@SuppressWarnings("deprecation")
@Before
public void setUp() throws Exception {
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor hcd = new HColumnDescriptor(family);
hcd.setMobEnabled(true);
hcd.setMobThreshold(3L);
hcd.setMaxVersions(4);
desc.addFamily(hcd);
admin = TEST_UTIL.getHBaseAdmin();
admin.createTable(desc);
table =
ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())
.getBufferedMutator(TableName.valueOf(tableName));
}