@Test(timeout = 60000)
public void testRollbackAndDoubleExecution() throws Exception {
final NamespaceDescriptor nsd =
NamespaceDescriptor.create("testRollbackAndDoubleExecution").build();
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
ProcedureTestingUtility.waitNoProcedureRunning(procExec);
ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
// Start the CreateNamespace procedure && kill the executor
long procId =
procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
int numberOfSteps = 0; // failing at pre operation
MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, numberOfSteps);
// Validate the non-existence of namespace
try {
NamespaceDescriptor nsDescriptor = UTIL.getAdmin().getNamespaceDescriptor(nsd.getName());
assertNull(nsDescriptor);
} catch (NamespaceNotFoundException nsnfe) {
// Expected
LOG.info("The namespace " + nsd.getName() + " is not created.");
}
}
public void doAnAction() throws Exception {
long iteration = numBulkLoads.getAndIncrement();
Path dir = UTIL.getDataTestDirOnTestFS(String.format("bulkLoad_%08d", iteration));
// create HFiles for different column families
FileSystem fs = UTIL.getTestFileSystem();
byte[] val = Bytes.toBytes(String.format("%010d", iteration));
final List<Pair<byte[], String>> famPaths = new ArrayList<Pair<byte[], String>>(NUM_CFS);
for (int i = 0; i < NUM_CFS; i++) {
Path hfile = new Path(dir, family(i));
byte[] fam = Bytes.toBytes(family(i));
createHFile(fs, hfile, fam, QUAL, val, 1000);
famPaths.add(new Pair<>(fam, hfile.toString()));
}
// bulk load HFiles
final ClusterConnection conn = (ClusterConnection) UTIL.getAdmin().getConnection();
RegionServerCallable<Void> callable =
new RegionServerCallable<Void>(conn, tableName, Bytes.toBytes("aaa")) {
@Override
public Void call(int callTimeout) throws Exception {
LOG.debug(
"Going to connect to server "
+ getLocation()
+ " for row "
+ Bytes.toStringBinary(getRow()));
byte[] regionName = getLocation().getRegionInfo().getRegionName();
BulkLoadHFileRequest request =
RequestConverter.buildBulkLoadHFileRequest(famPaths, regionName, true);
getStub().bulkLoadHFile(null, request);
return null;
}
};
RpcRetryingCallerFactory factory = new RpcRetryingCallerFactory(conf);
RpcRetryingCaller<Void> caller = factory.<Void>newCaller();
caller.callWithRetries(callable, Integer.MAX_VALUE);
// Periodically do compaction to reduce the number of open file handles.
if (numBulkLoads.get() % 5 == 0) {
// 5 * 50 = 250 open file handles!
callable =
new RegionServerCallable<Void>(conn, tableName, Bytes.toBytes("aaa")) {
@Override
public Void call(int callTimeout) throws Exception {
LOG.debug(
"compacting " + getLocation() + " for row " + Bytes.toStringBinary(getRow()));
AdminProtos.AdminService.BlockingInterface server =
conn.getAdmin(getLocation().getServerName());
CompactRegionRequest request =
RequestConverter.buildCompactRegionRequest(
getLocation().getRegionInfo().getRegionName(), true, null);
server.compactRegion(null, request);
numCompactions.incrementAndGet();
return null;
}
};
caller.callWithRetries(callable, Integer.MAX_VALUE);
}
}
@BeforeClass
public static void beforeAllTests() throws Exception {
// we will retry operations when PleaseHoldException is thrown
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
// Start a cluster of two regionservers.
TEST_UTIL.startMiniCluster(2);
admin = TEST_UTIL.getAdmin();
}
@Test(timeout = 60000)
public void testCreateSystemNamespace() throws Exception {
final NamespaceDescriptor nsd =
UTIL.getAdmin().getNamespaceDescriptor(NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
long procId =
procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
// Wait the completion
ProcedureTestingUtility.waitProcedure(procExec, procId);
ProcedureInfo result = procExec.getResult(procId);
assertTrue(result.isFailed());
LOG.debug("Create namespace failed with exception: " + result.getExceptionFullMessage());
assertTrue(
ProcedureTestingUtility.getExceptionCause(result) instanceof NamespaceExistException);
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
conf = HBaseConfiguration.create();
conf.setStrings(
CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, TestCoprocessor.class.getName());
util = new HBaseTestingUtility(conf);
util.startMiniCluster();
Admin admin = util.getAdmin();
if (admin.tableExists(tableName)) {
if (admin.isTableEnabled(tableName)) {
admin.disableTable(tableName);
}
admin.deleteTable(tableName);
}
Table ht = util.createTable(tableName, new byte[][] {dummy, test});
Put p = new Put(row1);
p.addColumn(dummy, dummy, dummy);
ht.put(p);
}
@Test
@SuppressWarnings("deprecation")
public void testMasterOpsWhileSplitting() throws Exception {
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
HMaster m = cluster.getMaster();
try (Table ht = TEST_UTIL.createTable(TABLENAME, FAMILYNAME)) {
assertTrue(m.getTableStateManager().isTableState(TABLENAME, TableState.State.ENABLED));
TEST_UTIL.loadTable(ht, FAMILYNAME, false);
}
List<Pair<HRegionInfo, ServerName>> tableRegions =
MetaTableAccessor.getTableRegionsAndLocations(m.getConnection(), TABLENAME);
LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions));
assertEquals(1, tableRegions.size());
assertArrayEquals(HConstants.EMPTY_START_ROW, tableRegions.get(0).getFirst().getStartKey());
assertArrayEquals(HConstants.EMPTY_END_ROW, tableRegions.get(0).getFirst().getEndKey());
// Now trigger a split and stop when the split is in progress
LOG.info("Splitting table");
TEST_UTIL.getAdmin().split(TABLENAME);
LOG.info("Waiting for split result to be about to open");
RegionStates regionStates = m.getAssignmentManager().getRegionStates();
while (regionStates.getRegionsOfTable(TABLENAME).size() <= 1) {
Thread.sleep(100);
}
LOG.info("Making sure we can call getTableRegions while opening");
tableRegions =
MetaTableAccessor.getTableRegionsAndLocations(m.getConnection(), TABLENAME, false);
LOG.info("Regions: " + Joiner.on(',').join(tableRegions));
// We have three regions because one is split-in-progress
assertEquals(3, tableRegions.size());
LOG.info("Making sure we can call getTableRegionClosest while opening");
Pair<HRegionInfo, ServerName> pair = m.getTableRegionForRow(TABLENAME, Bytes.toBytes("cde"));
LOG.info("Result is: " + pair);
Pair<HRegionInfo, ServerName> tableRegionFromName =
MetaTableAccessor.getRegion(m.getConnection(), pair.getFirst().getRegionName());
assertEquals(tableRegionFromName.getFirst(), pair.getFirst());
}
private void validateNamespaceCreated(NamespaceDescriptor nsd) throws IOException {
NamespaceDescriptor createdNsDescriptor = UTIL.getAdmin().getNamespaceDescriptor(nsd.getName());
assertNotNull(createdNsDescriptor);
}
@Test(timeout = 30000)
public void testBulkLoad() throws IOException {
// Create table then get the single region for our new table.
LOG.debug("Creating test table");
HTableDescriptor hdt = HTU.createTableDescriptor("testBulkLoad");
hdt.setRegionReplication(NB_SERVERS);
hdt.addCoprocessor(SlowMeCopro.class.getName());
Table table = HTU.createTable(hdt, new byte[][] {f}, null);
// create hfiles to load.
LOG.debug("Creating test data");
Path dir = HTU.getDataTestDirOnTestFS("testBulkLoad");
final int numRows = 10;
final byte[] qual = Bytes.toBytes("qual");
final byte[] val = Bytes.toBytes("val");
final List<Pair<byte[], String>> famPaths = new ArrayList<Pair<byte[], String>>();
for (HColumnDescriptor col : hdt.getColumnFamilies()) {
Path hfile = new Path(dir, col.getNameAsString());
TestHRegionServerBulkLoad.createHFile(
HTU.getTestFileSystem(), hfile, col.getName(), qual, val, numRows);
famPaths.add(new Pair<byte[], String>(col.getName(), hfile.toString()));
}
// bulk load HFiles
LOG.debug("Loading test data");
final ClusterConnection conn = (ClusterConnection) HTU.getAdmin().getConnection();
table = conn.getTable(hdt.getTableName());
final String bulkToken =
new SecureBulkLoadClient(HTU.getConfiguration(), table).prepareBulkLoad(conn);
RegionServerCallable<Void> callable =
new RegionServerCallable<Void>(
conn,
new RpcControllerFactory(HTU.getConfiguration()),
hdt.getTableName(),
TestHRegionServerBulkLoad.rowkey(0)) {
@Override
protected Void rpcCall() throws Exception {
LOG.debug(
"Going to connect to server "
+ getLocation()
+ " for row "
+ Bytes.toStringBinary(getRow()));
SecureBulkLoadClient secureClient = null;
byte[] regionName = getLocation().getRegionInfo().getRegionName();
try (Table table = conn.getTable(getTableName())) {
secureClient = new SecureBulkLoadClient(HTU.getConfiguration(), table);
secureClient.secureBulkLoadHFiles(
getStub(), famPaths, regionName, true, null, bulkToken);
}
return null;
}
};
RpcRetryingCallerFactory factory = new RpcRetryingCallerFactory(HTU.getConfiguration());
RpcRetryingCaller<Void> caller = factory.newCaller();
caller.callWithRetries(callable, 10000);
// verify we can read them from the primary
LOG.debug("Verifying data load");
for (int i = 0; i < numRows; i++) {
byte[] row = TestHRegionServerBulkLoad.rowkey(i);
Get g = new Get(row);
Result r = table.get(g);
Assert.assertFalse(r.isStale());
}
// verify we can read them from the replica
LOG.debug("Verifying replica queries");
try {
SlowMeCopro.cdl.set(new CountDownLatch(1));
for (int i = 0; i < numRows; i++) {
byte[] row = TestHRegionServerBulkLoad.rowkey(i);
Get g = new Get(row);
g.setConsistency(Consistency.TIMELINE);
Result r = table.get(g);
Assert.assertTrue(r.isStale());
}
SlowMeCopro.cdl.get().countDown();
} finally {
SlowMeCopro.cdl.get().countDown();
SlowMeCopro.sleepTime.set(0);
}
HTU.getHBaseAdmin().disableTable(hdt.getTableName());
HTU.deleteTable(hdt.getTableName());
}