@Test(timeout = 120000)
public void testMetaRebuildHoleFail() throws Exception {
// Fully remove a meta entry and hdfs region
byte[] startKey = splits[1];
byte[] endKey = splits[2];
deleteRegion(conf, htbl, startKey, endKey);
wipeOutMeta();
// is meta really messed up?
assertEquals(0, scanMeta());
assertErrors(
doFsck(conf, false),
new ERROR_CODE[] {
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
});
// Note, would like to check # of tables, but this takes a while to time
// out.
// shutdown the minicluster
TEST_UTIL.shutdownMiniHBaseCluster();
TEST_UTIL.shutdownMiniZKCluster();
// attempt to rebuild meta table from scratch
HBaseFsck fsck = new HBaseFsck(conf);
assertFalse(fsck.rebuildMeta(false));
// bring up the minicluster
TEST_UTIL.startMiniZKCluster(); // tables seem enabled by default
TEST_UTIL.restartHBaseCluster(3);
ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
LOG.info("Waiting for no more RIT");
ZKAssign.blockUntilNoRIT(zkw);
LOG.info("No more RIT in ZK, now doing final test verification");
// Meta still messed up.
assertEquals(0, scanMeta());
HTableDescriptor[] htbls = TEST_UTIL.getHBaseAdmin().listTables();
LOG.info("Tables present after restart: " + Arrays.toString(htbls));
// After HBASE-451 HBaseAdmin.listTables() gets table descriptors from FS,
// so the table is still present and this should be 1.
assertEquals(1, htbls.length);
assertErrors(
doFsck(conf, false),
new ERROR_CODE[] {
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
});
}
@Before
public void beforeMethod() throws Exception {
if (!initialized) {
LOG.info("Setting up IntegrationTestGroup");
LOG.info("Initializing cluster with " + NUM_SLAVES_BASE + " servers");
TEST_UTIL = new IntegrationTestingUtility();
((IntegrationTestingUtility) TEST_UTIL).initializeCluster(NUM_SLAVES_BASE);
// set shared configs
admin = TEST_UTIL.getAdmin();
cluster = TEST_UTIL.getHBaseClusterInterface();
rsGroupAdmin =
new VerifyingRSGroupAdminClient(
rsGroupAdmin.newClient(TEST_UTIL.getConnection()), TEST_UTIL.getConfiguration());
LOG.info("Done initializing cluster");
initialized = true;
// cluster may not be clean
// cleanup when initializing
afterMethod();
}
}
@After
public void afterMethod() throws Exception {
LOG.info("Cleaning up previous test run");
// cleanup previous artifacts
deleteTableIfNecessary();
deleteNamespaceIfNecessary();
deleteGroups();
admin.setBalancerRunning(true, true);
LOG.info("Restoring the cluster");
((IntegrationTestingUtility) TEST_UTIL).restoreCluster();
LOG.info("Done restoring the cluster");
TEST_UTIL.waitFor(
WAIT_TIMEOUT,
new Waiter.Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
LOG.info("Waiting for cleanup to finish " + rsGroupAdmin.listRSGroups());
// Might be greater since moving servers back to default
// is after starting a server
return rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()
>= NUM_SLAVES_BASE;
}
});
TEST_UTIL.waitFor(
WAIT_TIMEOUT,
new Waiter.Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
LOG.info("Waiting for regionservers to be registered " + rsGroupAdmin.listRSGroups());
// Might be greater since moving servers back to default
// is after starting a server
return rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()
== getNumServers();
}
});
LOG.info("Done cleaning up previous test run");
}
@Override
public void run() {
while (!isStopped()) {
try {
List<HRegionInfo> regions =
MetaScanner.listAllRegions(TEST_UTIL.getConfiguration(), connection, false);
// select a random region
HRegionInfo parent = regions.get(random.nextInt(regions.size()));
if (parent == null || !TABLENAME.equals(parent.getTable())) {
continue;
}
long startKey = 0, endKey = Long.MAX_VALUE;
byte[] start = parent.getStartKey();
byte[] end = parent.getEndKey();
if (!Bytes.equals(HConstants.EMPTY_START_ROW, parent.getStartKey())) {
startKey = Bytes.toLong(parent.getStartKey());
}
if (!Bytes.equals(HConstants.EMPTY_END_ROW, parent.getEndKey())) {
endKey = Bytes.toLong(parent.getEndKey());
}
if (startKey == endKey) {
continue;
}
long midKey =
BigDecimal.valueOf(startKey)
.add(BigDecimal.valueOf(endKey))
.divideToIntegralValue(BigDecimal.valueOf(2))
.longValue();
HRegionInfo splita = new HRegionInfo(TABLENAME, start, Bytes.toBytes(midKey));
HRegionInfo splitb = new HRegionInfo(TABLENAME, Bytes.toBytes(midKey), end);
MetaTableAccessor.splitRegion(
connection, parent, splita, splitb, ServerName.valueOf("fooserver", 1, 0));
Threads.sleep(random.nextInt(200));
} catch (Throwable e) {
ex = e;
Assert.fail(StringUtils.stringifyException(e));
}
}
}
