@Test(timeout = 60000)
public void testExceptionDuringInitialization() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); // Let's fail fast.
conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true);
conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, "");
TEST_UTIL.startMiniCluster(2);
try {
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
// Trigger one regionserver to fail as if it came up with a coprocessor
// that fails during initialization
final HRegionServer regionServer = cluster.getRegionServer(0);
conf.set(
CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
FailedInitializationObserver.class.getName());
regionServer
.getRegionServerCoprocessorHost()
.loadSystemCoprocessors(conf, CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
TEST_UTIL.waitFor(
10000,
1000,
new Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
return regionServer.isAborted();
}
});
} finally {
TEST_UTIL.shutdownMiniCluster();
}
}
@Test
public void testPreWALRestoreSkip() throws Exception {
LOG.info(TestRegionObserverInterface.class.getName() + ".testPreWALRestoreSkip");
TableName tableName = TableName.valueOf(SimpleRegionObserver.TABLE_SKIPPED);
HTable table = util.createTable(tableName, new byte[][] {A, B, C});
JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
ServerName sn2 = rs1.getRegionServer().getServerName();
String regEN = table.getRegionLocations().firstEntry().getKey().getEncodedName();
util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes());
while (!sn2.equals(table.getRegionLocations().firstEntry().getValue())) {
Thread.sleep(100);
}
Put put = new Put(ROW);
put.add(A, A, A);
put.add(B, B, B);
put.add(C, C, C);
table.put(put);
table.flushCommits();
cluster.killRegionServer(rs1.getRegionServer().getServerName());
Threads.sleep(20000); // just to be sure that the kill has fully started.
util.waitUntilAllRegionsAssigned(tableName);
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"getCtPreWALRestore", "getCtPostWALRestore"},
tableName,
new Integer[] {0, 0});
util.deleteTable(tableName);
table.close();
}
@BeforeClass
public static void beforeAllTests() throws Exception {
// Start a cluster
TEST_UTIL.startMiniCluster(1, NB_SERVERS, null, MyMaster.class, null);
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
master = cluster.getMaster();
master.balanceSwitch(false);
admin = TEST_UTIL.getHBaseAdmin();
}
@Test
public void testRecovery() throws Exception {
LOG.info(TestRegionObserverInterface.class.getName() + ".testRecovery");
TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testRecovery");
HTable table = util.createTable(tableName, new byte[][] {A, B, C});
try {
JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
ServerName sn2 = rs1.getRegionServer().getServerName();
String regEN = table.getRegionLocations().firstEntry().getKey().getEncodedName();
util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes());
while (!sn2.equals(table.getRegionLocations().firstEntry().getValue())) {
Thread.sleep(100);
}
Put put = new Put(ROW);
put.add(A, A, A);
put.add(B, B, B);
put.add(C, C, C);
table.put(put);
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {
"hadPreGet",
"hadPostGet",
"hadPrePut",
"hadPostPut",
"hadPreBatchMutate",
"hadPostBatchMutate",
"hadDelete"
},
tableName,
new Boolean[] {false, false, true, true, true, true, false});
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut", "getCtPostPut"},
tableName,
new Integer[] {0, 0, 1, 1});
cluster.killRegionServer(rs1.getRegionServer().getServerName());
Threads.sleep(1000); // Let the kill soak in.
util.waitUntilAllRegionsAssigned(tableName);
LOG.info("All regions assigned");
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"getCtPrePut", "getCtPostPut"},
tableName,
new Integer[] {0, 0});
} finally {
util.deleteTable(tableName);
table.close();
}
}
@BeforeClass
public static void startCluster() throws Exception {
LOG.info("Starting cluster");
TEST_UTIL = new HBaseTestingUtility();
TEST_UTIL.startMiniCluster(1, 1, 1, null, MyMaster.class, null);
cluster = TEST_UTIL.getHBaseCluster();
LOG.info("Waiting for active/ready master");
cluster.waitForActiveAndReadyMaster();
master = cluster.getMaster();
}
@Test
public void testMoveRegionWhenNotInitialized() {
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
HMaster m = cluster.getMaster();
try {
m.setInitialized(false); // fake it, set back later
HRegionInfo meta = HRegionInfo.FIRST_META_REGIONINFO;
m.move(meta.getEncodedNameAsBytes(), null);
fail("Region should not be moved since master is not initialized");
} catch (IOException ioe) {
assertTrue(ioe instanceof PleaseHoldException);
} finally {
m.setInitialized(true);
}
}
@Test(timeout = 300000)
public void testClusterRequests() throws Exception {
// sending fake request to master to see how metric value has changed
RegionServerStatusProtos.RegionServerReportRequest.Builder request =
RegionServerStatusProtos.RegionServerReportRequest.newBuilder();
ServerName serverName = cluster.getMaster(0).getServerName();
request.setServer(ProtobufUtil.toServerName(serverName));
MetricsMasterSource masterSource = master.getMasterMetrics().getMetricsSource();
ClusterStatusProtos.ServerLoad sl =
ClusterStatusProtos.ServerLoad.newBuilder().setTotalNumberOfRequests(10000).build();
masterSource.init();
request.setLoad(sl);
master.getMasterRpcServices().regionServerReport(null, request.build());
metricsHelper.assertCounter("cluster_requests", 10000, masterSource);
sl = ClusterStatusProtos.ServerLoad.newBuilder().setTotalNumberOfRequests(15000).build();
request.setLoad(sl);
master.getMasterRpcServices().regionServerReport(null, request.build());
metricsHelper.assertCounter("cluster_requests", 15000, masterSource);
master.getMasterRpcServices().regionServerReport(null, request.build());
metricsHelper.assertCounter("cluster_requests", 15000, masterSource);
master.stopMaster();
}
@Test
public void test() throws IOException, InterruptedException {
testUtil
.getHBaseAdmin()
.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString()).build());
Table table = testUtil.createTable(tableName, families);
table.put(
new Put(Bytes.toBytes("k")).addColumn(family, Bytes.toBytes("q"), Bytes.toBytes("v")));
MiniHBaseCluster cluster = testUtil.getMiniHBaseCluster();
List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
Region region = null;
for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
HRegionServer hrs = rsts.get(i).getRegionServer();
for (Region r : hrs.getOnlineRegions(tableName)) {
region = r;
break;
}
}
assertNotNull(region);
Thread.sleep(2000);
RegionStoreSequenceIds ids =
testUtil
.getHBaseCluster()
.getMaster()
.getLastSequenceId(region.getRegionInfo().getEncodedNameAsBytes());
assertEquals(HConstants.NO_SEQNUM, ids.getLastFlushedSequenceId());
// This will be the sequenceid just before that of the earliest edit in memstore.
long storeSequenceId = ids.getStoreSequenceId(0).getSequenceId();
assertTrue(storeSequenceId > 0);
testUtil.getHBaseAdmin().flush(tableName);
Thread.sleep(2000);
ids =
testUtil
.getHBaseCluster()
.getMaster()
.getLastSequenceId(region.getRegionInfo().getEncodedNameAsBytes());
assertTrue(
ids.getLastFlushedSequenceId() + " > " + storeSequenceId,
ids.getLastFlushedSequenceId() > storeSequenceId);
assertEquals(ids.getLastFlushedSequenceId(), ids.getStoreSequenceId(0).getSequenceId());
table.close();
}
@Test
public void testWholesomeMerge() throws Exception {
LOG.info("Starting testWholesomeMerge");
final TableName tableName = TableName.valueOf("testWholesomeMerge");
// Create table and load data.
Table table = createTableAndLoadData(master, tableName);
// Merge 1st and 2nd region
mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1, INITIAL_REGION_NUM - 1);
// Merge 2nd and 3th region
PairOfSameType<HRegionInfo> mergedRegions =
mergeRegionsAndVerifyRegionNum(master, tableName, 1, 2, INITIAL_REGION_NUM - 2);
verifyRowCount(table, ROWSIZE);
// Randomly choose one of the two merged regions
HRegionInfo hri =
RandomUtils.nextBoolean() ? mergedRegions.getFirst() : mergedRegions.getSecond();
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
AssignmentManager am = cluster.getMaster().getAssignmentManager();
RegionStates regionStates = am.getRegionStates();
long start = EnvironmentEdgeManager.currentTime();
while (!regionStates.isRegionInState(hri, State.MERGED)) {
assertFalse(
"Timed out in waiting one merged region to be in state MERGED",
EnvironmentEdgeManager.currentTime() - start > 60000);
Thread.sleep(500);
}
// We should not be able to assign it again
am.assign(hri, true);
assertFalse("Merged region can't be assigned", regionStates.isRegionInTransition(hri));
assertTrue(regionStates.isRegionInState(hri, State.MERGED));
// We should not be able to unassign it either
am.unassign(hri, null);
assertFalse("Merged region can't be unassigned", regionStates.isRegionInTransition(hri));
assertTrue(regionStates.isRegionInState(hri, State.MERGED));
table.close();
}
@Test
@SuppressWarnings("deprecation")
public void testMasterOpsWhileSplitting() throws Exception {
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
HMaster m = cluster.getMaster();
try (Table ht = TEST_UTIL.createTable(TABLENAME, FAMILYNAME)) {
assertTrue(m.getTableStateManager().isTableState(TABLENAME, TableState.State.ENABLED));
TEST_UTIL.loadTable(ht, FAMILYNAME, false);
}
List<Pair<HRegionInfo, ServerName>> tableRegions =
MetaTableAccessor.getTableRegionsAndLocations(m.getConnection(), TABLENAME);
LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions));
assertEquals(1, tableRegions.size());
assertArrayEquals(HConstants.EMPTY_START_ROW, tableRegions.get(0).getFirst().getStartKey());
assertArrayEquals(HConstants.EMPTY_END_ROW, tableRegions.get(0).getFirst().getEndKey());
// Now trigger a split and stop when the split is in progress
LOG.info("Splitting table");
TEST_UTIL.getAdmin().split(TABLENAME);
LOG.info("Waiting for split result to be about to open");
RegionStates regionStates = m.getAssignmentManager().getRegionStates();
while (regionStates.getRegionsOfTable(TABLENAME).size() <= 1) {
Thread.sleep(100);
}
LOG.info("Making sure we can call getTableRegions while opening");
tableRegions =
MetaTableAccessor.getTableRegionsAndLocations(m.getConnection(), TABLENAME, false);
LOG.info("Regions: " + Joiner.on(',').join(tableRegions));
// We have three regions because one is split-in-progress
assertEquals(3, tableRegions.size());
LOG.info("Making sure we can call getTableRegionClosest while opening");
Pair<HRegionInfo, ServerName> pair = m.getTableRegionForRow(TABLENAME, Bytes.toBytes("cde"));
LOG.info("Result is: " + pair);
Pair<HRegionInfo, ServerName> tableRegionFromName =
MetaTableAccessor.getRegion(m.getConnection(), pair.getFirst().getRegionName());
assertEquals(tableRegionFromName.getFirst(), pair.getFirst());
}
// check each region whether the coprocessor upcalls are called or not.
private void verifyMethodResult(
Class<?> c, String methodName[], TableName tableName, Object value[]) throws IOException {
try {
for (JVMClusterUtil.RegionServerThread t : cluster.getRegionServerThreads()) {
if (!t.isAlive() || t.getRegionServer().isAborted() || t.getRegionServer().isStopping()) {
continue;
}
for (HRegionInfo r : ProtobufUtil.getOnlineRegions(t.getRegionServer())) {
if (!r.getTable().equals(tableName)) {
continue;
}
RegionCoprocessorHost cph =
t.getRegionServer().getOnlineRegion(r.getRegionName()).getCoprocessorHost();
Coprocessor cp = cph.findCoprocessor(c.getName());
assertNotNull(cp);
for (int i = 0; i < methodName.length; ++i) {
Method m = c.getMethod(methodName[i]);
Object o = m.invoke(cp);
assertTrue(
"Result of "
+ c.getName()
+ "."
+ methodName[i]
+ " is expected to be "
+ value[i].toString()
+ ", while we get "
+ o.toString(),
o.equals(value[i]));
}
}
}
} catch (Exception e) {
throw new IOException(e.toString());
}
}
@Test(timeout = 60000)
public void testServerListener() throws IOException, InterruptedException {
ServerManager serverManager = TEST_UTIL.getHBaseCluster().getMaster().getServerManager();
DummyServerListener listener = new DummyServerListener();
serverManager.registerListener(listener);
try {
MiniHBaseCluster miniCluster = TEST_UTIL.getMiniHBaseCluster();
// Start a new Region Server
miniCluster.startRegionServer();
listener.awaitModifications(1);
assertEquals(1, listener.getAddedCount());
assertEquals(0, listener.getRemovedCount());
// Start another Region Server
listener.reset();
miniCluster.startRegionServer();
listener.awaitModifications(1);
assertEquals(1, listener.getAddedCount());
assertEquals(0, listener.getRemovedCount());
int nrs = miniCluster.getRegionServerThreads().size();
// Stop a Region Server
listener.reset();
miniCluster.stopRegionServer(nrs - 1);
listener.awaitModifications(1);
assertEquals(0, listener.getAddedCount());
assertEquals(1, listener.getRemovedCount());
// Stop another Region Server
listener.reset();
miniCluster.stopRegionServer(nrs - 2);
listener.awaitModifications(1);
assertEquals(0, listener.getAddedCount());
assertEquals(1, listener.getRemovedCount());
} finally {
serverManager.unregisterListener(listener);
}
}
/** This tests retaining assignments on a cluster restart */
@Test(timeout = 300000)
public void testRetainAssignmentOnRestart() throws Exception {
UTIL.startMiniCluster(2);
while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
Threads.sleep(1);
}
// Turn off balancer
UTIL.getMiniHBaseCluster().getMaster().getMasterRpcServices().synchronousBalanceSwitch(false);
LOG.info("\n\nCreating tables");
for (byte[] TABLE : TABLES) {
UTIL.createTable(TABLE, FAMILY);
}
for (byte[] TABLE : TABLES) {
UTIL.waitTableEnabled(TABLE);
}
HMaster master = UTIL.getMiniHBaseCluster().getMaster();
UTIL.waitUntilNoRegionsInTransition(120000);
// We don't have to use SnapshotOfRegionAssignmentFromMeta.
// We use it here because AM used to use it to load all user region placements
SnapshotOfRegionAssignmentFromMeta snapshot =
new SnapshotOfRegionAssignmentFromMeta(master.getShortCircuitConnection());
snapshot.initialize();
Map<HRegionInfo, ServerName> regionToRegionServerMap = snapshot.getRegionToRegionServerMap();
MiniHBaseCluster cluster = UTIL.getHBaseCluster();
List<JVMClusterUtil.RegionServerThread> threads = cluster.getLiveRegionServerThreads();
assertEquals(2, threads.size());
int[] rsPorts = new int[3];
for (int i = 0; i < 2; i++) {
rsPorts[i] = threads.get(i).getRegionServer().getServerName().getPort();
}
rsPorts[2] = cluster.getMaster().getServerName().getPort();
for (ServerName serverName : regionToRegionServerMap.values()) {
boolean found = false; // Test only, no need to optimize
for (int k = 0; k < 3 && !found; k++) {
found = serverName.getPort() == rsPorts[k];
}
assertTrue(found);
}
LOG.info("\n\nShutting down HBase cluster");
cluster.shutdown();
cluster.waitUntilShutDown();
LOG.info("\n\nSleeping a bit");
Thread.sleep(2000);
LOG.info("\n\nStarting cluster the second time with the same ports");
try {
cluster.getConf().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 4);
master = cluster.startMaster().getMaster();
for (int i = 0; i < 3; i++) {
cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, rsPorts[i]);
cluster.startRegionServer();
}
} finally {
// Reset region server port so as not to conflict with other tests
cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, 0);
cluster.getConf().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 2);
}
// Make sure live regionservers are on the same host/port
List<ServerName> localServers = master.getServerManager().getOnlineServersList();
assertEquals(4, localServers.size());
for (int i = 0; i < 3; i++) {
boolean found = false;
for (ServerName serverName : localServers) {
if (serverName.getPort() == rsPorts[i]) {
found = true;
break;
}
}
assertTrue(found);
}
// Wait till master is initialized and all regions are assigned
RegionStates regionStates = master.getAssignmentManager().getRegionStates();
int expectedRegions = regionToRegionServerMap.size() + 1;
while (!master.isInitialized()
|| regionStates.getRegionAssignments().size() != expectedRegions) {
Threads.sleep(100);
}
snapshot = new SnapshotOfRegionAssignmentFromMeta(master.getShortCircuitConnection());
snapshot.initialize();
Map<HRegionInfo, ServerName> newRegionToRegionServerMap = snapshot.getRegionToRegionServerMap();
assertEquals(regionToRegionServerMap.size(), newRegionToRegionServerMap.size());
for (Map.Entry<HRegionInfo, ServerName> entry : newRegionToRegionServerMap.entrySet()) {
if (TableName.NAMESPACE_TABLE_NAME.equals(entry.getKey().getTable())) continue;
ServerName oldServer = regionToRegionServerMap.get(entry.getKey());
ServerName currentServer = entry.getValue();
assertEquals(oldServer.getHostAndPort(), currentServer.getHostAndPort());
assertNotEquals(oldServer.getStartcode(), currentServer.getStartcode());
}
}
@Test
public void testRegionMerge() throws Exception {
String nsp1 = prefix + "_regiontest";
NamespaceDescriptor nspDesc =
NamespaceDescriptor.create(nsp1)
.addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3")
.addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
.build();
ADMIN.createNamespace(nspDesc);
final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
byte[] columnFamily = Bytes.toBytes("info");
HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo);
tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
final int initialRegions = 3;
ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("2000"), initialRegions);
Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
try (Table table = connection.getTable(tableTwo)) {
UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
}
ADMIN.flush(tableTwo);
List<HRegionInfo> hris = ADMIN.getTableRegions(tableTwo);
Collections.sort(hris);
// merge the two regions
final Set<String> encodedRegionNamesToMerge =
Sets.newHashSet(hris.get(0).getEncodedName(), hris.get(1).getEncodedName());
ADMIN.mergeRegions(
hris.get(0).getEncodedNameAsBytes(), hris.get(1).getEncodedNameAsBytes(), false);
UTIL.waitFor(
10000,
100,
new Waiter.ExplainingPredicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
RegionStates regionStates =
UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
return false;
}
if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
return false;
}
}
return true;
}
@Override
public String explainFailure() throws Exception {
RegionStates regionStates =
UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
return hri + " which is expected to be merged is still online";
}
if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
return hri + " is still in not opened";
}
}
return "Unknown";
}
});
hris = ADMIN.getTableRegions(tableTwo);
assertEquals(initialRegions - 1, hris.size());
Collections.sort(hris);
final HRegionInfo hriToSplit = hris.get(1);
ADMIN.split(tableTwo, Bytes.toBytes("500"));
UTIL.waitFor(
10000,
100,
new Waiter.ExplainingPredicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
RegionStates regionStates =
UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
return false;
}
if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
return false;
}
}
return true;
}
@Override
public String explainFailure() throws Exception {
RegionStates regionStates =
UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
return hriToSplit + " which is expected to be split is still online";
}
if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
return hri + " is still in not opened";
}
}
return "Unknown";
}
});
hris = ADMIN.getTableRegions(tableTwo);
assertEquals(initialRegions, hris.size());
Collections.sort(hris);
// fail region merge through Coprocessor hook
MiniHBaseCluster cluster = UTIL.getHBaseCluster();
HRegionServer regionServer = cluster.getRegionServer(0);
RegionServerCoprocessorHost cpHost = regionServer.getRegionServerCoprocessorHost();
Coprocessor coprocessor = cpHost.findCoprocessor(CPRegionServerObserver.class.getName());
CPRegionServerObserver regionServerObserver = (CPRegionServerObserver) coprocessor;
regionServerObserver.failMerge(true);
regionServerObserver.triggered = false;
ADMIN.mergeRegions(
hris.get(1).getEncodedNameAsBytes(), hris.get(2).getEncodedNameAsBytes(), false);
regionServerObserver.waitUtilTriggered();
hris = ADMIN.getTableRegions(tableTwo);
assertEquals(initialRegions, hris.size());
Collections.sort(hris);
// verify that we cannot split
HRegionInfo hriToSplit2 = hris.get(1);
ADMIN.split(
tableTwo,
TableInputFormatBase.getSplitKey(hriToSplit2.getStartKey(), hriToSplit2.getEndKey(), true));
Thread.sleep(2000);
assertEquals(initialRegions, ADMIN.getTableRegions(tableTwo).size());
}
@Test
public void testReducerNumEstimation() throws Exception {
// Skip the test for Tez. Tez use a different mechanism.
// Equivalent test is in TestTezAutoParallelism
Assume.assumeTrue("Skip this test for TEZ", Util.isMapredExecType(cluster.getExecType()));
// use the estimation
Configuration conf = HBaseConfiguration.create(new Configuration());
HBaseTestingUtility util = new HBaseTestingUtility(conf);
int clientPort = util.startMiniZKCluster().getClientPort();
util.startMiniHBaseCluster(1, 1);
String query = "a = load '/passwd';" + "b = group a by $0;" + "store b into 'output';";
PigServer ps = new PigServer(cluster.getExecType(), cluster.getProperties());
PhysicalPlan pp = Util.buildPp(ps, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
pc.getConf().setProperty("pig.exec.reducers.bytes.per.reducer", "100");
pc.getConf().setProperty("pig.exec.reducers.max", "10");
pc.getConf().setProperty(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort));
ConfigurationValidator.validatePigProperties(pc.getProperties());
conf = ConfigurationUtil.toConfiguration(pc.getProperties());
JobControlCompiler jcc = new JobControlCompiler(pc, conf);
JobControl jc = jcc.compile(mrPlan, "Test");
Job job = jc.getWaitingJobs().get(0);
long reducer =
Math.min(
(long) Math.ceil(new File("test/org/apache/pig/test/data/passwd").length() / 100.0),
10);
Util.assertParallelValues(-1, -1, reducer, reducer, job.getJobConf());
// use the PARALLEL key word, it will override the estimated reducer number
query = "a = load '/passwd';" + "b = group a by $0 PARALLEL 2;" + "store b into 'output';";
pp = Util.buildPp(ps, query);
mrPlan = Util.buildMRPlan(pp, pc);
pc.getConf().setProperty("pig.exec.reducers.bytes.per.reducer", "100");
pc.getConf().setProperty("pig.exec.reducers.max", "10");
ConfigurationValidator.validatePigProperties(pc.getProperties());
conf = ConfigurationUtil.toConfiguration(pc.getProperties());
jcc = new JobControlCompiler(pc, conf);
jc = jcc.compile(mrPlan, "Test");
job = jc.getWaitingJobs().get(0);
Util.assertParallelValues(-1, 2, -1, 2, job.getJobConf());
final byte[] COLUMNFAMILY = Bytes.toBytes("pig");
util.createTable(Bytes.toBytesBinary("test_table"), COLUMNFAMILY);
// the estimation won't take effect when it apply to non-dfs or the files doesn't exist, such as
// hbase
query =
"a = load 'hbase://test_table' using org.apache.pig.backend.hadoop.hbase.HBaseStorage('c:f1 c:f2');"
+ "b = group a by $0 ;"
+ "store b into 'output';";
pp = Util.buildPp(ps, query);
mrPlan = Util.buildMRPlan(pp, pc);
pc.getConf().setProperty("pig.exec.reducers.bytes.per.reducer", "100");
pc.getConf().setProperty("pig.exec.reducers.max", "10");
ConfigurationValidator.validatePigProperties(pc.getProperties());
conf = ConfigurationUtil.toConfiguration(pc.getProperties());
jcc = new JobControlCompiler(pc, conf);
jc = jcc.compile(mrPlan, "Test");
job = jc.getWaitingJobs().get(0);
Util.assertParallelValues(-1, -1, 1, 1, job.getJobConf());
util.deleteTable(Bytes.toBytesBinary("test_table"));
// In HBase 0.90.1 and above we can use util.shutdownMiniHBaseCluster()
// here instead.
MiniHBaseCluster hbc = util.getHBaseCluster();
if (hbc != null) {
hbc.shutdown();
hbc.join();
}
util.shutdownMiniZKCluster();
}
@Test(timeout = 60000)
public void testAssignmentListener() throws IOException, InterruptedException {
AssignmentManager am = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
Admin admin = TEST_UTIL.getHBaseAdmin();
DummyAssignmentListener listener = new DummyAssignmentListener();
am.registerListener(listener);
try {
final String TABLE_NAME_STR = "testtb";
final TableName TABLE_NAME = TableName.valueOf(TABLE_NAME_STR);
final byte[] FAMILY = Bytes.toBytes("cf");
// Create a new table, with a single region
LOG.info("Create Table");
TEST_UTIL.createTable(TABLE_NAME, FAMILY);
listener.awaitModifications(1);
assertEquals(1, listener.getLoadCount());
assertEquals(0, listener.getCloseCount());
// Add some data
HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLE_NAME);
try {
for (int i = 0; i < 10; ++i) {
byte[] key = Bytes.toBytes("row-" + i);
Put put = new Put(key);
put.add(FAMILY, null, key);
table.put(put);
}
} finally {
table.close();
}
// Split the table in two
LOG.info("Split Table");
listener.reset();
admin.split(TABLE_NAME_STR, "row-3");
listener.awaitModifications(3);
assertEquals(2, listener.getLoadCount()); // daughters added
assertEquals(1, listener.getCloseCount()); // parent removed
// Wait for the Regions to be mergeable
MiniHBaseCluster miniCluster = TEST_UTIL.getMiniHBaseCluster();
int mergeable = 0;
while (mergeable < 2) {
Thread.sleep(100);
admin.majorCompact(TABLE_NAME_STR);
mergeable = 0;
for (JVMClusterUtil.RegionServerThread regionThread :
miniCluster.getRegionServerThreads()) {
for (HRegion region : regionThread.getRegionServer().getOnlineRegions(TABLE_NAME)) {
mergeable += region.isMergeable() ? 1 : 0;
}
}
}
// Merge the two regions
LOG.info("Merge Regions");
listener.reset();
List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
assertEquals(2, regions.size());
admin.mergeRegions(
regions.get(0).getEncodedNameAsBytes(), regions.get(1).getEncodedNameAsBytes(), true);
listener.awaitModifications(3);
assertEquals(1, admin.getTableRegions(TABLE_NAME).size());
assertEquals(1, listener.getLoadCount()); // new merged region added
assertEquals(2, listener.getCloseCount()); // daughters removed
// Delete the table
LOG.info("Drop Table");
listener.reset();
TEST_UTIL.deleteTable(TABLE_NAME);
listener.awaitModifications(1);
assertEquals(0, listener.getLoadCount());
assertEquals(1, listener.getCloseCount());
} finally {
am.unregisterListener(listener);
}
}
@Test(timeout = 300000)
public void testDataCorrectnessReplayingRecoveredEdits() throws Exception {
final int NUM_MASTERS = 1;
final int NUM_RS = 3;
TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
try {
final byte[] TABLENAME = Bytes.toBytes("testDataCorrectnessReplayingRecoveredEdits");
final byte[] FAMILY = Bytes.toBytes("family");
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
HMaster master = cluster.getMaster();
// Create table
HTableDescriptor desc = new HTableDescriptor(TABLENAME);
desc.addFamily(new HColumnDescriptor(FAMILY));
HBaseAdmin hbaseAdmin = TEST_UTIL.getHBaseAdmin();
hbaseAdmin.createTable(desc);
assertTrue(hbaseAdmin.isTableAvailable(TABLENAME));
// Put data: r1->v1
Log.info("Loading r1 to v1 into " + Bytes.toString(TABLENAME));
HTable table = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
putDataAndVerify(table, "r1", FAMILY, "v1", 1);
// Move region to target server
HRegionInfo regionInfo = table.getRegionLocation("r1").getRegionInfo();
int originServerNum = cluster.getServerWith(regionInfo.getRegionName());
HRegionServer originServer = cluster.getRegionServer(originServerNum);
int targetServerNum = (originServerNum + 1) % NUM_RS;
HRegionServer targetServer = cluster.getRegionServer(targetServerNum);
assertFalse(originServer.equals(targetServer));
Log.info("Moving " + regionInfo.getEncodedName() + " to " + targetServer.getServerName());
hbaseAdmin.move(
regionInfo.getEncodedNameAsBytes(),
Bytes.toBytes(targetServer.getServerName().getServerName()));
do {
Thread.sleep(1);
} while (cluster.getServerWith(regionInfo.getRegionName()) == originServerNum);
// Put data: r2->v2
Log.info("Loading r2 to v2 into " + Bytes.toString(TABLENAME));
putDataAndVerify(table, "r2", FAMILY, "v2", 2);
// Move region to origin server
Log.info("Moving " + regionInfo.getEncodedName() + " to " + originServer.getServerName());
hbaseAdmin.move(
regionInfo.getEncodedNameAsBytes(),
Bytes.toBytes(originServer.getServerName().getServerName()));
do {
Thread.sleep(1);
} while (cluster.getServerWith(regionInfo.getRegionName()) == targetServerNum);
// Put data: r3->v3
Log.info("Loading r3 to v3 into " + Bytes.toString(TABLENAME));
putDataAndVerify(table, "r3", FAMILY, "v3", 3);
// Kill target server
Log.info("Killing target server " + targetServer.getServerName());
targetServer.kill();
cluster.getRegionServerThreads().get(targetServerNum).join();
// Wait until finish processing of shutdown
while (master.getServerManager().areDeadServersInProgress()) {
Thread.sleep(5);
}
// Kill origin server
Log.info("Killing origin server " + targetServer.getServerName());
originServer.kill();
cluster.getRegionServerThreads().get(originServerNum).join();
// Put data: r4->v4
Log.info("Loading r4 to v4 into " + Bytes.toString(TABLENAME));
putDataAndVerify(table, "r4", FAMILY, "v4", 4);
} finally {
TEST_UTIL.shutdownMiniCluster();
}
}
@Test(timeout = 300000)
public void testMultiSlaveReplication() throws Exception {
LOG.info("testCyclicReplication");
MiniHBaseCluster master = utility1.startMiniCluster();
utility2.startMiniCluster();
utility3.startMiniCluster();
ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
new HBaseAdmin(conf1).createTable(table);
new HBaseAdmin(conf2).createTable(table);
new HBaseAdmin(conf3).createTable(table);
HTable htable1 = new HTable(conf1, tableName);
htable1.setWriteBufferSize(1024);
HTable htable2 = new HTable(conf2, tableName);
htable2.setWriteBufferSize(1024);
HTable htable3 = new HTable(conf3, tableName);
htable3.setWriteBufferSize(1024);
admin1.addPeer("1", utility2.getClusterKey());
// put "row" and wait 'til it got around, then delete
putAndWait(row, famName, htable1, htable2);
deleteAndWait(row, htable1, htable2);
// check it wasn't replication to cluster 3
checkRow(row, 0, htable3);
putAndWait(row2, famName, htable1, htable2);
// now roll the region server's logs
new HBaseAdmin(conf1).rollHLogWriter(master.getRegionServer(0).getServerName().toString());
// after the log was rolled put a new row
putAndWait(row3, famName, htable1, htable2);
admin1.addPeer("2", utility3.getClusterKey());
// put a row, check it was replicated to all clusters
putAndWait(row1, famName, htable1, htable2, htable3);
// delete and verify
deleteAndWait(row1, htable1, htable2, htable3);
// make sure row2 did not get replicated after
// cluster 3 was added
checkRow(row2, 0, htable3);
// row3 will get replicated, because it was in the
// latest log
checkRow(row3, 1, htable3);
Put p = new Put(row);
p.add(famName, row, row);
htable1.put(p);
// now roll the logs again
new HBaseAdmin(conf1).rollHLogWriter(master.getRegionServer(0).getServerName().toString());
// cleanup "row2", also conveniently use this to wait replication
// to finish
deleteAndWait(row2, htable1, htable2, htable3);
// Even if the log was rolled in the middle of the replication
// "row" is still replication.
checkRow(row, 1, htable2, htable3);
// cleanup the rest
deleteAndWait(row, htable1, htable2, htable3);
deleteAndWait(row3, htable1, htable2, htable3);
utility3.shutdownMiniCluster();
utility2.shutdownMiniCluster();
utility1.shutdownMiniCluster();
}
/**
* Tests overriding compaction handling via coprocessor hooks
*
* @throws Exception
*/
@Test
public void testCompactionOverride() throws Exception {
byte[] compactTable = Bytes.toBytes("TestCompactionOverride");
HBaseAdmin admin = util.getHBaseAdmin();
if (admin.tableExists(compactTable)) {
admin.disableTable(compactTable);
admin.deleteTable(compactTable);
}
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(compactTable));
htd.addFamily(new HColumnDescriptor(A));
htd.addCoprocessor(EvenOnlyCompactor.class.getName());
admin.createTable(htd);
HTable table = new HTable(util.getConfiguration(), compactTable);
for (long i = 1; i <= 10; i++) {
byte[] iBytes = Bytes.toBytes(i);
Put put = new Put(iBytes);
put.setDurability(Durability.SKIP_WAL);
put.add(A, A, iBytes);
table.put(put);
}
HRegion firstRegion = cluster.getRegions(compactTable).get(0);
Coprocessor cp =
firstRegion.getCoprocessorHost().findCoprocessor(EvenOnlyCompactor.class.getName());
assertNotNull("EvenOnlyCompactor coprocessor should be loaded", cp);
EvenOnlyCompactor compactor = (EvenOnlyCompactor) cp;
// force a compaction
long ts = System.currentTimeMillis();
admin.flush(compactTable);
// wait for flush
for (int i = 0; i < 10; i++) {
if (compactor.lastFlush >= ts) {
break;
}
Thread.sleep(1000);
}
assertTrue("Flush didn't complete", compactor.lastFlush >= ts);
LOG.debug("Flush complete");
ts = compactor.lastFlush;
admin.majorCompact(compactTable);
// wait for compaction
for (int i = 0; i < 30; i++) {
if (compactor.lastCompaction >= ts) {
break;
}
Thread.sleep(1000);
}
LOG.debug("Last compaction was at " + compactor.lastCompaction);
assertTrue("Compaction didn't complete", compactor.lastCompaction >= ts);
// only even rows should remain
ResultScanner scanner = table.getScanner(new Scan());
try {
for (long i = 2; i <= 10; i += 2) {
Result r = scanner.next();
assertNotNull(r);
assertFalse(r.isEmpty());
byte[] iBytes = Bytes.toBytes(i);
assertArrayEquals("Row should be " + i, r.getRow(), iBytes);
assertArrayEquals("Value should be " + i, r.getValue(A, A), iBytes);
}
} finally {
scanner.close();
}
table.close();
}
@Test
public void testHostRank() throws Exception {
if (System.getProperty("prop.mapred.job.tracker") != null) {
if (LOG.isInfoEnabled())
LOG.info("testHBaseInputOutput: Ignore this test if not local mode.");
return;
}
File jarTest = new File(System.getProperty("prop.jarLocation"));
if (!jarTest.exists()) {
fail(
"Could not find Giraph jar at "
+ "location specified by 'prop.jarLocation'. "
+ "Make sure you built the main Giraph artifact?.");
}
MiniHBaseCluster cluster = null;
MiniZooKeeperCluster zkCluster = null;
FileSystem fs = null;
try {
// using the restart method allows us to avoid having the hbase
// root directory overwritten by /home/$username
zkCluster = testUtil.startMiniZKCluster();
testUtil.restartHBaseCluster(2);
cluster = testUtil.getMiniHBaseCluster();
final byte[] OL_BYTES = Bytes.toBytes("ol");
final byte[] S_BYTES = Bytes.toBytes("s");
final byte[] METADATA_BYTES = Bytes.toBytes("mtdt");
final byte[] HR_BYTES = Bytes.toBytes("_hr_");
final byte[] TAB = Bytes.toBytes(TABLE_NAME);
Configuration conf = cluster.getConfiguration();
HTableDescriptor desc = new HTableDescriptor(TAB);
desc.addFamily(new HColumnDescriptor(OL_BYTES));
desc.addFamily(new HColumnDescriptor(S_BYTES));
desc.addFamily(new HColumnDescriptor(METADATA_BYTES));
HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
if (hbaseAdmin.isTableAvailable(TABLE_NAME)) {
hbaseAdmin.disableTable(TABLE_NAME);
hbaseAdmin.deleteTable(TABLE_NAME);
}
hbaseAdmin.createTable(desc);
/**
* Enter the initial data (a,b), (b,c), (a,c) a = 1.0 - google b = 1.0 - yahoo c = 1.0 - bing
*/
HTable table = new HTable(conf, TABLE_NAME);
Put p1 = new Put(Bytes.toBytes("com.google.www"));
p1.add(OL_BYTES, Bytes.toBytes("www.yahoo.com"), Bytes.toBytes("ab"));
Put p2 = new Put(Bytes.toBytes("com.google.www"));
p2.add(OL_BYTES, Bytes.toBytes("www.bing.com"), Bytes.toBytes("ac"));
p2.add(OL_BYTES, Bytes.toBytes("www.bing.com"), Bytes.toBytes("invalid1"));
p2.add(OL_BYTES, Bytes.toBytes("www.google.com"), Bytes.toBytes("invalid2"));
Put p3 = new Put(Bytes.toBytes("com.yahoo.www"));
p3.add(OL_BYTES, Bytes.toBytes("www.bing.com"), Bytes.toBytes("bc"));
// p3.add(OL_BYTES, Bytes.toBytes(""), Bytes.toBytes("invalid4"));
Put p4 = new Put(Bytes.toBytes("com.bing.www"));
// TODO: Handle below case. use apache isValid method.
p4.add(OL_BYTES, Bytes.toBytes("http://invalidurl"), Bytes.toBytes("invalid5"));
p4.add(S_BYTES, S_BYTES, Bytes.toBytes(10.0d));
Put p5 = new Put(Bytes.toBytes("dummy"));
p5.add(S_BYTES, S_BYTES, Bytes.toBytes(10.0d));
table.put(p1);
table.put(p2);
table.put(p3);
table.put(p4);
table.put(p5);
// Set Giraph configuration
// now operate over HBase using Vertex I/O formats
conf.set(TableInputFormat.INPUT_TABLE, TABLE_NAME);
conf.set(TableOutputFormat.OUTPUT_TABLE, TABLE_NAME);
// Start the giraph job
GiraphJob giraphJob = new GiraphJob(conf, BspCase.getCallingMethodName());
GiraphConfiguration giraphConf = giraphJob.getConfiguration();
giraphConf.setZooKeeperConfiguration(cluster.getMaster().getZooKeeper().getQuorum());
setupConfiguration(giraphJob);
giraphConf.setComputationClass(LinkRankComputation.class);
giraphConf.setMasterComputeClass(LinkRankVertexMasterCompute.class);
giraphConf.setOutEdgesClass(ByteArrayEdges.class);
giraphConf.setVertexInputFormatClass(Nutch2HostInputFormat.class);
giraphConf.setVertexOutputFormatClass(Nutch2HostOutputFormat.class);
giraphConf.setInt("giraph.linkRank.superstepCount", 10);
giraphConf.setInt("giraph.linkRank.scale", 10);
giraphConf.set("giraph.linkRank.family", "mtdt");
giraphConf.set("giraph.linkRank.qualifier", "_hr_");
giraphConf.setVertexInputFilterClass(HostRankVertexFilter.class);
assertTrue(giraphJob.run(true));
if (LOG.isInfoEnabled()) LOG.info("Giraph job successful. Checking output qualifier.");
/** Check the results * */
Result result;
String key;
byte[] calculatedScoreByte;
HashMap expectedValues = new HashMap<String, Double>();
expectedValues.put("com.google.www", 1.3515060339386287d);
expectedValues.put("com.yahoo.www", 4.144902009567587d);
expectedValues.put("com.bing.www", 9.063893290511482d);
for (Object keyObject : expectedValues.keySet()) {
key = keyObject.toString();
result = table.get(new Get(key.getBytes()));
calculatedScoreByte = result.getValue(METADATA_BYTES, HR_BYTES);
assertNotNull(calculatedScoreByte);
assertTrue(calculatedScoreByte.length > 0);
Assert.assertEquals(
"Scores are not the same",
(Double) expectedValues.get(key),
Bytes.toDouble(calculatedScoreByte),
DELTA);
}
} finally {
if (cluster != null) {
cluster.shutdown();
}
if (zkCluster != null) {
zkCluster.shutdown();
}
// clean test files
if (fs != null) {
fs.delete(hbaseRootdir);
}
}
}