/**
* A region is online, won't be in transition any more. We can't confirm it is really online on
* specified region server because it hasn't been put in region server's online region list yet.
*/
public void regionOnline(final HRegionInfo hri, final ServerName serverName, long openSeqNum) {
String encodedName = hri.getEncodedName();
if (!serverManager.isServerOnline(serverName)) {
// This is possible if the region server dies before master gets a
// chance to handle ZK event in time. At this time, if the dead server
// is already processed by SSH, we should ignore this event.
// If not processed yet, ignore and let SSH deal with it.
LOG.warn("Ignored, " + encodedName + " was opened on a dead server: " + serverName);
return;
}
updateRegionState(hri, State.OPEN, serverName, openSeqNum);
synchronized (this) {
regionsInTransition.remove(encodedName);
ServerName oldServerName = regionAssignments.put(hri, serverName);
if (!serverName.equals(oldServerName)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Onlined " + hri.getShortNameToLog() + " on " + serverName);
}
addToServerHoldings(serverName, hri);
addToReplicaMapping(hri);
if (oldServerName == null) {
oldServerName = oldAssignments.remove(encodedName);
}
if (oldServerName != null
&& !oldServerName.equals(serverName)
&& serverHoldings.containsKey(oldServerName)) {
LOG.info("Offlined " + hri.getShortNameToLog() + " from " + oldServerName);
removeFromServerHoldings(oldServerName, hri);
}
}
}
}
synchronized boolean isServerDeadAndNotProcessed(ServerName server) {
if (server == null) return false;
if (serverManager.isServerOnline(server)) {
String hostAndPort = server.getHostAndPort();
long startCode = server.getStartcode();
Long deadCode = deadServers.get(hostAndPort);
if (deadCode == null || startCode > deadCode.longValue()) {
if (serverManager.isServerReachable(server)) {
return false;
}
// The size of deadServers won't grow unbounded.
deadServers.put(hostAndPort, Long.valueOf(startCode));
}
// Watch out! If the server is not dead, the region could
// remain unassigned. That's why ServerManager#isServerReachable
// should use some retry.
//
// We cache this info since it is very unlikely for that
// instance to come back up later on. We don't want to expire
// the server since we prefer to let it die naturally.
LOG.warn("Couldn't reach online server " + server);
}
// Now, we know it's dead. Check if it's processed
return !processedServers.containsKey(server);
}
/**
* Tests an on-the-fly RPC that was scheduled for the earlier RS on the same port for openRegion.
* The region server should reject this RPC. (HBASE-9721)
*/
@Test
public void testOpenCloseRegionRPCIntendedForPreviousServer() throws Exception {
Assert.assertTrue(getRS().getRegion(regionName).isAvailable());
ServerName sn = getRS().getServerName();
ServerName earlierServerName = ServerName.valueOf(sn.getHostname(), sn.getPort(), 1);
try {
CloseRegionRequest request =
RequestConverter.buildCloseRegionRequest(earlierServerName, regionName);
getRS().getRSRpcServices().closeRegion(null, request);
Assert.fail("The closeRegion should have been rejected");
} catch (ServiceException se) {
Assert.assertTrue(se.getCause() instanceof IOException);
Assert.assertTrue(
se.getCause().getMessage().contains("This RPC was intended for a different server"));
}
// actual close
closeRegionNoZK();
try {
AdminProtos.OpenRegionRequest orr =
RequestConverter.buildOpenRegionRequest(earlierServerName, hri, null, null);
getRS().getRSRpcServices().openRegion(null, orr);
Assert.fail("The openRegion should have been rejected");
} catch (ServiceException se) {
Assert.assertTrue(se.getCause() instanceof IOException);
Assert.assertTrue(
se.getCause().getMessage().contains("This RPC was intended for a different server"));
} finally {
openRegion(HTU, getRS(), hri);
}
}
/**
* Return how many regions will move per table since their primary RS will change
*
* @param newPlan - new AssignmentPlan
* @return how many primaries will move per table
*/
public Map<TableName, Integer> getRegionsMovement(FavoredNodesPlan newPlan) throws IOException {
Map<TableName, Integer> movesPerTable = new HashMap<TableName, Integer>();
SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot();
Map<TableName, List<HRegionInfo>> tableToRegions = snapshot.getTableToRegionMap();
FavoredNodesPlan oldPlan = snapshot.getExistingAssignmentPlan();
Set<TableName> tables = snapshot.getTableSet();
for (TableName table : tables) {
int movedPrimaries = 0;
if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) {
continue;
}
List<HRegionInfo> regions = tableToRegions.get(table);
for (HRegionInfo region : regions) {
List<ServerName> oldServers = oldPlan.getFavoredNodes(region);
List<ServerName> newServers = newPlan.getFavoredNodes(region);
if (oldServers != null && newServers != null) {
ServerName oldPrimary = oldServers.get(0);
ServerName newPrimary = newServers.get(0);
if (oldPrimary.compareTo(newPrimary) != 0) {
movedPrimaries++;
}
}
}
movesPerTable.put(table, movedPrimaries);
}
return movesPerTable;
}
public void reorderBlocks(Configuration conf, LocatedBlocks lbs, String src)
throws IOException {
ServerName sn = AbstractFSWALProvider.getServerNameFromWALDirectoryName(conf, src);
if (sn == null) {
// It's not an WAL
return;
}
// Ok, so it's an WAL
String hostName = sn.getHostname();
if (LOG.isTraceEnabled()) {
LOG.trace(src + " is an WAL file, so reordering blocks, last hostname will be:" + hostName);
}
// Just check for all blocks
for (LocatedBlock lb : lbs.getLocatedBlocks()) {
DatanodeInfo[] dnis = lb.getLocations();
if (dnis != null && dnis.length > 1) {
boolean found = false;
for (int i = 0; i < dnis.length - 1 && !found; i++) {
if (hostName.equals(dnis[i].getHostName())) {
// advance the other locations by one and put this one at the last place.
DatanodeInfo toLast = dnis[i];
System.arraycopy(dnis, i + 1, dnis, i, dnis.length - i - 1);
dnis[dnis.length - 1] = toLast;
found = true;
}
}
}
}
}
@Test
public void testPreWALRestoreSkip() throws Exception {
LOG.info(TestRegionObserverInterface.class.getName() + ".testPreWALRestoreSkip");
TableName tableName = TableName.valueOf(SimpleRegionObserver.TABLE_SKIPPED);
HTable table = util.createTable(tableName, new byte[][] {A, B, C});
JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
ServerName sn2 = rs1.getRegionServer().getServerName();
String regEN = table.getRegionLocations().firstEntry().getKey().getEncodedName();
util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes());
while (!sn2.equals(table.getRegionLocations().firstEntry().getValue())) {
Thread.sleep(100);
}
Put put = new Put(ROW);
put.add(A, A, A);
put.add(B, B, B);
put.add(C, C, C);
table.put(put);
table.flushCommits();
cluster.killRegionServer(rs1.getRegionServer().getServerName());
Threads.sleep(20000); // just to be sure that the kill has fully started.
util.waitUntilAllRegionsAssigned(tableName);
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"getCtPreWALRestore", "getCtPostWALRestore"},
tableName,
new Integer[] {0, 0});
util.deleteTable(tableName);
table.close();
}
@Override
public List<EndpointAffinity> getOperatorAffinity() {
watch.reset();
watch.start();
Map<String, DrillbitEndpoint> endpointMap = new HashMap<String, DrillbitEndpoint>();
for (DrillbitEndpoint ep : storagePlugin.getContext().getBits()) {
endpointMap.put(ep.getAddress(), ep);
}
Map<DrillbitEndpoint, EndpointAffinity> affinityMap =
new HashMap<DrillbitEndpoint, EndpointAffinity>();
for (ServerName sn : regionsToScan.values()) {
DrillbitEndpoint ep = endpointMap.get(sn.getHostname());
if (ep != null) {
EndpointAffinity affinity = affinityMap.get(ep);
if (affinity == null) {
affinityMap.put(ep, new EndpointAffinity(ep, 1));
} else {
affinity.addAffinity(1);
}
}
}
logger.debug("Took {} µs to get operator affinity", watch.elapsed(TimeUnit.NANOSECONDS) / 1000);
return Lists.newArrayList(affinityMap.values());
}
@Test
public void TestMap() throws Exception {
String prefix = "0000";
final String fileName = "19691231f2cd014ea28f42788214560a21a44cef";
final String mobFilePath = prefix + fileName;
ImmutableBytesWritable r = new ImmutableBytesWritable(Bytes.toBytes("r"));
final KeyValue[] kvList = new KeyValue[1];
kvList[0] =
new KeyValue(
Bytes.toBytes("row"),
Bytes.toBytes("family"),
Bytes.toBytes("column"),
Bytes.toBytes(mobFilePath));
Result columns = mock(Result.class);
when(columns.rawCells()).thenReturn(kvList);
Configuration configuration = new Configuration(TEST_UTIL.getConfiguration());
ZooKeeperWatcher zkw = new ZooKeeperWatcher(configuration, "1", new DummyMobAbortable());
TableName tn = TableName.valueOf("testSweepMapper");
TableName lockName = MobUtils.getTableLockName(tn);
String znode = ZKUtil.joinZNode(zkw.tableLockZNode, lockName.getNameAsString());
configuration.set(SweepJob.SWEEP_JOB_ID, "1");
configuration.set(SweepJob.SWEEP_JOB_TABLE_NODE, znode);
ServerName serverName = SweepJob.getCurrentServerName(configuration);
configuration.set(SweepJob.SWEEP_JOB_SERVERNAME, serverName.toString());
TableLockManager tableLockManager =
TableLockManager.createTableLockManager(configuration, zkw, serverName);
TableLock lock = tableLockManager.writeLock(lockName, "Run sweep tool");
lock.acquire();
try {
Mapper<ImmutableBytesWritable, Result, Text, KeyValue>.Context ctx =
mock(Mapper.Context.class);
when(ctx.getConfiguration()).thenReturn(configuration);
SweepMapper map = new SweepMapper();
doAnswer(
new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
Text text = (Text) invocation.getArguments()[0];
KeyValue kv = (KeyValue) invocation.getArguments()[1];
assertEquals(Bytes.toString(text.getBytes(), 0, text.getLength()), fileName);
assertEquals(0, Bytes.compareTo(kv.getKey(), kvList[0].getKey()));
return null;
}
})
.when(ctx)
.write(any(Text.class), any(KeyValue.class));
map.map(r, columns, ctx);
} finally {
lock.release();
}
}
public void printLocalityAndDispersionForCurrentPlan(
Map<String, Map<String, Float>> regionLocalityMap) throws IOException {
SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot();
FavoredNodesPlan assignmentPlan = snapshot.getExistingAssignmentPlan();
Set<TableName> tables = snapshot.getTableSet();
Map<TableName, List<HRegionInfo>> tableToRegionsMap = snapshot.getTableToRegionMap();
for (TableName table : tables) {
float[] locality = new float[3];
if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) {
continue;
}
List<HRegionInfo> regions = tableToRegionsMap.get(table);
for (HRegionInfo currentRegion : regions) {
Map<String, Float> regionLocality = regionLocalityMap.get(currentRegion.getEncodedName());
if (regionLocality == null) {
continue;
}
List<ServerName> servers = assignmentPlan.getFavoredNodes(currentRegion);
if (servers != null) {
int i = 0;
for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) {
ServerName server = servers.get(p.ordinal());
Float currentLocality = 0f;
if (servers != null) {
currentLocality = regionLocality.get(server.getHostname());
if (currentLocality == null) {
currentLocality = 0f;
}
locality[i] += currentLocality;
}
i++;
}
}
}
for (int i = 0; i < locality.length; i++) {
String copy = null;
if (i == 0) {
copy = "primary";
} else if (i == 1) {
copy = "secondary";
} else if (i == 2) {
copy = "tertiary";
}
float avgLocality = 100 * locality[i] / regions.size();
LOG.info(
"For Table: "
+ table
+ " ; #Total Regions: "
+ regions.size()
+ " ; The average locality for "
+ copy
+ " is "
+ avgLocality
+ " %");
}
printDispersionScores(table, snapshot, regions.size(), null, false);
}
}
private static Put addLocation(final Put p, final ServerName sn) {
p.add(
HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes(sn.getHostAndPort()));
p.add(
HConstants.CATALOG_FAMILY,
HConstants.STARTCODE_QUALIFIER,
Bytes.toBytes(sn.getStartcode()));
return p;
}
@Test
public void testRecovery() throws Exception {
LOG.info(TestRegionObserverInterface.class.getName() + ".testRecovery");
TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testRecovery");
HTable table = util.createTable(tableName, new byte[][] {A, B, C});
try {
JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
ServerName sn2 = rs1.getRegionServer().getServerName();
String regEN = table.getRegionLocations().firstEntry().getKey().getEncodedName();
util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes());
while (!sn2.equals(table.getRegionLocations().firstEntry().getValue())) {
Thread.sleep(100);
}
Put put = new Put(ROW);
put.add(A, A, A);
put.add(B, B, B);
put.add(C, C, C);
table.put(put);
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {
"hadPreGet",
"hadPostGet",
"hadPrePut",
"hadPostPut",
"hadPreBatchMutate",
"hadPostBatchMutate",
"hadDelete"
},
tableName,
new Boolean[] {false, false, true, true, true, true, false});
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut", "getCtPostPut"},
tableName,
new Integer[] {0, 0, 1, 1});
cluster.killRegionServer(rs1.getRegionServer().getServerName());
Threads.sleep(1000); // Let the kill soak in.
util.waitUntilAllRegionsAssigned(tableName);
LOG.info("All regions assigned");
verifyMethodResult(
SimpleRegionObserver.class,
new String[] {"getCtPrePut", "getCtPostPut"},
tableName,
new Integer[] {0, 0});
} finally {
util.deleteTable(tableName);
table.close();
}
}
public Put addLocation(final Put p, final ServerName sn, long openSeqNum) {
p.add(
HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes(sn.getHostAndPort()));
p.add(
HConstants.CATALOG_FAMILY,
HConstants.STARTCODE_QUALIFIER,
Bytes.toBytes(sn.getStartcode()));
p.add(HConstants.CATALOG_FAMILY, HConstants.SEQNUM_QUALIFIER, Bytes.toBytes(openSeqNum));
return p;
}
public void renderNoFlush(
@SuppressWarnings({"unused", "hiding"}) final java.io.Writer jamonWriter)
throws java.io.IOException {
// 35, 1
Collection<ServerName> backupMasters = null;
if (master.isActiveMaster()) {
ClusterStatus status = master.getClusterStatus();
backupMasters = status.getBackupMasters();
}
// 43, 1
jamonWriter.write("<table class=\"table table-striped\">\n");
// 44, 1
if ((backupMasters != null && backupMasters.size() > 0)) {
// 44, 59
jamonWriter.write(
"\n<tr>\n <th>ServerName</th>\n <th>Port</th>\n <th>Start Time</th>\n</tr>\n");
// 50, 1
ServerName[] serverNames = backupMasters.toArray(new ServerName[backupMasters.size()]);
Arrays.sort(serverNames);
for (ServerName serverName : serverNames) {
// 55, 1
jamonWriter.write("<tr>\n <td>");
// 56, 9
org.jamon.escaping.Escaping.HTML.write(
org.jamon.emit.StandardEmitter.valueOf(serverName.getHostname()), jamonWriter);
// 56, 39
jamonWriter.write("</td>\n <td>");
// 57, 9
org.jamon.escaping.Escaping.HTML.write(
org.jamon.emit.StandardEmitter.valueOf(serverName.getPort()), jamonWriter);
// 57, 35
jamonWriter.write("</td>\n <td>");
// 58, 9
org.jamon.escaping.Escaping.HTML.write(
org.jamon.emit.StandardEmitter.valueOf(new Date(serverName.getStartcode())),
jamonWriter);
// 58, 50
jamonWriter.write("</td>\n</tr>\n");
// 60, 1
}
}
// 63, 7
jamonWriter.write("\n<tr><td>Total:");
// 64, 15
org.jamon.escaping.Escaping.HTML.write(
org.jamon.emit.StandardEmitter.valueOf((backupMasters != null) ? backupMasters.size() : 0),
jamonWriter);
// 64, 71
jamonWriter.write("</td>\n</table>\n\n\n\n\n");
}
private void printHServerAddressSet(Set<ServerName> serverSet) {
if (serverSet == null) {
return;
}
int i = 0;
for (ServerName addr : serverSet) {
if ((i++) % 3 == 0) {
System.out.print("\n\t\t\t");
}
System.out.print(addr.getHostAndPort() + " ; ");
}
System.out.println("\n");
}
private Long getWriteRequestMetric(String tableName, ServerName serverName) {
Map<String, Long> serverMap = tableServerWriteRequestMap.get(tableName);
if (serverMap == null) {
serverMap = new HashMap<>();
tableServerWriteRequestMap.put(tableName, serverMap);
}
Long writeRequest = serverMap.get(serverName.getServerName());
if (writeRequest == null) {
writeRequest = 0L;
serverMap.put(serverName.getServerName(), writeRequest);
}
return writeRequest;
}
/**
* @param count
* @return Return <code>count</code> servernames.
*/
private static ServerName[] makeServerNames(final int count) {
ServerName[] sns = new ServerName[count];
for (int i = 0; i < count; i++) {
sns[i] = ServerName.valueOf("" + i + ".example.org", 60010, i);
}
return sns;
}
/**
* Inspect the log directory to recover any log file without
* an active region server.
*/
void splitLogAfterStartup() {
boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
HLog.SPLIT_SKIP_ERRORS_DEFAULT);
Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
do {
if (master.isStopped()) {
LOG.warn("Master stopped while splitting logs");
break;
}
List<ServerName> serverNames = new ArrayList<ServerName>();
try {
if (!this.fs.exists(logsDirPath)) return;
FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
// Get online servers after getting log folders to avoid log folder deletion of newly
// checked in region servers . see HBASE-5916
Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
.keySet();
if (logFolders == null || logFolders.length == 0) {
LOG.debug("No log files to split, proceeding...");
return;
}
for (FileStatus status : logFolders) {
String sn = status.getPath().getName();
// truncate splitting suffix if present (for ServerName parsing)
if (sn.endsWith(HLog.SPLITTING_EXT)) {
sn = sn.substring(0, sn.length() - HLog.SPLITTING_EXT.length());
}
ServerName serverName = ServerName.parseServerName(sn);
if (!onlineServers.contains(serverName)) {
LOG.info("Log folder " + status.getPath() + " doesn't belong "
+ "to a known region server, splitting");
serverNames.add(serverName);
} else {
LOG.info("Log folder " + status.getPath()
+ " belongs to an existing region server");
}
}
splitLog(serverNames);
retrySplitting = false;
} catch (IOException ioe) {
LOG.warn("Failed splitting of " + serverNames, ioe);
if (!checkFileSystem()) {
LOG.warn("Bad Filesystem, exiting");
Runtime.getRuntime().halt(1);
}
try {
if (retrySplitting) {
Thread.sleep(conf.getInt(
"hbase.hlog.split.failure.retry.interval", 30 * 1000));
}
} catch (InterruptedException e) {
LOG.warn("Interrupted, aborting since cannot return w/o splitting");
Thread.currentThread().interrupt();
retrySplitting = false;
Runtime.getRuntime().halt(1);
}
}
} while (retrySplitting);
}
static CellProtos.Cell getStartCode(final ByteString row) {
CellProtos.Cell.Builder cellBuilder = getBaseCellBuilder(row);
cellBuilder.setQualifier(HBaseZeroCopyByteString.wrap(HConstants.STARTCODE_QUALIFIER));
// TODO:
cellBuilder.setValue(
HBaseZeroCopyByteString.wrap(Bytes.toBytes(META_SERVERNAME.getStartcode())));
return cellBuilder.build();
}
private static String getPgPortEphemeralNodePath(ServerName sn, int port, boolean isMaster) {
String znode =
(isMaster ? "M" : "S")
+ ":"
+ sn.getHostAndPort()
+ Addressing.HOSTNAME_PORT_SEPARATOR
+ port;
return ZKUtil.joinZNode(ZooKeeperAdmin.PG_SERVER_NODE, znode);
}
/**
* @param favoredNodesStr The String of favored nodes
* @return the list of ServerName for the byte array of favored nodes.
*/
public static List<ServerName> getFavoredNodeList(String favoredNodesStr) {
String[] favoredNodesArray = StringUtils.split(favoredNodesStr, ",");
if (favoredNodesArray == null) return null;
List<ServerName> serverList = new ArrayList<ServerName>();
for (String hostNameAndPort : favoredNodesArray) {
serverList.add(ServerName.valueOf(hostNameAndPort, ServerName.NON_STARTCODE));
}
return serverList;
}
/**
* @param implementation An {@link HRegionInterface} instance; you'll likely want to pass a mocked
* HRS; can be null.
* @return Mock up a connection that returns a {@link org.apache.hadoop.conf.Configuration} when
* {@link HConnection#getConfiguration()} is called, a 'location' when {@link
* HConnection#getRegionLocation(byte[], byte[], boolean)} is called, and that returns the
* passed {@link HRegionInterface} instance when {@link
* HConnection#getHRegionConnection(String, int)} is called (Be sure call {@link
* HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)} when done with
* this mocked Connection.
* @throws IOException
*/
private HConnection mockConnection(final HRegionInterface implementation) throws IOException {
HConnection connection = HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
Mockito.doNothing().when(connection).close();
// Make it so we return any old location when asked.
final HRegionLocation anyLocation =
new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN.getHostname(), SN.getPort());
Mockito.when(
connection.getRegionLocation(
(byte[]) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean()))
.thenReturn(anyLocation);
Mockito.when(connection.locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any()))
.thenReturn(anyLocation);
if (implementation != null) {
// If a call to getHRegionConnection, return this implementation.
Mockito.when(connection.getHRegionConnection(Mockito.anyString(), Mockito.anyInt()))
.thenReturn(implementation);
}
return connection;
}
protected ServerAndLoad randomServer(final int numRegionsPerServer) {
if (!this.serverQueue.isEmpty()) {
ServerName sn = this.serverQueue.poll();
return new ServerAndLoad(sn, numRegionsPerServer);
}
String host = "srv" + rand.nextInt(Integer.MAX_VALUE);
int port = rand.nextInt(60000);
long startCode = rand.nextLong();
ServerName sn = ServerName.valueOf(host, port, startCode);
return new ServerAndLoad(sn, numRegionsPerServer);
}
@Test
public void testRemoveStaleRecoveringRegionsDuringMasterInitialization() throws Exception {
// this test is for when distributed log replay is enabled
if (!UTIL.getConfiguration().getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false)) return;
LOG.info("Starting testRemoveStaleRecoveringRegionsDuringMasterInitialization");
HMaster master = UTIL.getMiniHBaseCluster().getMaster();
MasterFileSystem fs = master.getMasterFileSystem();
String failedRegion = "failedRegoin1";
String staleRegion = "staleRegion";
ServerName inRecoveryServerName = ServerName.valueOf("mgr,1,1");
ServerName previouselyFaildServerName = ServerName.valueOf("previous,1,1");
String walPath =
"/hbase/data/.logs/" + inRecoveryServerName.getServerName() + "-splitting/test";
// Create a ZKW to use in the test
ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL);
zkw.getRecoverableZooKeeper()
.create(
ZKSplitLog.getEncodedNodeName(zkw, walPath),
new SplitLogTask.Owned(inRecoveryServerName, fs.getLogRecoveryMode()).toByteArray(),
Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
String staleRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, staleRegion);
ZKUtil.createWithParents(zkw, staleRegionPath);
String inRecoveringRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, failedRegion);
inRecoveringRegionPath =
ZKUtil.joinZNode(inRecoveringRegionPath, inRecoveryServerName.getServerName());
ZKUtil.createWithParents(zkw, inRecoveringRegionPath);
Set<ServerName> servers = new HashSet<ServerName>();
servers.add(previouselyFaildServerName);
fs.removeStaleRecoveringRegionsFromZK(servers);
// verification
assertFalse(ZKUtil.checkExists(zkw, staleRegionPath) != -1);
assertTrue(ZKUtil.checkExists(zkw, inRecoveringRegionPath) != -1);
ZKUtil.deleteChildrenRecursively(zkw, zkw.recoveringRegionsZNode);
ZKUtil.deleteChildrenRecursively(zkw, zkw.splitLogZNode);
zkw.close();
}
/**
* Get the list of all the region servers from the specified peer
*
* @param zkw zk connection to use
* @return list of region server addresses or an empty list if the slave is unavailable
*/
private static List<ServerName> fetchSlavesAddresses(ZooKeeperWatcher zkw)
throws KeeperException {
List<String> children = ZKUtil.listChildrenAndWatchForNewChildren(zkw, zkw.rsZNode);
if (children == null) {
return Collections.emptyList();
}
List<ServerName> addresses = new ArrayList<ServerName>(children.size());
for (String child : children) {
addresses.add(ServerName.parseServerName(child));
}
return addresses;
}
public static void addDaughter(
final CatalogTracker catalogTracker, final HRegionInfo regionInfo, final ServerName sn)
throws NotAllMetaRegionsOnlineException, IOException {
Put put = new Put(regionInfo.getRegionName());
addRegionInfo(put, regionInfo);
if (sn != null) addLocation(put, sn);
putToMetaTable(catalogTracker, put);
LOG.info(
"Added daughter "
+ regionInfo.getRegionNameAsString()
+ (sn == null ? ", serverName=null" : ", serverName=" + sn.toString()));
}
protected void deleteRegion(Configuration conf, final Table tbl, byte[] startKey, byte[] endKey)
throws IOException {
LOG.info("Before delete:");
HTableDescriptor htd = tbl.getTableDescriptor();
dumpMeta(htd);
List<HRegionLocation> regions;
try (RegionLocator rl = connection.getRegionLocator(tbl.getName())) {
regions = rl.getAllRegionLocations();
}
for (HRegionLocation e : regions) {
HRegionInfo hri = e.getRegionInfo();
ServerName hsa = e.getServerName();
if (Bytes.compareTo(hri.getStartKey(), startKey) == 0
&& Bytes.compareTo(hri.getEndKey(), endKey) == 0) {
LOG.info("RegionName: " + hri.getRegionNameAsString());
byte[] deleteRow = hri.getRegionName();
TEST_UTIL.getHBaseAdmin().unassign(deleteRow, true);
LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
Path rootDir = FSUtils.getRootDir(conf);
FileSystem fs = rootDir.getFileSystem(conf);
Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()), hri.getEncodedName());
fs.delete(p, true);
try (Table meta = this.connection.getTable(TableName.META_TABLE_NAME)) {
Delete delete = new Delete(deleteRow);
meta.delete(delete);
}
}
LOG.info(hri.toString() + hsa.toString());
}
TEST_UTIL.getMetaTableRows(htd.getTableName());
LOG.info("After delete:");
dumpMeta(htd);
}
@Test
public void testRPCException() throws Exception {
HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
TEST_UTIL.startMiniZKCluster();
Configuration conf = TEST_UTIL.getConfiguration();
conf.set(HConstants.MASTER_PORT, "0");
HMaster hm = new HMaster(conf);
ServerName sm = hm.getServerName();
InetSocketAddress isa = new InetSocketAddress(sm.getHostname(), sm.getPort());
int i = 0;
// retry the RPC a few times; we have seen SocketTimeoutExceptions if we
// try to connect too soon. Retry on SocketTimeoutException.
while (i < 20) {
try {
MasterMonitorProtocol inf =
(MasterMonitorProtocol)
HBaseClientRPC.getProxy(MasterMonitorProtocol.class, isa, conf, 100 * 10);
inf.isMasterRunning(null, IsMasterRunningRequest.getDefaultInstance());
fail();
} catch (ServiceException ex) {
IOException ie = ProtobufUtil.getRemoteException(ex);
if (!(ie instanceof SocketTimeoutException)) {
if (ie.getMessage()
.startsWith(
"org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet")) {
return;
}
} else {
System.err.println("Got SocketTimeoutException. Will retry. ");
}
} catch (Throwable t) {
fail("Unexpected throwable: " + t);
}
Thread.sleep(100);
i++;
}
fail();
}
/**
* Calls {@link #getMockedConnection(Configuration)} and then mocks a few more of the popular
* {@link HConnection} methods so they do 'normal' operation (see return doc below for list). Be
* sure to shutdown the connection when done by calling {@link
* HConnectionManager#deleteConnection(Configuration, boolean)} else it will stick around; this is
* probably not what you want.
*
* @param implementation An {@link HRegionInterface} instance; you'll likely want to pass a mocked
* HRS; can be null.
* @param conf Configuration to use
* @param implementation An HRegionInterface; can be null but is usually itself a mock.
* @param sn ServerName to include in the region location returned by this <code>implementation
* </code>
* @param hri HRegionInfo to include in the location returned when getRegionLocation is called on
* the mocked connection
* @return Mock up a connection that returns a {@link Configuration} when {@link
* HConnection#getConfiguration()} is called, a 'location' when {@link
* HConnection#getRegionLocation(byte[], byte[], boolean)} is called, and that returns the
* passed {@link HRegionInterface} instance when {@link
* HConnection#getHRegionConnection(String, int)} is called (Be sure call {@link
* HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration, boolean)} when
* done with this mocked Connection.
* @throws IOException
*/
public static HConnection getMockedConnectionAndDecorate(
final Configuration conf,
final HRegionInterface implementation,
final ServerName sn,
final HRegionInfo hri)
throws IOException {
HConnection c = HConnectionTestingUtility.getMockedConnection(conf);
Mockito.doNothing().when(c).close();
// Make it so we return a particular location when asked.
final HRegionLocation loc = new HRegionLocation(hri, sn.getHostname(), sn.getPort());
Mockito.when(
c.getRegionLocation(
(byte[]) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean()))
.thenReturn(loc);
Mockito.when(c.locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any())).thenReturn(loc);
if (implementation != null) {
// If a call to getHRegionConnection, return this implementation.
Mockito.when(c.getHRegionConnection(Mockito.anyString(), Mockito.anyInt()))
.thenReturn(implementation);
}
return c;
}
/**
* @return A mocked up Result that fakes a Get on a row in the <code>.META.</code> table.
* @throws IOException
*/
private Result getMetaTableRowResult() throws IOException {
List<KeyValue> kvs = new ArrayList<KeyValue>();
kvs.add(
new KeyValue(
HConstants.EMPTY_BYTE_ARRAY,
HConstants.CATALOG_FAMILY,
HConstants.REGIONINFO_QUALIFIER,
Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO)));
kvs.add(
new KeyValue(
HConstants.EMPTY_BYTE_ARRAY,
HConstants.CATALOG_FAMILY,
HConstants.SERVER_QUALIFIER,
Bytes.toBytes(SN.getHostAndPort())));
kvs.add(
new KeyValue(
HConstants.EMPTY_BYTE_ARRAY,
HConstants.CATALOG_FAMILY,
HConstants.STARTCODE_QUALIFIER,
Bytes.toBytes(SN.getStartcode())));
return new Result(kvs);
}
/**
* @param sn ServerName to get a connection against.
* @return The HRegionInterface we got when we connected to <code>sn</code> May have come from
* cache, may not be good, may have been setup by this invocation, or may be null.
* @throws IOException
*/
private HRegionInterface getCachedConnection(ServerName sn) throws IOException {
if (sn == null) {
return null;
}
HRegionInterface protocol = null;
try {
protocol = connection.getHRegionConnection(sn.getHostname(), sn.getPort());
} catch (RetriesExhaustedException e) {
if (e.getCause() != null && e.getCause() instanceof ConnectException) {
// Catch this; presume it means the cached connection has gone bad.
} else {
throw e;
}
} catch (SocketTimeoutException e) {
LOG.debug("Timed out connecting to " + sn);
} catch (NoRouteToHostException e) {
LOG.debug("Connecting to " + sn, e);
} catch (SocketException e) {
LOG.debug("Exception connecting to " + sn);
} catch (UnknownHostException e) {
LOG.debug("Unknown host exception connecting to " + sn);
} catch (IOException ioe) {
Throwable cause = ioe.getCause();
if (ioe instanceof ConnectException) {
// Catch. Connect refused.
} else if (cause != null && cause instanceof EOFException) {
// Catch. Other end disconnected us.
} else if (cause != null
&& cause.getMessage() != null
&& cause.getMessage().toLowerCase().contains("connection reset")) {
// Catch. Connection reset.
} else {
throw ioe;
}
}
return protocol;
}
