synchronized boolean isServerDeadAndNotProcessed(ServerName server) {
if (server == null) return false;
if (serverManager.isServerOnline(server)) {
String hostAndPort = server.getHostAndPort();
long startCode = server.getStartcode();
Long deadCode = deadServers.get(hostAndPort);
if (deadCode == null || startCode > deadCode.longValue()) {
if (serverManager.isServerReachable(server)) {
return false;
}
// The size of deadServers won't grow unbounded.
deadServers.put(hostAndPort, Long.valueOf(startCode));
}
// Watch out! If the server is not dead, the region could
// remain unassigned. That's why ServerManager#isServerReachable
// should use some retry.
//
// We cache this info since it is very unlikely for that
// instance to come back up later on. We don't want to expire
// the server since we prefer to let it die naturally.
LOG.warn("Couldn't reach online server " + server);
}
// Now, we know it's dead. Check if it's processed
return !processedServers.containsKey(server);
}
static CellProtos.Cell getStartCode(final ByteString row) {
CellProtos.Cell.Builder cellBuilder = getBaseCellBuilder(row);
cellBuilder.setQualifier(HBaseZeroCopyByteString.wrap(HConstants.STARTCODE_QUALIFIER));
// TODO:
cellBuilder.setValue(
HBaseZeroCopyByteString.wrap(Bytes.toBytes(META_SERVERNAME.getStartcode())));
return cellBuilder.build();
}
private static Put addLocation(final Put p, final ServerName sn) {
p.add(
HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes(sn.getHostAndPort()));
p.add(
HConstants.CATALOG_FAMILY,
HConstants.STARTCODE_QUALIFIER,
Bytes.toBytes(sn.getStartcode()));
return p;
}
public Put addLocation(final Put p, final ServerName sn, long openSeqNum) {
p.add(
HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes(sn.getHostAndPort()));
p.add(
HConstants.CATALOG_FAMILY,
HConstants.STARTCODE_QUALIFIER,
Bytes.toBytes(sn.getStartcode()));
p.add(HConstants.CATALOG_FAMILY, HConstants.SEQNUM_QUALIFIER, Bytes.toBytes(openSeqNum));
return p;
}
public void renderNoFlush(
@SuppressWarnings({"unused", "hiding"}) final java.io.Writer jamonWriter)
throws java.io.IOException {
// 35, 1
Collection<ServerName> backupMasters = null;
if (master.isActiveMaster()) {
ClusterStatus status = master.getClusterStatus();
backupMasters = status.getBackupMasters();
}
// 43, 1
jamonWriter.write("<table class=\"table table-striped\">\n");
// 44, 1
if ((backupMasters != null && backupMasters.size() > 0)) {
// 44, 59
jamonWriter.write(
"\n<tr>\n <th>ServerName</th>\n <th>Port</th>\n <th>Start Time</th>\n</tr>\n");
// 50, 1
ServerName[] serverNames = backupMasters.toArray(new ServerName[backupMasters.size()]);
Arrays.sort(serverNames);
for (ServerName serverName : serverNames) {
// 55, 1
jamonWriter.write("<tr>\n <td>");
// 56, 9
org.jamon.escaping.Escaping.HTML.write(
org.jamon.emit.StandardEmitter.valueOf(serverName.getHostname()), jamonWriter);
// 56, 39
jamonWriter.write("</td>\n <td>");
// 57, 9
org.jamon.escaping.Escaping.HTML.write(
org.jamon.emit.StandardEmitter.valueOf(serverName.getPort()), jamonWriter);
// 57, 35
jamonWriter.write("</td>\n <td>");
// 58, 9
org.jamon.escaping.Escaping.HTML.write(
org.jamon.emit.StandardEmitter.valueOf(new Date(serverName.getStartcode())),
jamonWriter);
// 58, 50
jamonWriter.write("</td>\n</tr>\n");
// 60, 1
}
}
// 63, 7
jamonWriter.write("\n<tr><td>Total:");
// 64, 15
org.jamon.escaping.Escaping.HTML.write(
org.jamon.emit.StandardEmitter.valueOf((backupMasters != null) ? backupMasters.size() : 0),
jamonWriter);
// 64, 71
jamonWriter.write("</td>\n</table>\n\n\n\n\n");
}
/**
* @return A mocked up Result that fakes a Get on a row in the <code>.META.</code> table.
* @throws IOException
*/
private Result getMetaTableRowResult() throws IOException {
List<KeyValue> kvs = new ArrayList<KeyValue>();
kvs.add(
new KeyValue(
HConstants.EMPTY_BYTE_ARRAY,
HConstants.CATALOG_FAMILY,
HConstants.REGIONINFO_QUALIFIER,
Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO)));
kvs.add(
new KeyValue(
HConstants.EMPTY_BYTE_ARRAY,
HConstants.CATALOG_FAMILY,
HConstants.SERVER_QUALIFIER,
Bytes.toBytes(SN.getHostAndPort())));
kvs.add(
new KeyValue(
HConstants.EMPTY_BYTE_ARRAY,
HConstants.CATALOG_FAMILY,
HConstants.STARTCODE_QUALIFIER,
Bytes.toBytes(SN.getStartcode())));
return new Result(kvs);
}
public void printStats() throws IOException {
Admin admin = connection.getAdmin();
ClusterStatus status =
admin.getClusterStatus(); // co ClusterStatusExample-1-GetStatus Get the cluster status.
System.out.println("Cluster Status:\n--------------");
System.out.println("HBase Version: " + status.getHBaseVersion());
System.out.println("Version: " + status.getVersion());
System.out.println("Cluster ID: " + status.getClusterId());
System.out.println("Master: " + status.getMaster());
System.out.println("No. Backup Masters: " + status.getBackupMastersSize());
System.out.println("Backup Masters: " + status.getBackupMasters());
System.out.println("No. Live Servers: " + status.getServersSize());
System.out.println("Servers: " + status.getServers());
System.out.println("No. Dead Servers: " + status.getDeadServers());
System.out.println("Dead Servers: " + status.getDeadServerNames());
System.out.println("No. Regions: " + status.getRegionsCount());
System.out.println("Regions in Transition: " + status.getRegionsInTransition());
System.out.println("No. Requests: " + status.getRequestsCount());
System.out.println("Avg Load: " + status.getAverageLoad());
System.out.println("Balancer On: " + status.getBalancerOn());
System.out.println("Is Balancer On: " + status.isBalancerOn());
System.out.println("Master Coprocessors: " + Arrays.asList(status.getMasterCoprocessors()));
System.out.println("\nServer Info:\n--------------");
for (ServerName server :
status
.getServers()) { // co ClusterStatusExample-2-ServerInfo Iterate over the included
// server instances.
System.out.println("Hostname: " + server.getHostname());
System.out.println("Host and Port: " + server.getHostAndPort());
System.out.println("Server Name: " + server.getServerName());
System.out.println("RPC Port: " + server.getPort());
System.out.println("Start Code: " + server.getStartcode());
ServerLoad load =
status.getLoad(
server); // co ClusterStatusExample-3-ServerLoad Retrieve the load details for the
// current server.
System.out.println("\nServer Load:\n--------------");
System.out.println("Info Port: " + load.getInfoServerPort());
System.out.println("Load: " + load.getLoad());
System.out.println("Max Heap (MB): " + load.getMaxHeapMB());
System.out.println("Used Heap (MB): " + load.getUsedHeapMB());
System.out.println("Memstore Size (MB): " + load.getMemstoreSizeInMB());
System.out.println("No. Regions: " + load.getNumberOfRegions());
System.out.println("No. Requests: " + load.getNumberOfRequests());
System.out.println("Total No. Requests: " + load.getTotalNumberOfRequests());
System.out.println("No. Requests per Sec: " + load.getRequestsPerSecond());
System.out.println("No. Read Requests: " + load.getReadRequestsCount());
System.out.println("No. Write Requests: " + load.getWriteRequestsCount());
System.out.println("No. Stores: " + load.getStores());
System.out.println("Store Size Uncompressed (MB): " + load.getStoreUncompressedSizeMB());
System.out.println("No. Storefiles: " + load.getStorefiles());
System.out.println("Storefile Size (MB): " + load.getStorefileSizeInMB());
System.out.println("Storefile Index Size (MB): " + load.getStorefileIndexSizeInMB());
System.out.println("Root Index Size: " + load.getRootIndexSizeKB());
System.out.println("Total Bloom Size: " + load.getTotalStaticBloomSizeKB());
System.out.println("Total Index Size: " + load.getTotalStaticIndexSizeKB());
System.out.println("Current Compacted Cells: " + load.getCurrentCompactedKVs());
System.out.println("Total Compacting Cells: " + load.getTotalCompactingKVs());
System.out.println("Coprocessors1: " + Arrays.asList(load.getRegionServerCoprocessors()));
System.out.println("Coprocessors2: " + Arrays.asList(load.getRsCoprocessors()));
System.out.println("Replication Load Sink: " + load.getReplicationLoadSink());
System.out.println("Replication Load Source: " + load.getReplicationLoadSourceList());
System.out.println("\nRegion Load:\n--------------");
for (Map.Entry<byte[], RegionLoad>
entry : // co ClusterStatusExample-4-Regions Iterate over the region details of the
// current server.
load.getRegionsLoad().entrySet()) {
System.out.println("Region: " + Bytes.toStringBinary(entry.getKey()));
RegionLoad regionLoad =
entry
.getValue(); // co ClusterStatusExample-5-RegionLoad Get the load details for the
// current region.
System.out.println("Name: " + Bytes.toStringBinary(regionLoad.getName()));
System.out.println("Name (as String): " + regionLoad.getNameAsString());
System.out.println("No. Requests: " + regionLoad.getRequestsCount());
System.out.println("No. Read Requests: " + regionLoad.getReadRequestsCount());
System.out.println("No. Write Requests: " + regionLoad.getWriteRequestsCount());
System.out.println("No. Stores: " + regionLoad.getStores());
System.out.println("No. Storefiles: " + regionLoad.getStorefiles());
System.out.println("Data Locality: " + regionLoad.getDataLocality());
System.out.println("Storefile Size (MB): " + regionLoad.getStorefileSizeMB());
System.out.println("Storefile Index Size (MB): " + regionLoad.getStorefileIndexSizeMB());
System.out.println("Memstore Size (MB): " + regionLoad.getMemStoreSizeMB());
System.out.println("Root Index Size: " + regionLoad.getRootIndexSizeKB());
System.out.println("Total Bloom Size: " + regionLoad.getTotalStaticBloomSizeKB());
System.out.println("Total Index Size: " + regionLoad.getTotalStaticIndexSizeKB());
System.out.println("Current Compacted Cells: " + regionLoad.getCurrentCompactedKVs());
System.out.println("Total Compacting Cells: " + regionLoad.getTotalCompactingKVs());
System.out.println();
}
}
}
/**
* Test that MetaReader will ride over server throwing "Server not running" IOEs.
*
* @see https://issues.apache.org/jira/browse/HBASE-3446
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testRideOverServerNotRunning() throws IOException, InterruptedException {
// Need a zk watcher.
ZooKeeperWatcher zkw =
new ZooKeeperWatcher(
UTIL.getConfiguration(), this.getClass().getSimpleName(), ABORTABLE, true);
// This is a servername we use in a few places below.
ServerName sn = new ServerName("example.com", 1234, System.currentTimeMillis());
HConnection connection = null;
CatalogTracker ct = null;
try {
// Mock an HRegionInterface. Our mock implementation will fail a few
// times when we go to open a scanner.
final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
// When openScanner called throw IOE 'Server not running' a few times
// before we return a scanner id. Whats WEIRD is that these
// exceptions do not show in the log because they are caught and only
// printed if we FAIL. We eventually succeed after retry so these don't
// show. We will know if they happened or not because we will ask
// mockito at the end of this test to verify that openscanner was indeed
// called the wanted number of times.
final long scannerid = 123L;
Mockito.when(implementation.openScanner((byte[]) Mockito.any(), (Scan) Mockito.any()))
.thenThrow(new IOException("Server not running (1 of 3)"))
.thenThrow(new IOException("Server not running (2 of 3)"))
.thenThrow(new IOException("Server not running (3 of 3)"))
.thenReturn(scannerid);
// Make it so a verifiable answer comes back when next is called. Return
// the verifiable answer and then a null so we stop scanning. Our
// verifiable answer is something that looks like a row in META with
// a server and startcode that is that of the above defined servername.
List<KeyValue> kvs = new ArrayList<KeyValue>();
final byte[] rowToVerify = Bytes.toBytes("rowToVerify");
kvs.add(
new KeyValue(
rowToVerify,
HConstants.CATALOG_FAMILY,
HConstants.REGIONINFO_QUALIFIER,
Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO)));
kvs.add(
new KeyValue(
rowToVerify,
HConstants.CATALOG_FAMILY,
HConstants.SERVER_QUALIFIER,
Bytes.toBytes(sn.getHostAndPort())));
kvs.add(
new KeyValue(
rowToVerify,
HConstants.CATALOG_FAMILY,
HConstants.STARTCODE_QUALIFIER,
Bytes.toBytes(sn.getStartcode())));
final Result[] result = new Result[] {new Result(kvs)};
Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt()))
.thenReturn(result)
.thenReturn(null);
// Associate a spied-upon HConnection with UTIL.getConfiguration. Need
// to shove this in here first so it gets picked up all over; e.g. by
// HTable.
connection = HConnectionTestingUtility.getSpiedConnection(UTIL.getConfiguration());
// Fix the location lookup so it 'works' though no network. First
// make an 'any location' object.
final HRegionLocation anyLocation =
new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, sn.getHostname(), sn.getPort());
// Return the any location object when locateRegion is called in HTable
// constructor and when its called by ServerCallable (it uses getRegionLocation).
// The ugly format below comes of 'Important gotcha on spying real objects!' from
// http://mockito.googlecode.com/svn/branches/1.6/javadoc/org/mockito/Mockito.html
Mockito.doReturn(anyLocation)
.when(connection)
.locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any());
Mockito.doReturn(anyLocation)
.when(connection)
.getRegionLocation((byte[]) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean());
// Now shove our HRI implementation into the spied-upon connection.
Mockito.doReturn(implementation)
.when(connection)
.getHRegionConnection(Mockito.anyString(), Mockito.anyInt());
// Now start up the catalogtracker with our doctored Connection.
ct = new CatalogTracker(zkw, null, connection, ABORTABLE, 0);
ct.start();
// Scan meta for user tables and verify we got back expected answer.
NavigableMap<HRegionInfo, Result> hris = MetaReader.getServerUserRegions(ct, sn);
assertTrue(hris.size() == 1);
assertTrue(hris.firstEntry().getKey().equals(HRegionInfo.FIRST_META_REGIONINFO));
assertTrue(Bytes.equals(rowToVerify, hris.firstEntry().getValue().getRow()));
// Finally verify that openscanner was called four times -- three times
// with exception and then on 4th attempt we succeed.
Mockito.verify(implementation, Mockito.times(4))
.openScanner((byte[]) Mockito.any(), (Scan) Mockito.any());
} finally {
if (ct != null) ct.stop();
HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
zkw.close();
}
}
/** This tests retaining assignments on a cluster restart */
@Test(timeout = 300000)
public void testRetainAssignmentOnRestart() throws Exception {
UTIL.startMiniCluster(2);
while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
Threads.sleep(1);
}
// Turn off balancer
UTIL.getMiniHBaseCluster().getMaster().getMasterRpcServices().synchronousBalanceSwitch(false);
LOG.info("\n\nCreating tables");
for (byte[] TABLE : TABLES) {
UTIL.createTable(TABLE, FAMILY);
}
for (byte[] TABLE : TABLES) {
UTIL.waitTableEnabled(TABLE);
}
HMaster master = UTIL.getMiniHBaseCluster().getMaster();
UTIL.waitUntilNoRegionsInTransition(120000);
// We don't have to use SnapshotOfRegionAssignmentFromMeta.
// We use it here because AM used to use it to load all user region placements
SnapshotOfRegionAssignmentFromMeta snapshot =
new SnapshotOfRegionAssignmentFromMeta(master.getShortCircuitConnection());
snapshot.initialize();
Map<HRegionInfo, ServerName> regionToRegionServerMap = snapshot.getRegionToRegionServerMap();
MiniHBaseCluster cluster = UTIL.getHBaseCluster();
List<JVMClusterUtil.RegionServerThread> threads = cluster.getLiveRegionServerThreads();
assertEquals(2, threads.size());
int[] rsPorts = new int[3];
for (int i = 0; i < 2; i++) {
rsPorts[i] = threads.get(i).getRegionServer().getServerName().getPort();
}
rsPorts[2] = cluster.getMaster().getServerName().getPort();
for (ServerName serverName : regionToRegionServerMap.values()) {
boolean found = false; // Test only, no need to optimize
for (int k = 0; k < 3 && !found; k++) {
found = serverName.getPort() == rsPorts[k];
}
assertTrue(found);
}
LOG.info("\n\nShutting down HBase cluster");
cluster.shutdown();
cluster.waitUntilShutDown();
LOG.info("\n\nSleeping a bit");
Thread.sleep(2000);
LOG.info("\n\nStarting cluster the second time with the same ports");
try {
cluster.getConf().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 4);
master = cluster.startMaster().getMaster();
for (int i = 0; i < 3; i++) {
cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, rsPorts[i]);
cluster.startRegionServer();
}
} finally {
// Reset region server port so as not to conflict with other tests
cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, 0);
cluster.getConf().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 2);
}
// Make sure live regionservers are on the same host/port
List<ServerName> localServers = master.getServerManager().getOnlineServersList();
assertEquals(4, localServers.size());
for (int i = 0; i < 3; i++) {
boolean found = false;
for (ServerName serverName : localServers) {
if (serverName.getPort() == rsPorts[i]) {
found = true;
break;
}
}
assertTrue(found);
}
// Wait till master is initialized and all regions are assigned
RegionStates regionStates = master.getAssignmentManager().getRegionStates();
int expectedRegions = regionToRegionServerMap.size() + 1;
while (!master.isInitialized()
|| regionStates.getRegionAssignments().size() != expectedRegions) {
Threads.sleep(100);
}
snapshot = new SnapshotOfRegionAssignmentFromMeta(master.getShortCircuitConnection());
snapshot.initialize();
Map<HRegionInfo, ServerName> newRegionToRegionServerMap = snapshot.getRegionToRegionServerMap();
assertEquals(regionToRegionServerMap.size(), newRegionToRegionServerMap.size());
for (Map.Entry<HRegionInfo, ServerName> entry : newRegionToRegionServerMap.entrySet()) {
if (TableName.NAMESPACE_TABLE_NAME.equals(entry.getKey().getTable())) continue;
ServerName oldServer = regionToRegionServerMap.get(entry.getKey());
ServerName currentServer = entry.getValue();
assertEquals(oldServer.getHostAndPort(), currentServer.getHostAndPort());
assertNotEquals(oldServer.getStartcode(), currentServer.getStartcode());
}
}
