synchronized boolean isServerDeadAndNotProcessed(ServerName server) { if (server == null) return false; if (serverManager.isServerOnline(server)) { String hostAndPort = server.getHostAndPort(); long startCode = server.getStartcode(); Long deadCode = deadServers.get(hostAndPort); if (deadCode == null || startCode > deadCode.longValue()) { if (serverManager.isServerReachable(server)) { return false; } // The size of deadServers won't grow unbounded. deadServers.put(hostAndPort, Long.valueOf(startCode)); } // Watch out! If the server is not dead, the region could // remain unassigned. That's why ServerManager#isServerReachable // should use some retry. // // We cache this info since it is very unlikely for that // instance to come back up later on. We don't want to expire // the server since we prefer to let it die naturally. LOG.warn("Couldn't reach online server " + server); } // Now, we know it's dead. Check if it's processed return !processedServers.containsKey(server); }
static CellProtos.Cell getStartCode(final ByteString row) { CellProtos.Cell.Builder cellBuilder = getBaseCellBuilder(row); cellBuilder.setQualifier(HBaseZeroCopyByteString.wrap(HConstants.STARTCODE_QUALIFIER)); // TODO: cellBuilder.setValue( HBaseZeroCopyByteString.wrap(Bytes.toBytes(META_SERVERNAME.getStartcode()))); return cellBuilder.build(); }
private static Put addLocation(final Put p, final ServerName sn) { p.add( HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes(sn.getHostAndPort())); p.add( HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(sn.getStartcode())); return p; }
public Put addLocation(final Put p, final ServerName sn, long openSeqNum) { p.add( HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes(sn.getHostAndPort())); p.add( HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(sn.getStartcode())); p.add(HConstants.CATALOG_FAMILY, HConstants.SEQNUM_QUALIFIER, Bytes.toBytes(openSeqNum)); return p; }
public void renderNoFlush( @SuppressWarnings({"unused", "hiding"}) final java.io.Writer jamonWriter) throws java.io.IOException { // 35, 1 Collection<ServerName> backupMasters = null; if (master.isActiveMaster()) { ClusterStatus status = master.getClusterStatus(); backupMasters = status.getBackupMasters(); } // 43, 1 jamonWriter.write("<table class=\"table table-striped\">\n"); // 44, 1 if ((backupMasters != null && backupMasters.size() > 0)) { // 44, 59 jamonWriter.write( "\n<tr>\n <th>ServerName</th>\n <th>Port</th>\n <th>Start Time</th>\n</tr>\n"); // 50, 1 ServerName[] serverNames = backupMasters.toArray(new ServerName[backupMasters.size()]); Arrays.sort(serverNames); for (ServerName serverName : serverNames) { // 55, 1 jamonWriter.write("<tr>\n <td>"); // 56, 9 org.jamon.escaping.Escaping.HTML.write( org.jamon.emit.StandardEmitter.valueOf(serverName.getHostname()), jamonWriter); // 56, 39 jamonWriter.write("</td>\n <td>"); // 57, 9 org.jamon.escaping.Escaping.HTML.write( org.jamon.emit.StandardEmitter.valueOf(serverName.getPort()), jamonWriter); // 57, 35 jamonWriter.write("</td>\n <td>"); // 58, 9 org.jamon.escaping.Escaping.HTML.write( org.jamon.emit.StandardEmitter.valueOf(new Date(serverName.getStartcode())), jamonWriter); // 58, 50 jamonWriter.write("</td>\n</tr>\n"); // 60, 1 } } // 63, 7 jamonWriter.write("\n<tr><td>Total:"); // 64, 15 org.jamon.escaping.Escaping.HTML.write( org.jamon.emit.StandardEmitter.valueOf((backupMasters != null) ? backupMasters.size() : 0), jamonWriter); // 64, 71 jamonWriter.write("</td>\n</table>\n\n\n\n\n"); }
/** * @return A mocked up Result that fakes a Get on a row in the <code>.META.</code> table. * @throws IOException */ private Result getMetaTableRowResult() throws IOException { List<KeyValue> kvs = new ArrayList<KeyValue>(); kvs.add( new KeyValue( HConstants.EMPTY_BYTE_ARRAY, HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO))); kvs.add( new KeyValue( HConstants.EMPTY_BYTE_ARRAY, HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes(SN.getHostAndPort()))); kvs.add( new KeyValue( HConstants.EMPTY_BYTE_ARRAY, HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(SN.getStartcode()))); return new Result(kvs); }
public void printStats() throws IOException { Admin admin = connection.getAdmin(); ClusterStatus status = admin.getClusterStatus(); // co ClusterStatusExample-1-GetStatus Get the cluster status. System.out.println("Cluster Status:\n--------------"); System.out.println("HBase Version: " + status.getHBaseVersion()); System.out.println("Version: " + status.getVersion()); System.out.println("Cluster ID: " + status.getClusterId()); System.out.println("Master: " + status.getMaster()); System.out.println("No. Backup Masters: " + status.getBackupMastersSize()); System.out.println("Backup Masters: " + status.getBackupMasters()); System.out.println("No. Live Servers: " + status.getServersSize()); System.out.println("Servers: " + status.getServers()); System.out.println("No. Dead Servers: " + status.getDeadServers()); System.out.println("Dead Servers: " + status.getDeadServerNames()); System.out.println("No. Regions: " + status.getRegionsCount()); System.out.println("Regions in Transition: " + status.getRegionsInTransition()); System.out.println("No. Requests: " + status.getRequestsCount()); System.out.println("Avg Load: " + status.getAverageLoad()); System.out.println("Balancer On: " + status.getBalancerOn()); System.out.println("Is Balancer On: " + status.isBalancerOn()); System.out.println("Master Coprocessors: " + Arrays.asList(status.getMasterCoprocessors())); System.out.println("\nServer Info:\n--------------"); for (ServerName server : status .getServers()) { // co ClusterStatusExample-2-ServerInfo Iterate over the included // server instances. System.out.println("Hostname: " + server.getHostname()); System.out.println("Host and Port: " + server.getHostAndPort()); System.out.println("Server Name: " + server.getServerName()); System.out.println("RPC Port: " + server.getPort()); System.out.println("Start Code: " + server.getStartcode()); ServerLoad load = status.getLoad( server); // co ClusterStatusExample-3-ServerLoad Retrieve the load details for the // current server. System.out.println("\nServer Load:\n--------------"); System.out.println("Info Port: " + load.getInfoServerPort()); System.out.println("Load: " + load.getLoad()); System.out.println("Max Heap (MB): " + load.getMaxHeapMB()); System.out.println("Used Heap (MB): " + load.getUsedHeapMB()); System.out.println("Memstore Size (MB): " + load.getMemstoreSizeInMB()); System.out.println("No. Regions: " + load.getNumberOfRegions()); System.out.println("No. Requests: " + load.getNumberOfRequests()); System.out.println("Total No. Requests: " + load.getTotalNumberOfRequests()); System.out.println("No. Requests per Sec: " + load.getRequestsPerSecond()); System.out.println("No. Read Requests: " + load.getReadRequestsCount()); System.out.println("No. Write Requests: " + load.getWriteRequestsCount()); System.out.println("No. Stores: " + load.getStores()); System.out.println("Store Size Uncompressed (MB): " + load.getStoreUncompressedSizeMB()); System.out.println("No. Storefiles: " + load.getStorefiles()); System.out.println("Storefile Size (MB): " + load.getStorefileSizeInMB()); System.out.println("Storefile Index Size (MB): " + load.getStorefileIndexSizeInMB()); System.out.println("Root Index Size: " + load.getRootIndexSizeKB()); System.out.println("Total Bloom Size: " + load.getTotalStaticBloomSizeKB()); System.out.println("Total Index Size: " + load.getTotalStaticIndexSizeKB()); System.out.println("Current Compacted Cells: " + load.getCurrentCompactedKVs()); System.out.println("Total Compacting Cells: " + load.getTotalCompactingKVs()); System.out.println("Coprocessors1: " + Arrays.asList(load.getRegionServerCoprocessors())); System.out.println("Coprocessors2: " + Arrays.asList(load.getRsCoprocessors())); System.out.println("Replication Load Sink: " + load.getReplicationLoadSink()); System.out.println("Replication Load Source: " + load.getReplicationLoadSourceList()); System.out.println("\nRegion Load:\n--------------"); for (Map.Entry<byte[], RegionLoad> entry : // co ClusterStatusExample-4-Regions Iterate over the region details of the // current server. load.getRegionsLoad().entrySet()) { System.out.println("Region: " + Bytes.toStringBinary(entry.getKey())); RegionLoad regionLoad = entry .getValue(); // co ClusterStatusExample-5-RegionLoad Get the load details for the // current region. System.out.println("Name: " + Bytes.toStringBinary(regionLoad.getName())); System.out.println("Name (as String): " + regionLoad.getNameAsString()); System.out.println("No. Requests: " + regionLoad.getRequestsCount()); System.out.println("No. Read Requests: " + regionLoad.getReadRequestsCount()); System.out.println("No. Write Requests: " + regionLoad.getWriteRequestsCount()); System.out.println("No. Stores: " + regionLoad.getStores()); System.out.println("No. Storefiles: " + regionLoad.getStorefiles()); System.out.println("Data Locality: " + regionLoad.getDataLocality()); System.out.println("Storefile Size (MB): " + regionLoad.getStorefileSizeMB()); System.out.println("Storefile Index Size (MB): " + regionLoad.getStorefileIndexSizeMB()); System.out.println("Memstore Size (MB): " + regionLoad.getMemStoreSizeMB()); System.out.println("Root Index Size: " + regionLoad.getRootIndexSizeKB()); System.out.println("Total Bloom Size: " + regionLoad.getTotalStaticBloomSizeKB()); System.out.println("Total Index Size: " + regionLoad.getTotalStaticIndexSizeKB()); System.out.println("Current Compacted Cells: " + regionLoad.getCurrentCompactedKVs()); System.out.println("Total Compacting Cells: " + regionLoad.getTotalCompactingKVs()); System.out.println(); } } }
/** * Test that MetaReader will ride over server throwing "Server not running" IOEs. * * @see https://issues.apache.org/jira/browse/HBASE-3446 * @throws IOException * @throws InterruptedException */ @Test public void testRideOverServerNotRunning() throws IOException, InterruptedException { // Need a zk watcher. ZooKeeperWatcher zkw = new ZooKeeperWatcher( UTIL.getConfiguration(), this.getClass().getSimpleName(), ABORTABLE, true); // This is a servername we use in a few places below. ServerName sn = new ServerName("example.com", 1234, System.currentTimeMillis()); HConnection connection = null; CatalogTracker ct = null; try { // Mock an HRegionInterface. Our mock implementation will fail a few // times when we go to open a scanner. final HRegionInterface implementation = Mockito.mock(HRegionInterface.class); // When openScanner called throw IOE 'Server not running' a few times // before we return a scanner id. Whats WEIRD is that these // exceptions do not show in the log because they are caught and only // printed if we FAIL. We eventually succeed after retry so these don't // show. We will know if they happened or not because we will ask // mockito at the end of this test to verify that openscanner was indeed // called the wanted number of times. final long scannerid = 123L; Mockito.when(implementation.openScanner((byte[]) Mockito.any(), (Scan) Mockito.any())) .thenThrow(new IOException("Server not running (1 of 3)")) .thenThrow(new IOException("Server not running (2 of 3)")) .thenThrow(new IOException("Server not running (3 of 3)")) .thenReturn(scannerid); // Make it so a verifiable answer comes back when next is called. Return // the verifiable answer and then a null so we stop scanning. Our // verifiable answer is something that looks like a row in META with // a server and startcode that is that of the above defined servername. List<KeyValue> kvs = new ArrayList<KeyValue>(); final byte[] rowToVerify = Bytes.toBytes("rowToVerify"); kvs.add( new KeyValue( rowToVerify, HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO))); kvs.add( new KeyValue( rowToVerify, HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes(sn.getHostAndPort()))); kvs.add( new KeyValue( rowToVerify, HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(sn.getStartcode()))); final Result[] result = new Result[] {new Result(kvs)}; Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt())) .thenReturn(result) .thenReturn(null); // Associate a spied-upon HConnection with UTIL.getConfiguration. Need // to shove this in here first so it gets picked up all over; e.g. by // HTable. connection = HConnectionTestingUtility.getSpiedConnection(UTIL.getConfiguration()); // Fix the location lookup so it 'works' though no network. First // make an 'any location' object. final HRegionLocation anyLocation = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, sn.getHostname(), sn.getPort()); // Return the any location object when locateRegion is called in HTable // constructor and when its called by ServerCallable (it uses getRegionLocation). // The ugly format below comes of 'Important gotcha on spying real objects!' from // http://mockito.googlecode.com/svn/branches/1.6/javadoc/org/mockito/Mockito.html Mockito.doReturn(anyLocation) .when(connection) .locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any()); Mockito.doReturn(anyLocation) .when(connection) .getRegionLocation((byte[]) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean()); // Now shove our HRI implementation into the spied-upon connection. Mockito.doReturn(implementation) .when(connection) .getHRegionConnection(Mockito.anyString(), Mockito.anyInt()); // Now start up the catalogtracker with our doctored Connection. ct = new CatalogTracker(zkw, null, connection, ABORTABLE, 0); ct.start(); // Scan meta for user tables and verify we got back expected answer. NavigableMap<HRegionInfo, Result> hris = MetaReader.getServerUserRegions(ct, sn); assertTrue(hris.size() == 1); assertTrue(hris.firstEntry().getKey().equals(HRegionInfo.FIRST_META_REGIONINFO)); assertTrue(Bytes.equals(rowToVerify, hris.firstEntry().getValue().getRow())); // Finally verify that openscanner was called four times -- three times // with exception and then on 4th attempt we succeed. Mockito.verify(implementation, Mockito.times(4)) .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any()); } finally { if (ct != null) ct.stop(); HConnectionManager.deleteConnection(UTIL.getConfiguration(), true); zkw.close(); } }
/** This tests retaining assignments on a cluster restart */ @Test(timeout = 300000) public void testRetainAssignmentOnRestart() throws Exception { UTIL.startMiniCluster(2); while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) { Threads.sleep(1); } // Turn off balancer UTIL.getMiniHBaseCluster().getMaster().getMasterRpcServices().synchronousBalanceSwitch(false); LOG.info("\n\nCreating tables"); for (byte[] TABLE : TABLES) { UTIL.createTable(TABLE, FAMILY); } for (byte[] TABLE : TABLES) { UTIL.waitTableEnabled(TABLE); } HMaster master = UTIL.getMiniHBaseCluster().getMaster(); UTIL.waitUntilNoRegionsInTransition(120000); // We don't have to use SnapshotOfRegionAssignmentFromMeta. // We use it here because AM used to use it to load all user region placements SnapshotOfRegionAssignmentFromMeta snapshot = new SnapshotOfRegionAssignmentFromMeta(master.getShortCircuitConnection()); snapshot.initialize(); Map<HRegionInfo, ServerName> regionToRegionServerMap = snapshot.getRegionToRegionServerMap(); MiniHBaseCluster cluster = UTIL.getHBaseCluster(); List<JVMClusterUtil.RegionServerThread> threads = cluster.getLiveRegionServerThreads(); assertEquals(2, threads.size()); int[] rsPorts = new int[3]; for (int i = 0; i < 2; i++) { rsPorts[i] = threads.get(i).getRegionServer().getServerName().getPort(); } rsPorts[2] = cluster.getMaster().getServerName().getPort(); for (ServerName serverName : regionToRegionServerMap.values()) { boolean found = false; // Test only, no need to optimize for (int k = 0; k < 3 && !found; k++) { found = serverName.getPort() == rsPorts[k]; } assertTrue(found); } LOG.info("\n\nShutting down HBase cluster"); cluster.shutdown(); cluster.waitUntilShutDown(); LOG.info("\n\nSleeping a bit"); Thread.sleep(2000); LOG.info("\n\nStarting cluster the second time with the same ports"); try { cluster.getConf().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 4); master = cluster.startMaster().getMaster(); for (int i = 0; i < 3; i++) { cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, rsPorts[i]); cluster.startRegionServer(); } } finally { // Reset region server port so as not to conflict with other tests cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, 0); cluster.getConf().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 2); } // Make sure live regionservers are on the same host/port List<ServerName> localServers = master.getServerManager().getOnlineServersList(); assertEquals(4, localServers.size()); for (int i = 0; i < 3; i++) { boolean found = false; for (ServerName serverName : localServers) { if (serverName.getPort() == rsPorts[i]) { found = true; break; } } assertTrue(found); } // Wait till master is initialized and all regions are assigned RegionStates regionStates = master.getAssignmentManager().getRegionStates(); int expectedRegions = regionToRegionServerMap.size() + 1; while (!master.isInitialized() || regionStates.getRegionAssignments().size() != expectedRegions) { Threads.sleep(100); } snapshot = new SnapshotOfRegionAssignmentFromMeta(master.getShortCircuitConnection()); snapshot.initialize(); Map<HRegionInfo, ServerName> newRegionToRegionServerMap = snapshot.getRegionToRegionServerMap(); assertEquals(regionToRegionServerMap.size(), newRegionToRegionServerMap.size()); for (Map.Entry<HRegionInfo, ServerName> entry : newRegionToRegionServerMap.entrySet()) { if (TableName.NAMESPACE_TABLE_NAME.equals(entry.getKey().getTable())) continue; ServerName oldServer = regionToRegionServerMap.get(entry.getKey()); ServerName currentServer = entry.getValue(); assertEquals(oldServer.getHostAndPort(), currentServer.getHostAndPort()); assertNotEquals(oldServer.getStartcode(), currentServer.getStartcode()); } }