/** * Return how many regions will move per table since their primary RS will change * * @param newPlan - new AssignmentPlan * @return how many primaries will move per table */ public Map<TableName, Integer> getRegionsMovement(FavoredNodesPlan newPlan) throws IOException { Map<TableName, Integer> movesPerTable = new HashMap<TableName, Integer>(); SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot(); Map<TableName, List<HRegionInfo>> tableToRegions = snapshot.getTableToRegionMap(); FavoredNodesPlan oldPlan = snapshot.getExistingAssignmentPlan(); Set<TableName> tables = snapshot.getTableSet(); for (TableName table : tables) { int movedPrimaries = 0; if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) { continue; } List<HRegionInfo> regions = tableToRegions.get(table); for (HRegionInfo region : regions) { List<ServerName> oldServers = oldPlan.getFavoredNodes(region); List<ServerName> newServers = newPlan.getFavoredNodes(region); if (oldServers != null && newServers != null) { ServerName oldPrimary = oldServers.get(0); ServerName newPrimary = newServers.get(0); if (oldPrimary.compareTo(newPrimary) != 0) { movedPrimaries++; } } } movesPerTable.put(table, movedPrimaries); } return movesPerTable; }
/** * Verify the region placement is consistent with the assignment plan * * @param isDetailMode * @return reports * @throws IOException */ public List<AssignmentVerificationReport> verifyRegionPlacement(boolean isDetailMode) throws IOException { System.out.println( "Start to verify the region assignment and " + "generate the verification report"); // Get the region assignment snapshot SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot(); // Get all the tables Set<TableName> tables = snapshot.getTableSet(); // Get the region locality map Map<String, Map<String, Float>> regionLocalityMap = null; if (this.enforceLocality == true) { regionLocalityMap = FSUtils.getRegionDegreeLocalityMappingFromFS(conf); } List<AssignmentVerificationReport> reports = new ArrayList<AssignmentVerificationReport>(); // Iterate all the tables to fill up the verification report for (TableName table : tables) { if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) { continue; } AssignmentVerificationReport report = new AssignmentVerificationReport(); report.fillUp(table, snapshot, regionLocalityMap); report.print(isDetailMode); reports.add(report); } return reports; }
public void printLocalityAndDispersionForCurrentPlan( Map<String, Map<String, Float>> regionLocalityMap) throws IOException { SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot(); FavoredNodesPlan assignmentPlan = snapshot.getExistingAssignmentPlan(); Set<TableName> tables = snapshot.getTableSet(); Map<TableName, List<HRegionInfo>> tableToRegionsMap = snapshot.getTableToRegionMap(); for (TableName table : tables) { float[] locality = new float[3]; if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) { continue; } List<HRegionInfo> regions = tableToRegionsMap.get(table); for (HRegionInfo currentRegion : regions) { Map<String, Float> regionLocality = regionLocalityMap.get(currentRegion.getEncodedName()); if (regionLocality == null) { continue; } List<ServerName> servers = assignmentPlan.getFavoredNodes(currentRegion); if (servers != null) { int i = 0; for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) { ServerName server = servers.get(p.ordinal()); Float currentLocality = 0f; if (servers != null) { currentLocality = regionLocality.get(server.getHostname()); if (currentLocality == null) { currentLocality = 0f; } locality[i] += currentLocality; } i++; } } } for (int i = 0; i < locality.length; i++) { String copy = null; if (i == 0) { copy = "primary"; } else if (i == 1) { copy = "secondary"; } else if (i == 2) { copy = "tertiary"; } float avgLocality = 100 * locality[i] / regions.size(); LOG.info( "For Table: " + table + " ; #Total Regions: " + regions.size() + " ; The average locality for " + copy + " is " + avgLocality + " %"); } printDispersionScores(table, snapshot, regions.size(), null, false); } }
public FavoredNodesPlan getNewAssignmentPlan() throws IOException { // Get the current region assignment snapshot by scanning from the META SnapshotOfRegionAssignmentFromMeta assignmentSnapshot = this.getRegionAssignmentSnapshot(); // Get the region locality map Map<String, Map<String, Float>> regionLocalityMap = null; if (this.enforceLocality) { regionLocalityMap = FSUtils.getRegionDegreeLocalityMappingFromFS(conf); } // Initialize the assignment plan FavoredNodesPlan plan = new FavoredNodesPlan(); // Get the table to region mapping Map<TableName, List<HRegionInfo>> tableToRegionMap = assignmentSnapshot.getTableToRegionMap(); LOG.info( "Start to generate the new assignment plan for the " + +tableToRegionMap.keySet().size() + " tables"); for (TableName table : tableToRegionMap.keySet()) { try { if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) { continue; } // TODO: maybe run the placement in parallel for each table genAssignmentPlan( table, assignmentSnapshot, regionLocalityMap, plan, USE_MUNKRES_FOR_PLACING_SECONDARY_AND_TERTIARY); } catch (Exception e) { LOG.error( "Get some exceptions for placing primary region server" + "for table " + table + " because " + e); } } LOG.info( "Finish to generate the new assignment plan for the " + +tableToRegionMap.keySet().size() + " tables"); return plan; }
/** * Compares two plans and check whether the locality dropped or increased (prints the information * as a string) also prints the baseline locality * * @param movesPerTable - how many primary regions will move per table * @param regionLocalityMap - locality map from FS * @param newPlan - new assignment plan * @throws IOException */ public void checkDifferencesWithOldPlan( Map<TableName, Integer> movesPerTable, Map<String, Map<String, Float>> regionLocalityMap, FavoredNodesPlan newPlan) throws IOException { // localities for primary, secondary and tertiary SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot(); FavoredNodesPlan oldPlan = snapshot.getExistingAssignmentPlan(); Set<TableName> tables = snapshot.getTableSet(); Map<TableName, List<HRegionInfo>> tableToRegionsMap = snapshot.getTableToRegionMap(); for (TableName table : tables) { float[] deltaLocality = new float[3]; float[] locality = new float[3]; if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) { continue; } List<HRegionInfo> regions = tableToRegionsMap.get(table); System.out.println("=================================================="); System.out.println("Assignment Plan Projection Report For Table: " + table); System.out.println("\t Total regions: " + regions.size()); System.out.println( "\t" + movesPerTable.get(table) + " primaries will move due to their primary has changed"); for (HRegionInfo currentRegion : regions) { Map<String, Float> regionLocality = regionLocalityMap.get(currentRegion.getEncodedName()); if (regionLocality == null) { continue; } List<ServerName> oldServers = oldPlan.getFavoredNodes(currentRegion); List<ServerName> newServers = newPlan.getFavoredNodes(currentRegion); if (newServers != null && oldServers != null) { int i = 0; for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) { ServerName newServer = newServers.get(p.ordinal()); ServerName oldServer = oldServers.get(p.ordinal()); Float oldLocality = 0f; if (oldServers != null) { oldLocality = regionLocality.get(oldServer.getHostname()); if (oldLocality == null) { oldLocality = 0f; } locality[i] += oldLocality; } Float newLocality = regionLocality.get(newServer.getHostname()); if (newLocality == null) { newLocality = 0f; } deltaLocality[i] += newLocality - oldLocality; i++; } } } DecimalFormat df = new java.text.DecimalFormat("#.##"); for (int i = 0; i < deltaLocality.length; i++) { System.out.print("\t\t Baseline locality for "); if (i == 0) { System.out.print("primary "); } else if (i == 1) { System.out.print("secondary "); } else if (i == 2) { System.out.print("tertiary "); } System.out.println(df.format(100 * locality[i] / regions.size()) + "%"); System.out.print("\t\t Locality will change with the new plan: "); System.out.println(df.format(100 * deltaLocality[i] / regions.size()) + "%"); } System.out.println("\t Baseline dispersion"); printDispersionScores(table, snapshot, regions.size(), null, true); System.out.println("\t Projected dispersion"); printDispersionScores(table, snapshot, regions.size(), newPlan, true); } }
/** * Generate the assignment plan for the existing table * * @param tableName * @param assignmentSnapshot * @param regionLocalityMap * @param plan * @param munkresForSecondaryAndTertiary if set on true the assignment plan for the tertiary and * secondary will be generated with Munkres algorithm, otherwise will be generated using * placeSecondaryAndTertiaryRS * @throws IOException */ private void genAssignmentPlan( TableName tableName, SnapshotOfRegionAssignmentFromMeta assignmentSnapshot, Map<String, Map<String, Float>> regionLocalityMap, FavoredNodesPlan plan, boolean munkresForSecondaryAndTertiary) throws IOException { // Get the all the regions for the current table List<HRegionInfo> regions = assignmentSnapshot.getTableToRegionMap().get(tableName); int numRegions = regions.size(); // Get the current assignment map Map<HRegionInfo, ServerName> currentAssignmentMap = assignmentSnapshot.getRegionToRegionServerMap(); // Get the all the region servers List<ServerName> servers = new ArrayList<ServerName>(); try (Admin admin = this.connection.getAdmin()) { servers.addAll(admin.getClusterStatus().getServers()); } LOG.info( "Start to generate assignment plan for " + numRegions + " regions from table " + tableName + " with " + servers.size() + " region servers"); int slotsPerServer = (int) Math.ceil((float) numRegions / servers.size()); int regionSlots = slotsPerServer * servers.size(); // Compute the primary, secondary and tertiary costs for each region/server // pair. These costs are based only on node locality and rack locality, and // will be modified later. float[][] primaryCost = new float[numRegions][regionSlots]; float[][] secondaryCost = new float[numRegions][regionSlots]; float[][] tertiaryCost = new float[numRegions][regionSlots]; if (this.enforceLocality && regionLocalityMap != null) { // Transform the locality mapping into a 2D array, assuming that any // unspecified locality value is 0. float[][] localityPerServer = new float[numRegions][regionSlots]; for (int i = 0; i < numRegions; i++) { Map<String, Float> serverLocalityMap = regionLocalityMap.get(regions.get(i).getEncodedName()); if (serverLocalityMap == null) { continue; } for (int j = 0; j < servers.size(); j++) { String serverName = servers.get(j).getHostname(); if (serverName == null) { continue; } Float locality = serverLocalityMap.get(serverName); if (locality == null) { continue; } for (int k = 0; k < slotsPerServer; k++) { // If we can't find the locality of a region to a server, which occurs // because locality is only reported for servers which have some // blocks of a region local, then the locality for that pair is 0. localityPerServer[i][j * slotsPerServer + k] = locality.floatValue(); } } } // Compute the total rack locality for each region in each rack. The total // rack locality is the sum of the localities of a region on all servers in // a rack. Map<String, Map<HRegionInfo, Float>> rackRegionLocality = new HashMap<String, Map<HRegionInfo, Float>>(); for (int i = 0; i < numRegions; i++) { HRegionInfo region = regions.get(i); for (int j = 0; j < regionSlots; j += slotsPerServer) { String rack = rackManager.getRack(servers.get(j / slotsPerServer)); Map<HRegionInfo, Float> rackLocality = rackRegionLocality.get(rack); if (rackLocality == null) { rackLocality = new HashMap<HRegionInfo, Float>(); rackRegionLocality.put(rack, rackLocality); } Float localityObj = rackLocality.get(region); float locality = localityObj == null ? 0 : localityObj.floatValue(); locality += localityPerServer[i][j]; rackLocality.put(region, locality); } } for (int i = 0; i < numRegions; i++) { for (int j = 0; j < regionSlots; j++) { String rack = rackManager.getRack(servers.get(j / slotsPerServer)); Float totalRackLocalityObj = rackRegionLocality.get(rack).get(regions.get(i)); float totalRackLocality = totalRackLocalityObj == null ? 0 : totalRackLocalityObj.floatValue(); // Primary cost aims to favor servers with high node locality and low // rack locality, so that secondaries and tertiaries can be chosen for // nodes with high rack locality. This might give primaries with // slightly less locality at first compared to a cost which only // considers the node locality, but should be better in the long run. primaryCost[i][j] = 1 - (2 * localityPerServer[i][j] - totalRackLocality); // Secondary cost aims to favor servers with high node locality and high // rack locality since the tertiary will be chosen from the same rack as // the secondary. This could be negative, but that is okay. secondaryCost[i][j] = 2 - (localityPerServer[i][j] + totalRackLocality); // Tertiary cost is only concerned with the node locality. It will later // be restricted to only hosts on the same rack as the secondary. tertiaryCost[i][j] = 1 - localityPerServer[i][j]; } } } if (this.enforceMinAssignmentMove && currentAssignmentMap != null) { // We want to minimize the number of regions which move as the result of a // new assignment. Therefore, slightly penalize any placement which is for // a host that is not currently serving the region. for (int i = 0; i < numRegions; i++) { for (int j = 0; j < servers.size(); j++) { ServerName currentAddress = currentAssignmentMap.get(regions.get(i)); if (currentAddress != null && !currentAddress.equals(servers.get(j))) { for (int k = 0; k < slotsPerServer; k++) { primaryCost[i][j * slotsPerServer + k] += NOT_CURRENT_HOST_PENALTY; } } } } } // Artificially increase cost of last slot of each server to evenly // distribute the slop, otherwise there will be a few servers with too few // regions and many servers with the max number of regions. for (int i = 0; i < numRegions; i++) { for (int j = 0; j < regionSlots; j += slotsPerServer) { primaryCost[i][j] += LAST_SLOT_COST_PENALTY; secondaryCost[i][j] += LAST_SLOT_COST_PENALTY; tertiaryCost[i][j] += LAST_SLOT_COST_PENALTY; } } RandomizedMatrix randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots); primaryCost = randomizedMatrix.transform(primaryCost); int[] primaryAssignment = new MunkresAssignment(primaryCost).solve(); primaryAssignment = randomizedMatrix.invertIndices(primaryAssignment); // Modify the secondary and tertiary costs for each region/server pair to // prevent a region from being assigned to the same rack for both primary // and either one of secondary or tertiary. for (int i = 0; i < numRegions; i++) { int slot = primaryAssignment[i]; String rack = rackManager.getRack(servers.get(slot / slotsPerServer)); for (int k = 0; k < servers.size(); k++) { if (!rackManager.getRack(servers.get(k)).equals(rack)) { continue; } if (k == slot / slotsPerServer) { // Same node, do not place secondary or tertiary here ever. for (int m = 0; m < slotsPerServer; m++) { secondaryCost[i][k * slotsPerServer + m] = MAX_COST; tertiaryCost[i][k * slotsPerServer + m] = MAX_COST; } } else { // Same rack, do not place secondary or tertiary here if possible. for (int m = 0; m < slotsPerServer; m++) { secondaryCost[i][k * slotsPerServer + m] = AVOID_COST; tertiaryCost[i][k * slotsPerServer + m] = AVOID_COST; } } } } if (munkresForSecondaryAndTertiary) { randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots); secondaryCost = randomizedMatrix.transform(secondaryCost); int[] secondaryAssignment = new MunkresAssignment(secondaryCost).solve(); secondaryAssignment = randomizedMatrix.invertIndices(secondaryAssignment); // Modify the tertiary costs for each region/server pair to ensure that a // region is assigned to a tertiary server on the same rack as its secondary // server, but not the same server in that rack. for (int i = 0; i < numRegions; i++) { int slot = secondaryAssignment[i]; String rack = rackManager.getRack(servers.get(slot / slotsPerServer)); for (int k = 0; k < servers.size(); k++) { if (k == slot / slotsPerServer) { // Same node, do not place tertiary here ever. for (int m = 0; m < slotsPerServer; m++) { tertiaryCost[i][k * slotsPerServer + m] = MAX_COST; } } else { if (rackManager.getRack(servers.get(k)).equals(rack)) { continue; } // Different rack, do not place tertiary here if possible. for (int m = 0; m < slotsPerServer; m++) { tertiaryCost[i][k * slotsPerServer + m] = AVOID_COST; } } } } randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots); tertiaryCost = randomizedMatrix.transform(tertiaryCost); int[] tertiaryAssignment = new MunkresAssignment(tertiaryCost).solve(); tertiaryAssignment = randomizedMatrix.invertIndices(tertiaryAssignment); for (int i = 0; i < numRegions; i++) { List<ServerName> favoredServers = new ArrayList<ServerName>(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM); ServerName s = servers.get(primaryAssignment[i] / slotsPerServer); favoredServers.add( ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE)); s = servers.get(secondaryAssignment[i] / slotsPerServer); favoredServers.add( ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE)); s = servers.get(tertiaryAssignment[i] / slotsPerServer); favoredServers.add( ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE)); // Update the assignment plan plan.updateAssignmentPlan(regions.get(i), favoredServers); } LOG.info( "Generated the assignment plan for " + numRegions + " regions from table " + tableName + " with " + servers.size() + " region servers"); LOG.info("Assignment plan for secondary and tertiary generated " + "using MunkresAssignment"); } else { Map<HRegionInfo, ServerName> primaryRSMap = new HashMap<HRegionInfo, ServerName>(); for (int i = 0; i < numRegions; i++) { primaryRSMap.put(regions.get(i), servers.get(primaryAssignment[i] / slotsPerServer)); } FavoredNodeAssignmentHelper favoredNodeHelper = new FavoredNodeAssignmentHelper(servers, conf); favoredNodeHelper.initialize(); Map<HRegionInfo, ServerName[]> secondaryAndTertiaryMap = favoredNodeHelper.placeSecondaryAndTertiaryWithRestrictions(primaryRSMap); for (int i = 0; i < numRegions; i++) { List<ServerName> favoredServers = new ArrayList<ServerName>(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM); HRegionInfo currentRegion = regions.get(i); ServerName s = primaryRSMap.get(currentRegion); favoredServers.add( ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE)); ServerName[] secondaryAndTertiary = secondaryAndTertiaryMap.get(currentRegion); s = secondaryAndTertiary[0]; favoredServers.add( ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE)); s = secondaryAndTertiary[1]; favoredServers.add( ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE)); // Update the assignment plan plan.updateAssignmentPlan(regions.get(i), favoredServers); } LOG.info( "Generated the assignment plan for " + numRegions + " regions from table " + tableName + " with " + servers.size() + " region servers"); LOG.info( "Assignment plan for secondary and tertiary generated " + "using placeSecondaryAndTertiaryWithRestrictions method"); } }
/** * @return the new RegionAssignmentSnapshot * @throws IOException */ public SnapshotOfRegionAssignmentFromMeta getRegionAssignmentSnapshot() throws IOException { SnapshotOfRegionAssignmentFromMeta currentAssignmentShapshot = new SnapshotOfRegionAssignmentFromMeta(ConnectionFactory.createConnection(conf)); currentAssignmentShapshot.initialize(); return currentAssignmentShapshot; }
public void fillUp( TableName tableName, SnapshotOfRegionAssignmentFromMeta snapshot, Map<String, Map<String, Float>> regionLocalityMap) { // Set the table name this.tableName = tableName; // Get all the regions for this table List<HRegionInfo> regionInfoList = snapshot.getTableToRegionMap().get(tableName); // Get the total region num for the current table this.totalRegions = regionInfoList.size(); // Get the existing assignment plan FavoredNodesPlan favoredNodesAssignment = snapshot.getExistingAssignmentPlan(); // Get the region to region server mapping Map<HRegionInfo, ServerName> currentAssignment = snapshot.getRegionToRegionServerMap(); // Initialize the server to its hosing region counter map Map<ServerName, Integer> serverToHostingRegionCounterMap = new HashMap<ServerName, Integer>(); Map<ServerName, Integer> primaryRSToRegionCounterMap = new HashMap<ServerName, Integer>(); Map<ServerName, Set<ServerName>> primaryToSecTerRSMap = new HashMap<ServerName, Set<ServerName>>(); // Check the favored nodes and its locality information // Also keep tracker of the most loaded and least loaded region servers for (HRegionInfo region : regionInfoList) { try { ServerName currentRS = currentAssignment.get(region); // Handle unassigned regions if (currentRS == null) { unAssignedRegionsList.add(region); continue; } // Keep updating the server to is hosting region counter map Integer hostRegionCounter = serverToHostingRegionCounterMap.get(currentRS); if (hostRegionCounter == null) { hostRegionCounter = Integer.valueOf(0); } hostRegionCounter = hostRegionCounter.intValue() + 1; serverToHostingRegionCounterMap.put(currentRS, hostRegionCounter); // Get the favored nodes from the assignment plan and verify it. List<ServerName> favoredNodes = favoredNodesAssignment.getFavoredNodes(region); if (favoredNodes == null || favoredNodes.size() != FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) { regionsWithoutValidFavoredNodes.add(region); continue; } // Get the primary, secondary and tertiary region server ServerName primaryRS = favoredNodes.get(FavoredNodesPlan.Position.PRIMARY.ordinal()); ServerName secondaryRS = favoredNodes.get(FavoredNodesPlan.Position.SECONDARY.ordinal()); ServerName tertiaryRS = favoredNodes.get(FavoredNodesPlan.Position.TERTIARY.ordinal()); // Update the primary rs to its region set map Integer regionCounter = primaryRSToRegionCounterMap.get(primaryRS); if (regionCounter == null) { regionCounter = Integer.valueOf(0); } regionCounter = regionCounter.intValue() + 1; primaryRSToRegionCounterMap.put(primaryRS, regionCounter); // Update the primary rs to secondary and tertiary rs map Set<ServerName> secAndTerSet = primaryToSecTerRSMap.get(primaryRS); if (secAndTerSet == null) { secAndTerSet = new HashSet<ServerName>(); } secAndTerSet.add(secondaryRS); secAndTerSet.add(tertiaryRS); primaryToSecTerRSMap.put(primaryRS, secAndTerSet); // Get the position of the current region server in the favored nodes list FavoredNodesPlan.Position favoredNodePosition = FavoredNodesPlan.getFavoredServerPosition(favoredNodes, currentRS); // Handle the non favored assignment. if (favoredNodePosition == null) { nonFavoredAssignedRegionList.add(region); continue; } // Increase the favored nodes assignment. this.favoredNodes[favoredNodePosition.ordinal()]++; totalFavoredAssignments++; // Summary the locality information for each favored nodes if (regionLocalityMap != null) { // Set the enforce locality as true; this.enforceLocality = true; // Get the region degree locality map Map<String, Float> regionDegreeLocalityMap = regionLocalityMap.get(region.getEncodedName()); if (regionDegreeLocalityMap == null) { continue; // ignore the region which doesn't have any store files. } // Get the locality summary for each favored nodes for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) { ServerName favoredNode = favoredNodes.get(p.ordinal()); // Get the locality for the current favored nodes Float locality = regionDegreeLocalityMap.get(favoredNode.getHostname()); if (locality != null) { this.favoredNodesLocalitySummary[p.ordinal()] += locality; } } // Get the locality summary for the current region server Float actualLocality = regionDegreeLocalityMap.get(currentRS.getHostname()); if (actualLocality != null) { this.actualLocalitySummary += actualLocality; } } } catch (Exception e) { LOG.error( "Cannot verify the region assignment for region " + ((region == null) ? " null " : region.getRegionNameAsString()) + "because of " + e); } } float dispersionScoreSummary = 0; float dispersionNumSummary = 0; // Calculate the secondary score for each primary region server for (Map.Entry<ServerName, Integer> entry : primaryRSToRegionCounterMap.entrySet()) { ServerName primaryRS = entry.getKey(); Integer regionsOnPrimary = entry.getValue(); // Process the dispersion number and score float dispersionScore = 0; int dispersionNum = 0; if (primaryToSecTerRSMap.get(primaryRS) != null && regionsOnPrimary.intValue() != 0) { dispersionNum = primaryToSecTerRSMap.get(primaryRS).size(); dispersionScore = dispersionNum / ((float) regionsOnPrimary.intValue() * 2); } // Update the max dispersion score if (dispersionScore > this.maxDispersionScore) { this.maxDispersionScoreServerSet.clear(); this.maxDispersionScoreServerSet.add(primaryRS); this.maxDispersionScore = dispersionScore; } else if (dispersionScore == this.maxDispersionScore) { this.maxDispersionScoreServerSet.add(primaryRS); } // Update the max dispersion num if (dispersionNum > this.maxDispersionNum) { this.maxDispersionNumServerSet.clear(); this.maxDispersionNumServerSet.add(primaryRS); this.maxDispersionNum = dispersionNum; } else if (dispersionNum == this.maxDispersionNum) { this.maxDispersionNumServerSet.add(primaryRS); } // Update the min dispersion score if (dispersionScore < this.minDispersionScore) { this.minDispersionScoreServerSet.clear(); this.minDispersionScoreServerSet.add(primaryRS); this.minDispersionScore = dispersionScore; } else if (dispersionScore == this.minDispersionScore) { this.minDispersionScoreServerSet.add(primaryRS); } // Update the min dispersion num if (dispersionNum < this.minDispersionNum) { this.minDispersionNumServerSet.clear(); this.minDispersionNumServerSet.add(primaryRS); this.minDispersionNum = dispersionNum; } else if (dispersionNum == this.minDispersionNum) { this.minDispersionNumServerSet.add(primaryRS); } dispersionScoreSummary += dispersionScore; dispersionNumSummary += dispersionNum; } // Update the avg dispersion score if (primaryRSToRegionCounterMap.keySet().size() != 0) { this.avgDispersionScore = dispersionScoreSummary / (float) primaryRSToRegionCounterMap.keySet().size(); this.avgDispersionNum = dispersionNumSummary / (float) primaryRSToRegionCounterMap.keySet().size(); } // Fill up the most loaded and least loaded region server information for (Map.Entry<ServerName, Integer> entry : serverToHostingRegionCounterMap.entrySet()) { ServerName currentRS = entry.getKey(); int hostRegionCounter = entry.getValue().intValue(); // Update the most loaded region server list and maxRegionsOnRS if (hostRegionCounter > this.maxRegionsOnRS) { maxRegionsOnRS = hostRegionCounter; this.mostLoadedRSSet.clear(); this.mostLoadedRSSet.add(currentRS); } else if (hostRegionCounter == this.maxRegionsOnRS) { this.mostLoadedRSSet.add(currentRS); } // Update the least loaded region server list and minRegionsOnRS if (hostRegionCounter < this.minRegionsOnRS) { this.minRegionsOnRS = hostRegionCounter; this.leastLoadedRSSet.clear(); this.leastLoadedRSSet.add(currentRS); } else if (hostRegionCounter == this.minRegionsOnRS) { this.leastLoadedRSSet.add(currentRS); } } // and total region servers this.totalRegionServers = serverToHostingRegionCounterMap.keySet().size(); this.avgRegionsOnRS = (totalRegionServers == 0) ? 0 : (totalRegions / (float) totalRegionServers); // Set the isFilledUp as true isFilledUp = true; }
/** * Use this to project the dispersion scores * * @param tableName * @param snapshot * @param newPlan */ public void fillUpDispersion( TableName tableName, SnapshotOfRegionAssignmentFromMeta snapshot, FavoredNodesPlan newPlan) { // Set the table name this.tableName = tableName; // Get all the regions for this table List<HRegionInfo> regionInfoList = snapshot.getTableToRegionMap().get(tableName); // Get the total region num for the current table this.totalRegions = regionInfoList.size(); FavoredNodesPlan plan = null; if (newPlan == null) { plan = snapshot.getExistingAssignmentPlan(); } else { plan = newPlan; } // Get the region to region server mapping Map<ServerName, Integer> primaryRSToRegionCounterMap = new HashMap<ServerName, Integer>(); Map<ServerName, Set<ServerName>> primaryToSecTerRSMap = new HashMap<ServerName, Set<ServerName>>(); // Check the favored nodes and its locality information // Also keep tracker of the most loaded and least loaded region servers for (HRegionInfo region : regionInfoList) { try { // Get the favored nodes from the assignment plan and verify it. List<ServerName> favoredNodes = plan.getFavoredNodes(region); if (favoredNodes == null || favoredNodes.size() != FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) { regionsWithoutValidFavoredNodes.add(region); continue; } // Get the primary, secondary and tertiary region server ServerName primaryRS = favoredNodes.get(FavoredNodesPlan.Position.PRIMARY.ordinal()); ServerName secondaryRS = favoredNodes.get(FavoredNodesPlan.Position.SECONDARY.ordinal()); ServerName tertiaryRS = favoredNodes.get(FavoredNodesPlan.Position.TERTIARY.ordinal()); // Update the primary rs to its region set map Integer regionCounter = primaryRSToRegionCounterMap.get(primaryRS); if (regionCounter == null) { regionCounter = Integer.valueOf(0); } regionCounter = regionCounter.intValue() + 1; primaryRSToRegionCounterMap.put(primaryRS, regionCounter); // Update the primary rs to secondary and tertiary rs map Set<ServerName> secAndTerSet = primaryToSecTerRSMap.get(primaryRS); if (secAndTerSet == null) { secAndTerSet = new HashSet<ServerName>(); } secAndTerSet.add(secondaryRS); secAndTerSet.add(tertiaryRS); primaryToSecTerRSMap.put(primaryRS, secAndTerSet); } catch (Exception e) { LOG.error( "Cannot verify the region assignment for region " + ((region == null) ? " null " : region.getRegionNameAsString()) + "because of " + e); } } float dispersionScoreSummary = 0; float dispersionNumSummary = 0; // Calculate the secondary score for each primary region server for (Map.Entry<ServerName, Integer> entry : primaryRSToRegionCounterMap.entrySet()) { ServerName primaryRS = entry.getKey(); Integer regionsOnPrimary = entry.getValue(); // Process the dispersion number and score float dispersionScore = 0; int dispersionNum = 0; if (primaryToSecTerRSMap.get(primaryRS) != null && regionsOnPrimary.intValue() != 0) { dispersionNum = primaryToSecTerRSMap.get(primaryRS).size(); dispersionScore = dispersionNum / ((float) regionsOnPrimary.intValue() * 2); } // Update the max dispersion num if (dispersionNum > this.maxDispersionNum) { this.maxDispersionNumServerSet.clear(); this.maxDispersionNumServerSet.add(primaryRS); this.maxDispersionNum = dispersionNum; } else if (dispersionNum == this.maxDispersionNum) { this.maxDispersionNumServerSet.add(primaryRS); } // Update the min dispersion score if (dispersionScore < this.minDispersionScore) { this.minDispersionScoreServerSet.clear(); this.minDispersionScoreServerSet.add(primaryRS); this.minDispersionScore = dispersionScore; } else if (dispersionScore == this.minDispersionScore) { this.minDispersionScoreServerSet.add(primaryRS); } // Update the min dispersion num if (dispersionNum < this.minDispersionNum) { this.minDispersionNumServerSet.clear(); this.minDispersionNumServerSet.add(primaryRS); this.minDispersionNum = dispersionNum; } else if (dispersionNum == this.minDispersionNum) { this.minDispersionNumServerSet.add(primaryRS); } dispersionScoreSummary += dispersionScore; dispersionNumSummary += dispersionNum; } // Update the avg dispersion score if (primaryRSToRegionCounterMap.keySet().size() != 0) { this.avgDispersionScore = dispersionScoreSummary / (float) primaryRSToRegionCounterMap.keySet().size(); this.avgDispersionNum = dispersionNumSummary / (float) primaryRSToRegionCounterMap.keySet().size(); } }
/** This tests retaining assignments on a cluster restart */ @Test(timeout = 300000) public void testRetainAssignmentOnRestart() throws Exception { UTIL.startMiniCluster(2); while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) { Threads.sleep(1); } // Turn off balancer UTIL.getMiniHBaseCluster().getMaster().getMasterRpcServices().synchronousBalanceSwitch(false); LOG.info("\n\nCreating tables"); for (byte[] TABLE : TABLES) { UTIL.createTable(TABLE, FAMILY); } for (byte[] TABLE : TABLES) { UTIL.waitTableEnabled(TABLE); } HMaster master = UTIL.getMiniHBaseCluster().getMaster(); UTIL.waitUntilNoRegionsInTransition(120000); // We don't have to use SnapshotOfRegionAssignmentFromMeta. // We use it here because AM used to use it to load all user region placements SnapshotOfRegionAssignmentFromMeta snapshot = new SnapshotOfRegionAssignmentFromMeta(master.getShortCircuitConnection()); snapshot.initialize(); Map<HRegionInfo, ServerName> regionToRegionServerMap = snapshot.getRegionToRegionServerMap(); MiniHBaseCluster cluster = UTIL.getHBaseCluster(); List<JVMClusterUtil.RegionServerThread> threads = cluster.getLiveRegionServerThreads(); assertEquals(2, threads.size()); int[] rsPorts = new int[3]; for (int i = 0; i < 2; i++) { rsPorts[i] = threads.get(i).getRegionServer().getServerName().getPort(); } rsPorts[2] = cluster.getMaster().getServerName().getPort(); for (ServerName serverName : regionToRegionServerMap.values()) { boolean found = false; // Test only, no need to optimize for (int k = 0; k < 3 && !found; k++) { found = serverName.getPort() == rsPorts[k]; } assertTrue(found); } LOG.info("\n\nShutting down HBase cluster"); cluster.shutdown(); cluster.waitUntilShutDown(); LOG.info("\n\nSleeping a bit"); Thread.sleep(2000); LOG.info("\n\nStarting cluster the second time with the same ports"); try { cluster.getConf().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 4); master = cluster.startMaster().getMaster(); for (int i = 0; i < 3; i++) { cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, rsPorts[i]); cluster.startRegionServer(); } } finally { // Reset region server port so as not to conflict with other tests cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, 0); cluster.getConf().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 2); } // Make sure live regionservers are on the same host/port List<ServerName> localServers = master.getServerManager().getOnlineServersList(); assertEquals(4, localServers.size()); for (int i = 0; i < 3; i++) { boolean found = false; for (ServerName serverName : localServers) { if (serverName.getPort() == rsPorts[i]) { found = true; break; } } assertTrue(found); } // Wait till master is initialized and all regions are assigned RegionStates regionStates = master.getAssignmentManager().getRegionStates(); int expectedRegions = regionToRegionServerMap.size() + 1; while (!master.isInitialized() || regionStates.getRegionAssignments().size() != expectedRegions) { Threads.sleep(100); } snapshot = new SnapshotOfRegionAssignmentFromMeta(master.getShortCircuitConnection()); snapshot.initialize(); Map<HRegionInfo, ServerName> newRegionToRegionServerMap = snapshot.getRegionToRegionServerMap(); assertEquals(regionToRegionServerMap.size(), newRegionToRegionServerMap.size()); for (Map.Entry<HRegionInfo, ServerName> entry : newRegionToRegionServerMap.entrySet()) { if (TableName.NAMESPACE_TABLE_NAME.equals(entry.getKey().getTable())) continue; ServerName oldServer = regionToRegionServerMap.get(entry.getKey()); ServerName currentServer = entry.getValue(); assertEquals(oldServer.getHostAndPort(), currentServer.getHostAndPort()); assertNotEquals(oldServer.getStartcode(), currentServer.getStartcode()); } }