/**
* Return how many regions will move per table since their primary RS will change
*
* @param newPlan - new AssignmentPlan
* @return how many primaries will move per table
*/
public Map<TableName, Integer> getRegionsMovement(FavoredNodesPlan newPlan) throws IOException {
Map<TableName, Integer> movesPerTable = new HashMap<TableName, Integer>();
SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot();
Map<TableName, List<HRegionInfo>> tableToRegions = snapshot.getTableToRegionMap();
FavoredNodesPlan oldPlan = snapshot.getExistingAssignmentPlan();
Set<TableName> tables = snapshot.getTableSet();
for (TableName table : tables) {
int movedPrimaries = 0;
if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) {
continue;
}
List<HRegionInfo> regions = tableToRegions.get(table);
for (HRegionInfo region : regions) {
List<ServerName> oldServers = oldPlan.getFavoredNodes(region);
List<ServerName> newServers = newPlan.getFavoredNodes(region);
if (oldServers != null && newServers != null) {
ServerName oldPrimary = oldServers.get(0);
ServerName newPrimary = newServers.get(0);
if (oldPrimary.compareTo(newPrimary) != 0) {
movedPrimaries++;
}
}
}
movesPerTable.put(table, movedPrimaries);
}
return movesPerTable;
}
/**
* Verify the region placement is consistent with the assignment plan
*
* @param isDetailMode
* @return reports
* @throws IOException
*/
public List<AssignmentVerificationReport> verifyRegionPlacement(boolean isDetailMode)
throws IOException {
System.out.println(
"Start to verify the region assignment and " + "generate the verification report");
// Get the region assignment snapshot
SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot();
// Get all the tables
Set<TableName> tables = snapshot.getTableSet();
// Get the region locality map
Map<String, Map<String, Float>> regionLocalityMap = null;
if (this.enforceLocality == true) {
regionLocalityMap = FSUtils.getRegionDegreeLocalityMappingFromFS(conf);
}
List<AssignmentVerificationReport> reports = new ArrayList<AssignmentVerificationReport>();
// Iterate all the tables to fill up the verification report
for (TableName table : tables) {
if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) {
continue;
}
AssignmentVerificationReport report = new AssignmentVerificationReport();
report.fillUp(table, snapshot, regionLocalityMap);
report.print(isDetailMode);
reports.add(report);
}
return reports;
}
public void printLocalityAndDispersionForCurrentPlan(
Map<String, Map<String, Float>> regionLocalityMap) throws IOException {
SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot();
FavoredNodesPlan assignmentPlan = snapshot.getExistingAssignmentPlan();
Set<TableName> tables = snapshot.getTableSet();
Map<TableName, List<HRegionInfo>> tableToRegionsMap = snapshot.getTableToRegionMap();
for (TableName table : tables) {
float[] locality = new float[3];
if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) {
continue;
}
List<HRegionInfo> regions = tableToRegionsMap.get(table);
for (HRegionInfo currentRegion : regions) {
Map<String, Float> regionLocality = regionLocalityMap.get(currentRegion.getEncodedName());
if (regionLocality == null) {
continue;
}
List<ServerName> servers = assignmentPlan.getFavoredNodes(currentRegion);
if (servers != null) {
int i = 0;
for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) {
ServerName server = servers.get(p.ordinal());
Float currentLocality = 0f;
if (servers != null) {
currentLocality = regionLocality.get(server.getHostname());
if (currentLocality == null) {
currentLocality = 0f;
}
locality[i] += currentLocality;
}
i++;
}
}
}
for (int i = 0; i < locality.length; i++) {
String copy = null;
if (i == 0) {
copy = "primary";
} else if (i == 1) {
copy = "secondary";
} else if (i == 2) {
copy = "tertiary";
}
float avgLocality = 100 * locality[i] / regions.size();
LOG.info(
"For Table: "
+ table
+ " ; #Total Regions: "
+ regions.size()
+ " ; The average locality for "
+ copy
+ " is "
+ avgLocality
+ " %");
}
printDispersionScores(table, snapshot, regions.size(), null, false);
}
}
public FavoredNodesPlan getNewAssignmentPlan() throws IOException {
// Get the current region assignment snapshot by scanning from the META
SnapshotOfRegionAssignmentFromMeta assignmentSnapshot = this.getRegionAssignmentSnapshot();
// Get the region locality map
Map<String, Map<String, Float>> regionLocalityMap = null;
if (this.enforceLocality) {
regionLocalityMap = FSUtils.getRegionDegreeLocalityMappingFromFS(conf);
}
// Initialize the assignment plan
FavoredNodesPlan plan = new FavoredNodesPlan();
// Get the table to region mapping
Map<TableName, List<HRegionInfo>> tableToRegionMap = assignmentSnapshot.getTableToRegionMap();
LOG.info(
"Start to generate the new assignment plan for the "
+ +tableToRegionMap.keySet().size()
+ " tables");
for (TableName table : tableToRegionMap.keySet()) {
try {
if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) {
continue;
}
// TODO: maybe run the placement in parallel for each table
genAssignmentPlan(
table,
assignmentSnapshot,
regionLocalityMap,
plan,
USE_MUNKRES_FOR_PLACING_SECONDARY_AND_TERTIARY);
} catch (Exception e) {
LOG.error(
"Get some exceptions for placing primary region server"
+ "for table "
+ table
+ " because "
+ e);
}
}
LOG.info(
"Finish to generate the new assignment plan for the "
+ +tableToRegionMap.keySet().size()
+ " tables");
return plan;
}
/**
* Compares two plans and check whether the locality dropped or increased (prints the information
* as a string) also prints the baseline locality
*
* @param movesPerTable - how many primary regions will move per table
* @param regionLocalityMap - locality map from FS
* @param newPlan - new assignment plan
* @throws IOException
*/
public void checkDifferencesWithOldPlan(
Map<TableName, Integer> movesPerTable,
Map<String, Map<String, Float>> regionLocalityMap,
FavoredNodesPlan newPlan)
throws IOException {
// localities for primary, secondary and tertiary
SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot();
FavoredNodesPlan oldPlan = snapshot.getExistingAssignmentPlan();
Set<TableName> tables = snapshot.getTableSet();
Map<TableName, List<HRegionInfo>> tableToRegionsMap = snapshot.getTableToRegionMap();
for (TableName table : tables) {
float[] deltaLocality = new float[3];
float[] locality = new float[3];
if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) {
continue;
}
List<HRegionInfo> regions = tableToRegionsMap.get(table);
System.out.println("==================================================");
System.out.println("Assignment Plan Projection Report For Table: " + table);
System.out.println("\t Total regions: " + regions.size());
System.out.println(
"\t"
+ movesPerTable.get(table)
+ " primaries will move due to their primary has changed");
for (HRegionInfo currentRegion : regions) {
Map<String, Float> regionLocality = regionLocalityMap.get(currentRegion.getEncodedName());
if (regionLocality == null) {
continue;
}
List<ServerName> oldServers = oldPlan.getFavoredNodes(currentRegion);
List<ServerName> newServers = newPlan.getFavoredNodes(currentRegion);
if (newServers != null && oldServers != null) {
int i = 0;
for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) {
ServerName newServer = newServers.get(p.ordinal());
ServerName oldServer = oldServers.get(p.ordinal());
Float oldLocality = 0f;
if (oldServers != null) {
oldLocality = regionLocality.get(oldServer.getHostname());
if (oldLocality == null) {
oldLocality = 0f;
}
locality[i] += oldLocality;
}
Float newLocality = regionLocality.get(newServer.getHostname());
if (newLocality == null) {
newLocality = 0f;
}
deltaLocality[i] += newLocality - oldLocality;
i++;
}
}
}
DecimalFormat df = new java.text.DecimalFormat("#.##");
for (int i = 0; i < deltaLocality.length; i++) {
System.out.print("\t\t Baseline locality for ");
if (i == 0) {
System.out.print("primary ");
} else if (i == 1) {
System.out.print("secondary ");
} else if (i == 2) {
System.out.print("tertiary ");
}
System.out.println(df.format(100 * locality[i] / regions.size()) + "%");
System.out.print("\t\t Locality will change with the new plan: ");
System.out.println(df.format(100 * deltaLocality[i] / regions.size()) + "%");
}
System.out.println("\t Baseline dispersion");
printDispersionScores(table, snapshot, regions.size(), null, true);
System.out.println("\t Projected dispersion");
printDispersionScores(table, snapshot, regions.size(), newPlan, true);
}
}
/**
* Generate the assignment plan for the existing table
*
* @param tableName
* @param assignmentSnapshot
* @param regionLocalityMap
* @param plan
* @param munkresForSecondaryAndTertiary if set on true the assignment plan for the tertiary and
* secondary will be generated with Munkres algorithm, otherwise will be generated using
* placeSecondaryAndTertiaryRS
* @throws IOException
*/
private void genAssignmentPlan(
TableName tableName,
SnapshotOfRegionAssignmentFromMeta assignmentSnapshot,
Map<String, Map<String, Float>> regionLocalityMap,
FavoredNodesPlan plan,
boolean munkresForSecondaryAndTertiary)
throws IOException {
// Get the all the regions for the current table
List<HRegionInfo> regions = assignmentSnapshot.getTableToRegionMap().get(tableName);
int numRegions = regions.size();
// Get the current assignment map
Map<HRegionInfo, ServerName> currentAssignmentMap =
assignmentSnapshot.getRegionToRegionServerMap();
// Get the all the region servers
List<ServerName> servers = new ArrayList<ServerName>();
try (Admin admin = this.connection.getAdmin()) {
servers.addAll(admin.getClusterStatus().getServers());
}
LOG.info(
"Start to generate assignment plan for "
+ numRegions
+ " regions from table "
+ tableName
+ " with "
+ servers.size()
+ " region servers");
int slotsPerServer = (int) Math.ceil((float) numRegions / servers.size());
int regionSlots = slotsPerServer * servers.size();
// Compute the primary, secondary and tertiary costs for each region/server
// pair. These costs are based only on node locality and rack locality, and
// will be modified later.
float[][] primaryCost = new float[numRegions][regionSlots];
float[][] secondaryCost = new float[numRegions][regionSlots];
float[][] tertiaryCost = new float[numRegions][regionSlots];
if (this.enforceLocality && regionLocalityMap != null) {
// Transform the locality mapping into a 2D array, assuming that any
// unspecified locality value is 0.
float[][] localityPerServer = new float[numRegions][regionSlots];
for (int i = 0; i < numRegions; i++) {
Map<String, Float> serverLocalityMap =
regionLocalityMap.get(regions.get(i).getEncodedName());
if (serverLocalityMap == null) {
continue;
}
for (int j = 0; j < servers.size(); j++) {
String serverName = servers.get(j).getHostname();
if (serverName == null) {
continue;
}
Float locality = serverLocalityMap.get(serverName);
if (locality == null) {
continue;
}
for (int k = 0; k < slotsPerServer; k++) {
// If we can't find the locality of a region to a server, which occurs
// because locality is only reported for servers which have some
// blocks of a region local, then the locality for that pair is 0.
localityPerServer[i][j * slotsPerServer + k] = locality.floatValue();
}
}
}
// Compute the total rack locality for each region in each rack. The total
// rack locality is the sum of the localities of a region on all servers in
// a rack.
Map<String, Map<HRegionInfo, Float>> rackRegionLocality =
new HashMap<String, Map<HRegionInfo, Float>>();
for (int i = 0; i < numRegions; i++) {
HRegionInfo region = regions.get(i);
for (int j = 0; j < regionSlots; j += slotsPerServer) {
String rack = rackManager.getRack(servers.get(j / slotsPerServer));
Map<HRegionInfo, Float> rackLocality = rackRegionLocality.get(rack);
if (rackLocality == null) {
rackLocality = new HashMap<HRegionInfo, Float>();
rackRegionLocality.put(rack, rackLocality);
}
Float localityObj = rackLocality.get(region);
float locality = localityObj == null ? 0 : localityObj.floatValue();
locality += localityPerServer[i][j];
rackLocality.put(region, locality);
}
}
for (int i = 0; i < numRegions; i++) {
for (int j = 0; j < regionSlots; j++) {
String rack = rackManager.getRack(servers.get(j / slotsPerServer));
Float totalRackLocalityObj = rackRegionLocality.get(rack).get(regions.get(i));
float totalRackLocality =
totalRackLocalityObj == null ? 0 : totalRackLocalityObj.floatValue();
// Primary cost aims to favor servers with high node locality and low
// rack locality, so that secondaries and tertiaries can be chosen for
// nodes with high rack locality. This might give primaries with
// slightly less locality at first compared to a cost which only
// considers the node locality, but should be better in the long run.
primaryCost[i][j] = 1 - (2 * localityPerServer[i][j] - totalRackLocality);
// Secondary cost aims to favor servers with high node locality and high
// rack locality since the tertiary will be chosen from the same rack as
// the secondary. This could be negative, but that is okay.
secondaryCost[i][j] = 2 - (localityPerServer[i][j] + totalRackLocality);
// Tertiary cost is only concerned with the node locality. It will later
// be restricted to only hosts on the same rack as the secondary.
tertiaryCost[i][j] = 1 - localityPerServer[i][j];
}
}
}
if (this.enforceMinAssignmentMove && currentAssignmentMap != null) {
// We want to minimize the number of regions which move as the result of a
// new assignment. Therefore, slightly penalize any placement which is for
// a host that is not currently serving the region.
for (int i = 0; i < numRegions; i++) {
for (int j = 0; j < servers.size(); j++) {
ServerName currentAddress = currentAssignmentMap.get(regions.get(i));
if (currentAddress != null && !currentAddress.equals(servers.get(j))) {
for (int k = 0; k < slotsPerServer; k++) {
primaryCost[i][j * slotsPerServer + k] += NOT_CURRENT_HOST_PENALTY;
}
}
}
}
}
// Artificially increase cost of last slot of each server to evenly
// distribute the slop, otherwise there will be a few servers with too few
// regions and many servers with the max number of regions.
for (int i = 0; i < numRegions; i++) {
for (int j = 0; j < regionSlots; j += slotsPerServer) {
primaryCost[i][j] += LAST_SLOT_COST_PENALTY;
secondaryCost[i][j] += LAST_SLOT_COST_PENALTY;
tertiaryCost[i][j] += LAST_SLOT_COST_PENALTY;
}
}
RandomizedMatrix randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots);
primaryCost = randomizedMatrix.transform(primaryCost);
int[] primaryAssignment = new MunkresAssignment(primaryCost).solve();
primaryAssignment = randomizedMatrix.invertIndices(primaryAssignment);
// Modify the secondary and tertiary costs for each region/server pair to
// prevent a region from being assigned to the same rack for both primary
// and either one of secondary or tertiary.
for (int i = 0; i < numRegions; i++) {
int slot = primaryAssignment[i];
String rack = rackManager.getRack(servers.get(slot / slotsPerServer));
for (int k = 0; k < servers.size(); k++) {
if (!rackManager.getRack(servers.get(k)).equals(rack)) {
continue;
}
if (k == slot / slotsPerServer) {
// Same node, do not place secondary or tertiary here ever.
for (int m = 0; m < slotsPerServer; m++) {
secondaryCost[i][k * slotsPerServer + m] = MAX_COST;
tertiaryCost[i][k * slotsPerServer + m] = MAX_COST;
}
} else {
// Same rack, do not place secondary or tertiary here if possible.
for (int m = 0; m < slotsPerServer; m++) {
secondaryCost[i][k * slotsPerServer + m] = AVOID_COST;
tertiaryCost[i][k * slotsPerServer + m] = AVOID_COST;
}
}
}
}
if (munkresForSecondaryAndTertiary) {
randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots);
secondaryCost = randomizedMatrix.transform(secondaryCost);
int[] secondaryAssignment = new MunkresAssignment(secondaryCost).solve();
secondaryAssignment = randomizedMatrix.invertIndices(secondaryAssignment);
// Modify the tertiary costs for each region/server pair to ensure that a
// region is assigned to a tertiary server on the same rack as its secondary
// server, but not the same server in that rack.
for (int i = 0; i < numRegions; i++) {
int slot = secondaryAssignment[i];
String rack = rackManager.getRack(servers.get(slot / slotsPerServer));
for (int k = 0; k < servers.size(); k++) {
if (k == slot / slotsPerServer) {
// Same node, do not place tertiary here ever.
for (int m = 0; m < slotsPerServer; m++) {
tertiaryCost[i][k * slotsPerServer + m] = MAX_COST;
}
} else {
if (rackManager.getRack(servers.get(k)).equals(rack)) {
continue;
}
// Different rack, do not place tertiary here if possible.
for (int m = 0; m < slotsPerServer; m++) {
tertiaryCost[i][k * slotsPerServer + m] = AVOID_COST;
}
}
}
}
randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots);
tertiaryCost = randomizedMatrix.transform(tertiaryCost);
int[] tertiaryAssignment = new MunkresAssignment(tertiaryCost).solve();
tertiaryAssignment = randomizedMatrix.invertIndices(tertiaryAssignment);
for (int i = 0; i < numRegions; i++) {
List<ServerName> favoredServers =
new ArrayList<ServerName>(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM);
ServerName s = servers.get(primaryAssignment[i] / slotsPerServer);
favoredServers.add(
ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));
s = servers.get(secondaryAssignment[i] / slotsPerServer);
favoredServers.add(
ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));
s = servers.get(tertiaryAssignment[i] / slotsPerServer);
favoredServers.add(
ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));
// Update the assignment plan
plan.updateAssignmentPlan(regions.get(i), favoredServers);
}
LOG.info(
"Generated the assignment plan for "
+ numRegions
+ " regions from table "
+ tableName
+ " with "
+ servers.size()
+ " region servers");
LOG.info("Assignment plan for secondary and tertiary generated " + "using MunkresAssignment");
} else {
Map<HRegionInfo, ServerName> primaryRSMap = new HashMap<HRegionInfo, ServerName>();
for (int i = 0; i < numRegions; i++) {
primaryRSMap.put(regions.get(i), servers.get(primaryAssignment[i] / slotsPerServer));
}
FavoredNodeAssignmentHelper favoredNodeHelper =
new FavoredNodeAssignmentHelper(servers, conf);
favoredNodeHelper.initialize();
Map<HRegionInfo, ServerName[]> secondaryAndTertiaryMap =
favoredNodeHelper.placeSecondaryAndTertiaryWithRestrictions(primaryRSMap);
for (int i = 0; i < numRegions; i++) {
List<ServerName> favoredServers =
new ArrayList<ServerName>(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM);
HRegionInfo currentRegion = regions.get(i);
ServerName s = primaryRSMap.get(currentRegion);
favoredServers.add(
ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));
ServerName[] secondaryAndTertiary = secondaryAndTertiaryMap.get(currentRegion);
s = secondaryAndTertiary[0];
favoredServers.add(
ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));
s = secondaryAndTertiary[1];
favoredServers.add(
ServerName.valueOf(s.getHostname(), s.getPort(), ServerName.NON_STARTCODE));
// Update the assignment plan
plan.updateAssignmentPlan(regions.get(i), favoredServers);
}
LOG.info(
"Generated the assignment plan for "
+ numRegions
+ " regions from table "
+ tableName
+ " with "
+ servers.size()
+ " region servers");
LOG.info(
"Assignment plan for secondary and tertiary generated "
+ "using placeSecondaryAndTertiaryWithRestrictions method");
}
}
/**
* @return the new RegionAssignmentSnapshot
* @throws IOException
*/
public SnapshotOfRegionAssignmentFromMeta getRegionAssignmentSnapshot() throws IOException {
SnapshotOfRegionAssignmentFromMeta currentAssignmentShapshot =
new SnapshotOfRegionAssignmentFromMeta(ConnectionFactory.createConnection(conf));
currentAssignmentShapshot.initialize();
return currentAssignmentShapshot;
}
public void fillUp(
TableName tableName,
SnapshotOfRegionAssignmentFromMeta snapshot,
Map<String, Map<String, Float>> regionLocalityMap) {
// Set the table name
this.tableName = tableName;
// Get all the regions for this table
List<HRegionInfo> regionInfoList = snapshot.getTableToRegionMap().get(tableName);
// Get the total region num for the current table
this.totalRegions = regionInfoList.size();
// Get the existing assignment plan
FavoredNodesPlan favoredNodesAssignment = snapshot.getExistingAssignmentPlan();
// Get the region to region server mapping
Map<HRegionInfo, ServerName> currentAssignment = snapshot.getRegionToRegionServerMap();
// Initialize the server to its hosing region counter map
Map<ServerName, Integer> serverToHostingRegionCounterMap = new HashMap<ServerName, Integer>();
Map<ServerName, Integer> primaryRSToRegionCounterMap = new HashMap<ServerName, Integer>();
Map<ServerName, Set<ServerName>> primaryToSecTerRSMap =
new HashMap<ServerName, Set<ServerName>>();
// Check the favored nodes and its locality information
// Also keep tracker of the most loaded and least loaded region servers
for (HRegionInfo region : regionInfoList) {
try {
ServerName currentRS = currentAssignment.get(region);
// Handle unassigned regions
if (currentRS == null) {
unAssignedRegionsList.add(region);
continue;
}
// Keep updating the server to is hosting region counter map
Integer hostRegionCounter = serverToHostingRegionCounterMap.get(currentRS);
if (hostRegionCounter == null) {
hostRegionCounter = Integer.valueOf(0);
}
hostRegionCounter = hostRegionCounter.intValue() + 1;
serverToHostingRegionCounterMap.put(currentRS, hostRegionCounter);
// Get the favored nodes from the assignment plan and verify it.
List<ServerName> favoredNodes = favoredNodesAssignment.getFavoredNodes(region);
if (favoredNodes == null
|| favoredNodes.size() != FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) {
regionsWithoutValidFavoredNodes.add(region);
continue;
}
// Get the primary, secondary and tertiary region server
ServerName primaryRS = favoredNodes.get(FavoredNodesPlan.Position.PRIMARY.ordinal());
ServerName secondaryRS = favoredNodes.get(FavoredNodesPlan.Position.SECONDARY.ordinal());
ServerName tertiaryRS = favoredNodes.get(FavoredNodesPlan.Position.TERTIARY.ordinal());
// Update the primary rs to its region set map
Integer regionCounter = primaryRSToRegionCounterMap.get(primaryRS);
if (regionCounter == null) {
regionCounter = Integer.valueOf(0);
}
regionCounter = regionCounter.intValue() + 1;
primaryRSToRegionCounterMap.put(primaryRS, regionCounter);
// Update the primary rs to secondary and tertiary rs map
Set<ServerName> secAndTerSet = primaryToSecTerRSMap.get(primaryRS);
if (secAndTerSet == null) {
secAndTerSet = new HashSet<ServerName>();
}
secAndTerSet.add(secondaryRS);
secAndTerSet.add(tertiaryRS);
primaryToSecTerRSMap.put(primaryRS, secAndTerSet);
// Get the position of the current region server in the favored nodes list
FavoredNodesPlan.Position favoredNodePosition =
FavoredNodesPlan.getFavoredServerPosition(favoredNodes, currentRS);
// Handle the non favored assignment.
if (favoredNodePosition == null) {
nonFavoredAssignedRegionList.add(region);
continue;
}
// Increase the favored nodes assignment.
this.favoredNodes[favoredNodePosition.ordinal()]++;
totalFavoredAssignments++;
// Summary the locality information for each favored nodes
if (regionLocalityMap != null) {
// Set the enforce locality as true;
this.enforceLocality = true;
// Get the region degree locality map
Map<String, Float> regionDegreeLocalityMap =
regionLocalityMap.get(region.getEncodedName());
if (regionDegreeLocalityMap == null) {
continue; // ignore the region which doesn't have any store files.
}
// Get the locality summary for each favored nodes
for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) {
ServerName favoredNode = favoredNodes.get(p.ordinal());
// Get the locality for the current favored nodes
Float locality = regionDegreeLocalityMap.get(favoredNode.getHostname());
if (locality != null) {
this.favoredNodesLocalitySummary[p.ordinal()] += locality;
}
}
// Get the locality summary for the current region server
Float actualLocality = regionDegreeLocalityMap.get(currentRS.getHostname());
if (actualLocality != null) {
this.actualLocalitySummary += actualLocality;
}
}
} catch (Exception e) {
LOG.error(
"Cannot verify the region assignment for region "
+ ((region == null) ? " null " : region.getRegionNameAsString())
+ "because of "
+ e);
}
}
float dispersionScoreSummary = 0;
float dispersionNumSummary = 0;
// Calculate the secondary score for each primary region server
for (Map.Entry<ServerName, Integer> entry : primaryRSToRegionCounterMap.entrySet()) {
ServerName primaryRS = entry.getKey();
Integer regionsOnPrimary = entry.getValue();
// Process the dispersion number and score
float dispersionScore = 0;
int dispersionNum = 0;
if (primaryToSecTerRSMap.get(primaryRS) != null && regionsOnPrimary.intValue() != 0) {
dispersionNum = primaryToSecTerRSMap.get(primaryRS).size();
dispersionScore = dispersionNum / ((float) regionsOnPrimary.intValue() * 2);
}
// Update the max dispersion score
if (dispersionScore > this.maxDispersionScore) {
this.maxDispersionScoreServerSet.clear();
this.maxDispersionScoreServerSet.add(primaryRS);
this.maxDispersionScore = dispersionScore;
} else if (dispersionScore == this.maxDispersionScore) {
this.maxDispersionScoreServerSet.add(primaryRS);
}
// Update the max dispersion num
if (dispersionNum > this.maxDispersionNum) {
this.maxDispersionNumServerSet.clear();
this.maxDispersionNumServerSet.add(primaryRS);
this.maxDispersionNum = dispersionNum;
} else if (dispersionNum == this.maxDispersionNum) {
this.maxDispersionNumServerSet.add(primaryRS);
}
// Update the min dispersion score
if (dispersionScore < this.minDispersionScore) {
this.minDispersionScoreServerSet.clear();
this.minDispersionScoreServerSet.add(primaryRS);
this.minDispersionScore = dispersionScore;
} else if (dispersionScore == this.minDispersionScore) {
this.minDispersionScoreServerSet.add(primaryRS);
}
// Update the min dispersion num
if (dispersionNum < this.minDispersionNum) {
this.minDispersionNumServerSet.clear();
this.minDispersionNumServerSet.add(primaryRS);
this.minDispersionNum = dispersionNum;
} else if (dispersionNum == this.minDispersionNum) {
this.minDispersionNumServerSet.add(primaryRS);
}
dispersionScoreSummary += dispersionScore;
dispersionNumSummary += dispersionNum;
}
// Update the avg dispersion score
if (primaryRSToRegionCounterMap.keySet().size() != 0) {
this.avgDispersionScore =
dispersionScoreSummary / (float) primaryRSToRegionCounterMap.keySet().size();
this.avgDispersionNum =
dispersionNumSummary / (float) primaryRSToRegionCounterMap.keySet().size();
}
// Fill up the most loaded and least loaded region server information
for (Map.Entry<ServerName, Integer> entry : serverToHostingRegionCounterMap.entrySet()) {
ServerName currentRS = entry.getKey();
int hostRegionCounter = entry.getValue().intValue();
// Update the most loaded region server list and maxRegionsOnRS
if (hostRegionCounter > this.maxRegionsOnRS) {
maxRegionsOnRS = hostRegionCounter;
this.mostLoadedRSSet.clear();
this.mostLoadedRSSet.add(currentRS);
} else if (hostRegionCounter == this.maxRegionsOnRS) {
this.mostLoadedRSSet.add(currentRS);
}
// Update the least loaded region server list and minRegionsOnRS
if (hostRegionCounter < this.minRegionsOnRS) {
this.minRegionsOnRS = hostRegionCounter;
this.leastLoadedRSSet.clear();
this.leastLoadedRSSet.add(currentRS);
} else if (hostRegionCounter == this.minRegionsOnRS) {
this.leastLoadedRSSet.add(currentRS);
}
}
// and total region servers
this.totalRegionServers = serverToHostingRegionCounterMap.keySet().size();
this.avgRegionsOnRS =
(totalRegionServers == 0) ? 0 : (totalRegions / (float) totalRegionServers);
// Set the isFilledUp as true
isFilledUp = true;
}
/**
* Use this to project the dispersion scores
*
* @param tableName
* @param snapshot
* @param newPlan
*/
public void fillUpDispersion(
TableName tableName, SnapshotOfRegionAssignmentFromMeta snapshot, FavoredNodesPlan newPlan) {
// Set the table name
this.tableName = tableName;
// Get all the regions for this table
List<HRegionInfo> regionInfoList = snapshot.getTableToRegionMap().get(tableName);
// Get the total region num for the current table
this.totalRegions = regionInfoList.size();
FavoredNodesPlan plan = null;
if (newPlan == null) {
plan = snapshot.getExistingAssignmentPlan();
} else {
plan = newPlan;
}
// Get the region to region server mapping
Map<ServerName, Integer> primaryRSToRegionCounterMap = new HashMap<ServerName, Integer>();
Map<ServerName, Set<ServerName>> primaryToSecTerRSMap =
new HashMap<ServerName, Set<ServerName>>();
// Check the favored nodes and its locality information
// Also keep tracker of the most loaded and least loaded region servers
for (HRegionInfo region : regionInfoList) {
try {
// Get the favored nodes from the assignment plan and verify it.
List<ServerName> favoredNodes = plan.getFavoredNodes(region);
if (favoredNodes == null
|| favoredNodes.size() != FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) {
regionsWithoutValidFavoredNodes.add(region);
continue;
}
// Get the primary, secondary and tertiary region server
ServerName primaryRS = favoredNodes.get(FavoredNodesPlan.Position.PRIMARY.ordinal());
ServerName secondaryRS = favoredNodes.get(FavoredNodesPlan.Position.SECONDARY.ordinal());
ServerName tertiaryRS = favoredNodes.get(FavoredNodesPlan.Position.TERTIARY.ordinal());
// Update the primary rs to its region set map
Integer regionCounter = primaryRSToRegionCounterMap.get(primaryRS);
if (regionCounter == null) {
regionCounter = Integer.valueOf(0);
}
regionCounter = regionCounter.intValue() + 1;
primaryRSToRegionCounterMap.put(primaryRS, regionCounter);
// Update the primary rs to secondary and tertiary rs map
Set<ServerName> secAndTerSet = primaryToSecTerRSMap.get(primaryRS);
if (secAndTerSet == null) {
secAndTerSet = new HashSet<ServerName>();
}
secAndTerSet.add(secondaryRS);
secAndTerSet.add(tertiaryRS);
primaryToSecTerRSMap.put(primaryRS, secAndTerSet);
} catch (Exception e) {
LOG.error(
"Cannot verify the region assignment for region "
+ ((region == null) ? " null " : region.getRegionNameAsString())
+ "because of "
+ e);
}
}
float dispersionScoreSummary = 0;
float dispersionNumSummary = 0;
// Calculate the secondary score for each primary region server
for (Map.Entry<ServerName, Integer> entry : primaryRSToRegionCounterMap.entrySet()) {
ServerName primaryRS = entry.getKey();
Integer regionsOnPrimary = entry.getValue();
// Process the dispersion number and score
float dispersionScore = 0;
int dispersionNum = 0;
if (primaryToSecTerRSMap.get(primaryRS) != null && regionsOnPrimary.intValue() != 0) {
dispersionNum = primaryToSecTerRSMap.get(primaryRS).size();
dispersionScore = dispersionNum / ((float) regionsOnPrimary.intValue() * 2);
}
// Update the max dispersion num
if (dispersionNum > this.maxDispersionNum) {
this.maxDispersionNumServerSet.clear();
this.maxDispersionNumServerSet.add(primaryRS);
this.maxDispersionNum = dispersionNum;
} else if (dispersionNum == this.maxDispersionNum) {
this.maxDispersionNumServerSet.add(primaryRS);
}
// Update the min dispersion score
if (dispersionScore < this.minDispersionScore) {
this.minDispersionScoreServerSet.clear();
this.minDispersionScoreServerSet.add(primaryRS);
this.minDispersionScore = dispersionScore;
} else if (dispersionScore == this.minDispersionScore) {
this.minDispersionScoreServerSet.add(primaryRS);
}
// Update the min dispersion num
if (dispersionNum < this.minDispersionNum) {
this.minDispersionNumServerSet.clear();
this.minDispersionNumServerSet.add(primaryRS);
this.minDispersionNum = dispersionNum;
} else if (dispersionNum == this.minDispersionNum) {
this.minDispersionNumServerSet.add(primaryRS);
}
dispersionScoreSummary += dispersionScore;
dispersionNumSummary += dispersionNum;
}
// Update the avg dispersion score
if (primaryRSToRegionCounterMap.keySet().size() != 0) {
this.avgDispersionScore =
dispersionScoreSummary / (float) primaryRSToRegionCounterMap.keySet().size();
this.avgDispersionNum =
dispersionNumSummary / (float) primaryRSToRegionCounterMap.keySet().size();
}
}
/** This tests retaining assignments on a cluster restart */
@Test(timeout = 300000)
public void testRetainAssignmentOnRestart() throws Exception {
UTIL.startMiniCluster(2);
while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
Threads.sleep(1);
}
// Turn off balancer
UTIL.getMiniHBaseCluster().getMaster().getMasterRpcServices().synchronousBalanceSwitch(false);
LOG.info("\n\nCreating tables");
for (byte[] TABLE : TABLES) {
UTIL.createTable(TABLE, FAMILY);
}
for (byte[] TABLE : TABLES) {
UTIL.waitTableEnabled(TABLE);
}
HMaster master = UTIL.getMiniHBaseCluster().getMaster();
UTIL.waitUntilNoRegionsInTransition(120000);
// We don't have to use SnapshotOfRegionAssignmentFromMeta.
// We use it here because AM used to use it to load all user region placements
SnapshotOfRegionAssignmentFromMeta snapshot =
new SnapshotOfRegionAssignmentFromMeta(master.getShortCircuitConnection());
snapshot.initialize();
Map<HRegionInfo, ServerName> regionToRegionServerMap = snapshot.getRegionToRegionServerMap();
MiniHBaseCluster cluster = UTIL.getHBaseCluster();
List<JVMClusterUtil.RegionServerThread> threads = cluster.getLiveRegionServerThreads();
assertEquals(2, threads.size());
int[] rsPorts = new int[3];
for (int i = 0; i < 2; i++) {
rsPorts[i] = threads.get(i).getRegionServer().getServerName().getPort();
}
rsPorts[2] = cluster.getMaster().getServerName().getPort();
for (ServerName serverName : regionToRegionServerMap.values()) {
boolean found = false; // Test only, no need to optimize
for (int k = 0; k < 3 && !found; k++) {
found = serverName.getPort() == rsPorts[k];
}
assertTrue(found);
}
LOG.info("\n\nShutting down HBase cluster");
cluster.shutdown();
cluster.waitUntilShutDown();
LOG.info("\n\nSleeping a bit");
Thread.sleep(2000);
LOG.info("\n\nStarting cluster the second time with the same ports");
try {
cluster.getConf().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 4);
master = cluster.startMaster().getMaster();
for (int i = 0; i < 3; i++) {
cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, rsPorts[i]);
cluster.startRegionServer();
}
} finally {
// Reset region server port so as not to conflict with other tests
cluster.getConf().setInt(HConstants.REGIONSERVER_PORT, 0);
cluster.getConf().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 2);
}
// Make sure live regionservers are on the same host/port
List<ServerName> localServers = master.getServerManager().getOnlineServersList();
assertEquals(4, localServers.size());
for (int i = 0; i < 3; i++) {
boolean found = false;
for (ServerName serverName : localServers) {
if (serverName.getPort() == rsPorts[i]) {
found = true;
break;
}
}
assertTrue(found);
}
// Wait till master is initialized and all regions are assigned
RegionStates regionStates = master.getAssignmentManager().getRegionStates();
int expectedRegions = regionToRegionServerMap.size() + 1;
while (!master.isInitialized()
|| regionStates.getRegionAssignments().size() != expectedRegions) {
Threads.sleep(100);
}
snapshot = new SnapshotOfRegionAssignmentFromMeta(master.getShortCircuitConnection());
snapshot.initialize();
Map<HRegionInfo, ServerName> newRegionToRegionServerMap = snapshot.getRegionToRegionServerMap();
assertEquals(regionToRegionServerMap.size(), newRegionToRegionServerMap.size());
for (Map.Entry<HRegionInfo, ServerName> entry : newRegionToRegionServerMap.entrySet()) {
if (TableName.NAMESPACE_TABLE_NAME.equals(entry.getKey().getTable())) continue;
ServerName oldServer = regionToRegionServerMap.get(entry.getKey());
ServerName currentServer = entry.getValue();
assertEquals(oldServer.getHostAndPort(), currentServer.getHostAndPort());
assertNotEquals(oldServer.getStartcode(), currentServer.getStartcode());
}
}