コード例 #1
0
  public void fillUp(
      TableName tableName,
      SnapshotOfRegionAssignmentFromMeta snapshot,
      Map<String, Map<String, Float>> regionLocalityMap) {
    // Set the table name
    this.tableName = tableName;

    // Get all the regions for this table
    List<HRegionInfo> regionInfoList = snapshot.getTableToRegionMap().get(tableName);
    // Get the total region num for the current table
    this.totalRegions = regionInfoList.size();

    // Get the existing assignment plan
    FavoredNodesPlan favoredNodesAssignment = snapshot.getExistingAssignmentPlan();
    // Get the region to region server mapping
    Map<HRegionInfo, ServerName> currentAssignment = snapshot.getRegionToRegionServerMap();
    // Initialize the server to its hosing region counter map
    Map<ServerName, Integer> serverToHostingRegionCounterMap = new HashMap<ServerName, Integer>();

    Map<ServerName, Integer> primaryRSToRegionCounterMap = new HashMap<ServerName, Integer>();
    Map<ServerName, Set<ServerName>> primaryToSecTerRSMap =
        new HashMap<ServerName, Set<ServerName>>();

    // Check the favored nodes and its locality information
    // Also keep tracker of the most loaded and least loaded region servers
    for (HRegionInfo region : regionInfoList) {
      try {
        ServerName currentRS = currentAssignment.get(region);
        // Handle unassigned regions
        if (currentRS == null) {
          unAssignedRegionsList.add(region);
          continue;
        }

        // Keep updating the server to is hosting region counter map
        Integer hostRegionCounter = serverToHostingRegionCounterMap.get(currentRS);
        if (hostRegionCounter == null) {
          hostRegionCounter = Integer.valueOf(0);
        }
        hostRegionCounter = hostRegionCounter.intValue() + 1;
        serverToHostingRegionCounterMap.put(currentRS, hostRegionCounter);

        // Get the favored nodes from the assignment plan and verify it.
        List<ServerName> favoredNodes = favoredNodesAssignment.getFavoredNodes(region);
        if (favoredNodes == null
            || favoredNodes.size() != FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) {
          regionsWithoutValidFavoredNodes.add(region);
          continue;
        }
        // Get the primary, secondary and tertiary region server
        ServerName primaryRS = favoredNodes.get(FavoredNodesPlan.Position.PRIMARY.ordinal());
        ServerName secondaryRS = favoredNodes.get(FavoredNodesPlan.Position.SECONDARY.ordinal());
        ServerName tertiaryRS = favoredNodes.get(FavoredNodesPlan.Position.TERTIARY.ordinal());

        // Update the primary rs to its region set map
        Integer regionCounter = primaryRSToRegionCounterMap.get(primaryRS);
        if (regionCounter == null) {
          regionCounter = Integer.valueOf(0);
        }
        regionCounter = regionCounter.intValue() + 1;
        primaryRSToRegionCounterMap.put(primaryRS, regionCounter);

        // Update the primary rs to secondary and tertiary rs map
        Set<ServerName> secAndTerSet = primaryToSecTerRSMap.get(primaryRS);
        if (secAndTerSet == null) {
          secAndTerSet = new HashSet<ServerName>();
        }
        secAndTerSet.add(secondaryRS);
        secAndTerSet.add(tertiaryRS);
        primaryToSecTerRSMap.put(primaryRS, secAndTerSet);

        // Get the position of the current region server in the favored nodes list
        FavoredNodesPlan.Position favoredNodePosition =
            FavoredNodesPlan.getFavoredServerPosition(favoredNodes, currentRS);

        // Handle the non favored assignment.
        if (favoredNodePosition == null) {
          nonFavoredAssignedRegionList.add(region);
          continue;
        }
        // Increase the favored nodes assignment.
        this.favoredNodes[favoredNodePosition.ordinal()]++;
        totalFavoredAssignments++;

        // Summary the locality information for each favored nodes
        if (regionLocalityMap != null) {
          // Set the enforce locality as true;
          this.enforceLocality = true;

          // Get the region degree locality map
          Map<String, Float> regionDegreeLocalityMap =
              regionLocalityMap.get(region.getEncodedName());
          if (regionDegreeLocalityMap == null) {
            continue; // ignore the region which doesn't have any store files.
          }

          // Get the locality summary for each favored nodes
          for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) {
            ServerName favoredNode = favoredNodes.get(p.ordinal());
            // Get the locality for the current favored nodes
            Float locality = regionDegreeLocalityMap.get(favoredNode.getHostname());
            if (locality != null) {
              this.favoredNodesLocalitySummary[p.ordinal()] += locality;
            }
          }

          // Get the locality summary for the current region server
          Float actualLocality = regionDegreeLocalityMap.get(currentRS.getHostname());
          if (actualLocality != null) {
            this.actualLocalitySummary += actualLocality;
          }
        }
      } catch (Exception e) {
        LOG.error(
            "Cannot verify the region assignment for region "
                + ((region == null) ? " null " : region.getRegionNameAsString())
                + "because of "
                + e);
      }
    }

    float dispersionScoreSummary = 0;
    float dispersionNumSummary = 0;
    // Calculate the secondary score for each primary region server
    for (Map.Entry<ServerName, Integer> entry : primaryRSToRegionCounterMap.entrySet()) {
      ServerName primaryRS = entry.getKey();
      Integer regionsOnPrimary = entry.getValue();

      // Process the dispersion number and score
      float dispersionScore = 0;
      int dispersionNum = 0;
      if (primaryToSecTerRSMap.get(primaryRS) != null && regionsOnPrimary.intValue() != 0) {
        dispersionNum = primaryToSecTerRSMap.get(primaryRS).size();
        dispersionScore = dispersionNum / ((float) regionsOnPrimary.intValue() * 2);
      }
      // Update the max dispersion score
      if (dispersionScore > this.maxDispersionScore) {
        this.maxDispersionScoreServerSet.clear();
        this.maxDispersionScoreServerSet.add(primaryRS);
        this.maxDispersionScore = dispersionScore;
      } else if (dispersionScore == this.maxDispersionScore) {
        this.maxDispersionScoreServerSet.add(primaryRS);
      }

      // Update the max dispersion num
      if (dispersionNum > this.maxDispersionNum) {
        this.maxDispersionNumServerSet.clear();
        this.maxDispersionNumServerSet.add(primaryRS);
        this.maxDispersionNum = dispersionNum;
      } else if (dispersionNum == this.maxDispersionNum) {
        this.maxDispersionNumServerSet.add(primaryRS);
      }

      // Update the min dispersion score
      if (dispersionScore < this.minDispersionScore) {
        this.minDispersionScoreServerSet.clear();
        this.minDispersionScoreServerSet.add(primaryRS);
        this.minDispersionScore = dispersionScore;
      } else if (dispersionScore == this.minDispersionScore) {
        this.minDispersionScoreServerSet.add(primaryRS);
      }

      // Update the min dispersion num
      if (dispersionNum < this.minDispersionNum) {
        this.minDispersionNumServerSet.clear();
        this.minDispersionNumServerSet.add(primaryRS);
        this.minDispersionNum = dispersionNum;
      } else if (dispersionNum == this.minDispersionNum) {
        this.minDispersionNumServerSet.add(primaryRS);
      }

      dispersionScoreSummary += dispersionScore;
      dispersionNumSummary += dispersionNum;
    }

    // Update the avg dispersion score
    if (primaryRSToRegionCounterMap.keySet().size() != 0) {
      this.avgDispersionScore =
          dispersionScoreSummary / (float) primaryRSToRegionCounterMap.keySet().size();
      this.avgDispersionNum =
          dispersionNumSummary / (float) primaryRSToRegionCounterMap.keySet().size();
    }

    // Fill up the most loaded and least loaded region server information
    for (Map.Entry<ServerName, Integer> entry : serverToHostingRegionCounterMap.entrySet()) {
      ServerName currentRS = entry.getKey();
      int hostRegionCounter = entry.getValue().intValue();

      // Update the most loaded region server list and maxRegionsOnRS
      if (hostRegionCounter > this.maxRegionsOnRS) {
        maxRegionsOnRS = hostRegionCounter;
        this.mostLoadedRSSet.clear();
        this.mostLoadedRSSet.add(currentRS);
      } else if (hostRegionCounter == this.maxRegionsOnRS) {
        this.mostLoadedRSSet.add(currentRS);
      }

      // Update the least loaded region server list and minRegionsOnRS
      if (hostRegionCounter < this.minRegionsOnRS) {
        this.minRegionsOnRS = hostRegionCounter;
        this.leastLoadedRSSet.clear();
        this.leastLoadedRSSet.add(currentRS);
      } else if (hostRegionCounter == this.minRegionsOnRS) {
        this.leastLoadedRSSet.add(currentRS);
      }
    }

    // and total region servers
    this.totalRegionServers = serverToHostingRegionCounterMap.keySet().size();
    this.avgRegionsOnRS =
        (totalRegionServers == 0) ? 0 : (totalRegions / (float) totalRegionServers);
    // Set the isFilledUp as true
    isFilledUp = true;
  }
コード例 #2
0
  /**
   * Use this to project the dispersion scores
   *
   * @param tableName
   * @param snapshot
   * @param newPlan
   */
  public void fillUpDispersion(
      TableName tableName, SnapshotOfRegionAssignmentFromMeta snapshot, FavoredNodesPlan newPlan) {
    // Set the table name
    this.tableName = tableName;
    // Get all the regions for this table
    List<HRegionInfo> regionInfoList = snapshot.getTableToRegionMap().get(tableName);
    // Get the total region num for the current table
    this.totalRegions = regionInfoList.size();
    FavoredNodesPlan plan = null;
    if (newPlan == null) {
      plan = snapshot.getExistingAssignmentPlan();
    } else {
      plan = newPlan;
    }
    // Get the region to region server mapping
    Map<ServerName, Integer> primaryRSToRegionCounterMap = new HashMap<ServerName, Integer>();
    Map<ServerName, Set<ServerName>> primaryToSecTerRSMap =
        new HashMap<ServerName, Set<ServerName>>();

    // Check the favored nodes and its locality information
    // Also keep tracker of the most loaded and least loaded region servers
    for (HRegionInfo region : regionInfoList) {
      try {
        // Get the favored nodes from the assignment plan and verify it.
        List<ServerName> favoredNodes = plan.getFavoredNodes(region);
        if (favoredNodes == null
            || favoredNodes.size() != FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) {
          regionsWithoutValidFavoredNodes.add(region);
          continue;
        }
        // Get the primary, secondary and tertiary region server
        ServerName primaryRS = favoredNodes.get(FavoredNodesPlan.Position.PRIMARY.ordinal());
        ServerName secondaryRS = favoredNodes.get(FavoredNodesPlan.Position.SECONDARY.ordinal());
        ServerName tertiaryRS = favoredNodes.get(FavoredNodesPlan.Position.TERTIARY.ordinal());

        // Update the primary rs to its region set map
        Integer regionCounter = primaryRSToRegionCounterMap.get(primaryRS);
        if (regionCounter == null) {
          regionCounter = Integer.valueOf(0);
        }
        regionCounter = regionCounter.intValue() + 1;
        primaryRSToRegionCounterMap.put(primaryRS, regionCounter);

        // Update the primary rs to secondary and tertiary rs map
        Set<ServerName> secAndTerSet = primaryToSecTerRSMap.get(primaryRS);
        if (secAndTerSet == null) {
          secAndTerSet = new HashSet<ServerName>();
        }
        secAndTerSet.add(secondaryRS);
        secAndTerSet.add(tertiaryRS);
        primaryToSecTerRSMap.put(primaryRS, secAndTerSet);
      } catch (Exception e) {
        LOG.error(
            "Cannot verify the region assignment for region "
                + ((region == null) ? " null " : region.getRegionNameAsString())
                + "because of "
                + e);
      }
    }
    float dispersionScoreSummary = 0;
    float dispersionNumSummary = 0;
    // Calculate the secondary score for each primary region server
    for (Map.Entry<ServerName, Integer> entry : primaryRSToRegionCounterMap.entrySet()) {
      ServerName primaryRS = entry.getKey();
      Integer regionsOnPrimary = entry.getValue();

      // Process the dispersion number and score
      float dispersionScore = 0;
      int dispersionNum = 0;
      if (primaryToSecTerRSMap.get(primaryRS) != null && regionsOnPrimary.intValue() != 0) {
        dispersionNum = primaryToSecTerRSMap.get(primaryRS).size();
        dispersionScore = dispersionNum / ((float) regionsOnPrimary.intValue() * 2);
      }

      // Update the max dispersion num
      if (dispersionNum > this.maxDispersionNum) {
        this.maxDispersionNumServerSet.clear();
        this.maxDispersionNumServerSet.add(primaryRS);
        this.maxDispersionNum = dispersionNum;
      } else if (dispersionNum == this.maxDispersionNum) {
        this.maxDispersionNumServerSet.add(primaryRS);
      }

      // Update the min dispersion score
      if (dispersionScore < this.minDispersionScore) {
        this.minDispersionScoreServerSet.clear();
        this.minDispersionScoreServerSet.add(primaryRS);
        this.minDispersionScore = dispersionScore;
      } else if (dispersionScore == this.minDispersionScore) {
        this.minDispersionScoreServerSet.add(primaryRS);
      }

      // Update the min dispersion num
      if (dispersionNum < this.minDispersionNum) {
        this.minDispersionNumServerSet.clear();
        this.minDispersionNumServerSet.add(primaryRS);
        this.minDispersionNum = dispersionNum;
      } else if (dispersionNum == this.minDispersionNum) {
        this.minDispersionNumServerSet.add(primaryRS);
      }

      dispersionScoreSummary += dispersionScore;
      dispersionNumSummary += dispersionNum;
    }

    // Update the avg dispersion score
    if (primaryRSToRegionCounterMap.keySet().size() != 0) {
      this.avgDispersionScore =
          dispersionScoreSummary / (float) primaryRSToRegionCounterMap.keySet().size();
      this.avgDispersionNum =
          dispersionNumSummary / (float) primaryRSToRegionCounterMap.keySet().size();
    }
  }