Example #1
0
 @Override
 public Iterator<T> iterator() {
   return checkNotNull(fStartTimesIndex.values().iterator());
 }
  /**
   * Returns a HashMap containing the scores of the hosts. The slowest *normal* host gets a score of
   * 1.0 and the fastest gets 0. If the host was a straggler, its score is set to
   * HostMetric.STRAGGLER.
   *
   * @return a HashMap of the host name and its score
   */
  @Override
  public HashMap<String, Double> getHostsScores(String queryID) {
    HashMap<String, Double> scores = new HashMap<>();
    List<String> hostNames = myMetadataTracker.getHostNamesInQuery(queryID);
    if (hostNames == null || hostNames.size() == 0) {
      // Unknown query ID
      return null;
    }

    // Order the hosts by their slowness and assign a slowness score for each starting from 0 to 1.0
    TreeMultimap<Long, String> hostsDeltaTime = TreeMultimap.create();
    double sum = 0;
    double sumSquared = 0;
    double numberOfSamples = 0;

    synchronized (hostNames) {
      for (String hostName : hostNames) {
        Long deltaTime = myMetadataTracker.getHostDeltaTime(queryID, hostName);
        if (deltaTime == null && !myMetadataTracker.isRusher(queryID, hostName)) {
          // That's not a rusher and didn't finish then it is a straggler
          scores.put(hostName, HostMetric.STRAGGLER);
          continue;
        }

        if (deltaTime != null) {
          sum += deltaTime;
          sumSquared += deltaTime * deltaTime;
          numberOfSamples++;
          hostsDeltaTime.put(deltaTime, hostName);
        }

        // Else this host is a rusher
      }
    }

    if (numberOfSamples == 0) {
      // We had one host only and it is a straggler
      return scores;
    }

    // n cannot be 0 from here on....
    // Now mark the non-stragglers but slow hosts
    double mean = sum / numberOfSamples;
    double variance =
        (sumSquared - 2 * sum * mean + numberOfSamples * mean * mean) / numberOfSamples;
    double stdDevsAway = mean + multipleStdDevs * Math.sqrt(variance);

    if (log.isTraceEnabled()) {
      log.trace(
          "mean:"
              + mean
              + " stddev:"
              + Math.sqrt(variance)
              + " multiple:"
              + multipleStdDevs
              + " stdDevsAway:"
              + stdDevsAway);
    }

    ArrayList<Long> keysToDelete = new ArrayList<>();

    for (Map.Entry<Long, String> e : hostsDeltaTime.entries()) {
      if (e.getKey() > stdDevsAway) {
        keysToDelete.add(e.getKey());
        scores.put(e.getValue(), HostMetric.SLOW_HOST);
        if (log.isTraceEnabled()) {
          log.trace("Marking host:" + e.getValue() + " as slow");
        }
      }
    }

    // Remove the hosts that needs to be remove
    for (Long key : keysToDelete) {
      hostsDeltaTime.removeAll(key);
    }

    // If we have only one host to determine score for, then put 0.5
    if (hostsDeltaTime.size() == 1) {
      // Only one host responded
      scores.put(hostsDeltaTime.values().iterator().next(), 0.5);
      return scores;
    }

    // The number of segments is 1 less than the number of hosts to score. So if we have 3 hosts to
    // occupy
    // the score range from 0 -> 1, then we have two segments and their scores should be 0, 0.5 and
    // 1.
    double segmentSize = 1.0 / (hostsDeltaTime.size() - 1);

    // Since the data is sorting ascending in turnaround time, the first we meet are the fastest
    // hosts
    // and so they should get the lowest slowness score
    double score = 0;
    for (Map.Entry<Long, String> e : hostsDeltaTime.entries()) {
      scores.put(e.getValue(), score);
      score += segmentSize;
    }

    return scores;
  }