private void buildAverageDiffs() throws TasteException {
   try {
     buildAveragesLock.writeLock().lock();
     DataModel dataModel = getDataModel();
     LongPrimitiveIterator it = dataModel.getUserIDs();
     while (it.hasNext()) {
       long userID = it.nextLong();
       PreferenceArray prefs = dataModel.getPreferencesFromUser(userID);
       int size = prefs.length();
       for (int i = 0; i < size; i++) {
         long itemID = prefs.getItemID(i);
         float value = prefs.getValue(i);
         addDatumAndCreateIfNeeded(itemID, value, itemAverages);
         addDatumAndCreateIfNeeded(userID, value, userAverages);
         overallAveragePrefValue.addDatum(value);
       }
     }
   } finally {
     buildAveragesLock.writeLock().unlock();
   }
 }
  public void testIUF() throws Exception {
    DataModel dataModel =
        getDataModel(
            new long[] {1, 2, 3, 4, 5},
            new Double[][] {
              {0.1}, {0.2, 0.3}, {0.4, 0.5, 0.6}, {0.7, 0.8, 0.9, 1.0}, {1.0, 1.0, 1.0, 1.0, 1.0},
            });

    InverseUserFrequency iuf = new InverseUserFrequency(dataModel, 10.0);

    PreferenceArray user5Prefs = dataModel.getPreferencesFromUser(5);

    for (int i = 0; i < 5; i++) {
      Preference pref = user5Prefs.get(i);
      assertNotNull(pref);
      assertEquals(
          Math.log(5.0 / (double) (5 - i)) / Math.log(iuf.getLogBase()),
          iuf.getTransformedValue(pref),
          EPSILON);
    }

    // Make sure this doesn't throw an exception
    iuf.refresh(null);
  }
  @Override
  public double userSimilarity(long userID1, long userID2) throws TasteException {
    PreferenceArray xPrefs = dataModel.getPreferencesFromUser(userID1);
    PreferenceArray yPrefs = dataModel.getPreferencesFromUser(userID2);
    int xLength = xPrefs.length();
    int yLength = yPrefs.length();

    if ((xLength == 0) || (yLength == 0)) {
      return Double.NaN;
    }

    long xIndex = xPrefs.getItemID(0);
    long yIndex = yPrefs.getItemID(0);
    int xPrefIndex = 0;
    int yPrefIndex = 0;

    double sumX = 0.0;
    double sumX2 = 0.0;
    double sumY = 0.0;
    double sumY2 = 0.0;
    double sumXY = 0.0;
    double sumXYdiff2 = 0.0;
    int count = 0;

    boolean hasInferrer = inferrer != null;
    boolean hasPrefTransform = prefTransform != null;

    while (true) {
      int compare = xIndex < yIndex ? -1 : xIndex > yIndex ? 1 : 0;
      if (hasInferrer || compare == 0) {
        double x;
        double y;
        if (xIndex == yIndex) {
          // Both users expressed a preference for the item
          if (hasPrefTransform) {
            x = prefTransform.getTransformedValue(xPrefs.get(xPrefIndex));
            y = prefTransform.getTransformedValue(yPrefs.get(yPrefIndex));
          } else {
            x = xPrefs.getValue(xPrefIndex);
            y = yPrefs.getValue(yPrefIndex);
          }
        } else {
          // Only one user expressed a preference, but infer the other one's preference and tally
          // as if the other user expressed that preference
          if (compare < 0) {
            // X has a value; infer Y's
            x =
                hasPrefTransform
                    ? prefTransform.getTransformedValue(xPrefs.get(xPrefIndex))
                    : xPrefs.getValue(xPrefIndex);
            y = inferrer.inferPreference(userID2, xIndex);
          } else {
            // compare > 0
            // Y has a value; infer X's
            x = inferrer.inferPreference(userID1, yIndex);
            y =
                hasPrefTransform
                    ? prefTransform.getTransformedValue(yPrefs.get(yPrefIndex))
                    : yPrefs.getValue(yPrefIndex);
          }
        }
        sumXY += x * y;
        sumX += x;
        sumX2 += x * x;
        sumY += y;
        sumY2 += y * y;
        double diff = x - y;
        sumXYdiff2 += diff * diff;
        count++;
      }
      if (compare <= 0) {
        if (++xPrefIndex >= xLength) {
          if (hasInferrer) {
            // Must count other Ys; pretend next X is far away
            if (yIndex == Long.MAX_VALUE) {
              // ... but stop if both are done!
              break;
            }
            xIndex = Long.MAX_VALUE;
          } else {
            break;
          }
        } else {
          xIndex = xPrefs.getItemID(xPrefIndex);
        }
      }
      if (compare >= 0) {
        if (++yPrefIndex >= yLength) {
          if (hasInferrer) {
            // Must count other Xs; pretend next Y is far away
            if (xIndex == Long.MAX_VALUE) {
              // ... but stop if both are done!
              break;
            }
            yIndex = Long.MAX_VALUE;
          } else {
            break;
          }
        } else {
          yIndex = yPrefs.getItemID(yPrefIndex);
        }
      }
    }

    // "Center" the data. If my math is correct, this'll do it.
    double result;
    if (centerData) {
      double n = count;
      double meanX = sumX / n;
      double meanY = sumY / n;
      // double centeredSumXY = sumXY - meanY * sumX - meanX * sumY + n * meanX * meanY;
      double centeredSumXY = sumXY - meanY * sumX;
      // double centeredSumX2 = sumX2 - 2.0 * meanX * sumX + n * meanX * meanX;
      double centeredSumX2 = sumX2 - meanX * sumX;
      // double centeredSumY2 = sumY2 - 2.0 * meanY * sumY + n * meanY * meanY;
      double centeredSumY2 = sumY2 - meanY * sumY;
      result = computeResult(count, centeredSumXY, centeredSumX2, centeredSumY2, sumXYdiff2);
    } else {
      result = computeResult(count, sumXY, sumX2, sumY2, sumXYdiff2);
    }

    if (similarityTransform != null) {
      result = similarityTransform.transformSimilarity(userID1, userID2, result);
    }

    if (!Double.isNaN(result)) {
      result = normalizeWeightResult(result, count, cachedNumItems);
    }
    return result;
  }
  private void splitOneUsersPrefs(
      double trainingPercentage,
      FastByIDMap<PreferenceArray> trainingPrefs,
      FastByIDMap<PreferenceArray> testPrefs,
      long userID,
      DataModel dataModel)
      throws TasteException {
    List<Preference> oneUserTrainingPrefs = null;
    List<Preference> oneUserTestPrefs = null;
    PreferenceArray prefs = dataModel.getPreferencesFromUser(userID);
    int size = prefs.length();
    boolean isInstanceOfContextualUserPreferenceArray =
        prefs instanceof ContextualUserPreferenceArray;

    for (int i = 0; i < size; i++) {
      Preference newPref =
          isInstanceOfContextualUserPreferenceArray
              ? new ContextualPreference(
                  userID,
                  prefs.getItemID(i),
                  prefs.getValue(i),
                  ((ContextualUserPreferenceArray) prefs).getContextualPreferences(i))
              : new GenericPreference(userID, prefs.getItemID(i), prefs.getValue(i));

      if (this.idrescorer != null && this.idrescorer.isFiltered(newPref.getItemID())) { // adiciona
        // ratings
        // de
        // source
        // domain
        // sempre
        // em
        // training
        // set
        if (oneUserTrainingPrefs == null) {
          oneUserTrainingPrefs = Lists.newArrayListWithCapacity(3);
        }
        oneUserTrainingPrefs.add(newPref);
        totalOfTrainingRatingsFromSource++;
        continue;
      }

      if (this.contextualCriteria != null
          && isInstanceOfContextualUserPreferenceArray) { // adiciona
        // ratings
        // de outro
        // contexto
        // sempre em
        // training
        // set
        ContextualPreference contextualPref = (ContextualPreference) newPref;
        if (!this.contextualCriteria.containsAllContextualAttributes(
            contextualPref.getContextualPreferences())) {
          if (oneUserTrainingPrefs == null) {
            oneUserTrainingPrefs = Lists.newArrayListWithCapacity(3);
          }
          oneUserTrainingPrefs.add(newPref);
          totalOfTrainingRatingsFromTargetWithoutContext++;
          continue;
        }
      }

      // para ratings do target e do contexto, fazer proporcao definida
      if (random.nextDouble() < trainingPercentage) {
        if (oneUserTrainingPrefs == null) {
          oneUserTrainingPrefs = Lists.newArrayListWithCapacity(3);
        }
        oneUserTrainingPrefs.add(newPref);
        totalOfTrainingRatingsFromTargetWithContext++;
      } else {
        if (oneUserTestPrefs == null) {
          oneUserTestPrefs = Lists.newArrayListWithCapacity(3);
        }
        oneUserTestPrefs.add(newPref);
        totalOfTestRatings++;
      }

      // OLD training/test set
      /*
       * if (random.nextDouble() < trainingPercentage) { if
       * (oneUserTrainingPrefs == null) { oneUserTrainingPrefs =
       * Lists.newArrayListWithCapacity(3); }
       * oneUserTrainingPrefs.add(newPref); totalOfTrainingRatings++; }
       * else { if (oneUserTestPrefs == null) { oneUserTestPrefs =
       * Lists.newArrayListWithCapacity(3); } //testa somente com um tipo
       * de rating (rescorer e/ou context) if(this.idrescorer == null &&
       * this.contextualCriteria == null){ oneUserTestPrefs.add(newPref);
       * totalOfTestRatings++; }else{ if(this.idrescorer != null &&
       * !this.idrescorer.isFiltered(newPref.getItemID())){
       * if(this.contextualCriteria != null &&
       * isInstanceOfContextualUserPreferenceArray){ ContextualPreference
       * contextualPref = (ContextualPreference) newPref;
       * if(this.contextualCriteria
       * .containsAllContextualAttributes(contextualPref
       * .getContextualPreferences())){ oneUserTestPrefs.add(newPref);
       * totalOfTestRatings++; } }else{ oneUserTestPrefs.add(newPref);
       * totalOfTestRatings++; } }else if(this.idrescorer == null &&
       * this.contextualCriteria != null &&
       * isInstanceOfContextualUserPreferenceArray){ ContextualPreference
       * contextualPref = (ContextualPreference) newPref;
       * if(this.contextualCriteria
       * .containsAllContextualAttributes(contextualPref
       * .getContextualPreferences())){ oneUserTestPrefs.add(newPref);
       * totalOfTestRatings++; } } } }
       */
    }
    if (oneUserTrainingPrefs != null) {
      trainingPrefs.put(
          userID,
          isInstanceOfContextualUserPreferenceArray
              ? new ContextualUserPreferenceArray(oneUserTrainingPrefs)
              : new GenericUserPreferenceArray(oneUserTrainingPrefs));
      if (oneUserTestPrefs != null) {
        testPrefs.put(
            userID,
            isInstanceOfContextualUserPreferenceArray
                ? new ContextualUserPreferenceArray(oneUserTestPrefs)
                : new GenericUserPreferenceArray(oneUserTestPrefs));
      }
    }
  }