@Test
public void testStrategy() throws TasteException {
FastIDSet itemIDsFromUser123 = new FastIDSet();
itemIDsFromUser123.add(1L);
FastIDSet itemIDsFromUser456 = new FastIDSet();
itemIDsFromUser456.add(1L);
itemIDsFromUser456.add(2L);
List<Preference> prefs = new ArrayList<Preference>();
prefs.add(new GenericPreference(123L, 1L, 1.0f));
prefs.add(new GenericPreference(456L, 1L, 1.0f));
PreferenceArray preferencesForItem1 = new GenericItemPreferenceArray(prefs);
DataModel dataModel = EasyMock.createMock(DataModel.class);
EasyMock.expect(dataModel.getPreferencesForItem(1L)).andReturn(preferencesForItem1);
EasyMock.expect(dataModel.getItemIDsFromUser(123L)).andReturn(itemIDsFromUser123);
EasyMock.expect(dataModel.getItemIDsFromUser(456L)).andReturn(itemIDsFromUser456);
PreferenceArray prefArrayOfUser123 =
new GenericUserPreferenceArray(Arrays.asList(new GenericPreference(123L, 1L, 1.0f)));
CandidateItemsStrategy strategy = new PreferredItemsNeighborhoodCandidateItemsStrategy();
EasyMock.replay(dataModel);
FastIDSet candidateItems = strategy.getCandidateItems(123L, prefArrayOfUser123, dataModel);
assertEquals(1, candidateItems.size());
assertTrue(candidateItems.contains(2L));
EasyMock.verify(dataModel);
}
@Override
public void setPreference(long userID, long itemID, float value) throws TasteException {
DataModel dataModel = getDataModel();
double prefDelta;
try {
Float oldPref = dataModel.getPreferenceValue(userID, itemID);
prefDelta = oldPref == null ? value : value - oldPref;
} catch (NoSuchUserException nsee) {
prefDelta = value;
}
super.setPreference(userID, itemID, value);
try {
buildAveragesLock.writeLock().lock();
RunningAverage itemAverage = itemAverages.get(itemID);
if (itemAverage == null) {
RunningAverage newItemAverage = new FullRunningAverage();
newItemAverage.addDatum(prefDelta);
itemAverages.put(itemID, newItemAverage);
} else {
itemAverage.changeDatum(prefDelta);
}
RunningAverage userAverage = userAverages.get(userID);
if (userAverage == null) {
RunningAverage newUserAveragae = new FullRunningAverage();
newUserAveragae.addDatum(prefDelta);
userAverages.put(userID, newUserAveragae);
} else {
userAverage.changeDatum(prefDelta);
}
overallAveragePrefValue.changeDatum(prefDelta);
} finally {
buildAveragesLock.writeLock().unlock();
}
}
@Override
public float estimatePreference(long userID, long itemID) throws TasteException {
DataModel dataModel = getDataModel();
Float actualPref = dataModel.getPreferenceValue(userID, itemID);
if (actualPref != null) {
return actualPref;
}
return doEstimatePreference(userID, itemID);
}
/**
* Exports the simple user IDs and associated item IDs in the data model.
*
* @return a {@link FastByIDMap} mapping user IDs to {@link FastIDSet}s representing that user's
* associated items
*/
public static FastByIDMap<FastIDSet> toDataMap(DataModel dataModel) throws TasteException {
FastByIDMap<FastIDSet> data = new FastByIDMap<>(dataModel.getNumUsers());
LongPrimitiveIterator it = dataModel.getUserIDs();
while (it.hasNext()) {
long userID = it.nextLong();
data.put(userID, dataModel.getItemIDsFromUser(userID));
}
return data;
}
@Override
public double itemSimilarity(long itemID1, long itemID2) throws TasteException {
int preferring1and2 = dataModel.getNumUsersWithPreferenceFor(itemID1, itemID2);
if (preferring1and2 == 0) {
return Double.NaN;
}
int preferring1 = dataModel.getNumUsersWithPreferenceFor(itemID1);
int preferring2 = dataModel.getNumUsersWithPreferenceFor(itemID2);
int numUsers = dataModel.getNumUsers();
double logLikelihood =
twoLogLambda(
preferring1and2, preferring1 - preferring1and2, preferring2, numUsers - preferring2);
return 1.0 - 1.0 / (1.0 + logLikelihood);
}
@Override
public double userSimilarity(long userID1, long userID2) throws TasteException {
FastIDSet prefs1 = dataModel.getItemIDsFromUser(userID1);
FastIDSet prefs2 = dataModel.getItemIDsFromUser(userID2);
int prefs1Size = prefs1.size();
int prefs2Size = prefs2.size();
int intersectionSize =
prefs1Size < prefs2Size ? prefs2.intersectionSize(prefs1) : prefs1.intersectionSize(prefs2);
if (intersectionSize == 0) {
return Double.NaN;
}
// int numItems = dataModel.getNumItems();
// int numItems = Math.max(prefs1Size, prefs2Size);
// int numItems = prefs1Size;
// double distance1 = (double) (prefs1Size - intersectionSize)/(double) 2;
// double distance2 = (double) (prefs2Size - intersectionSize)/(double) 2;
// double distance = (distance1 + distance2);
double distance = (double) (prefs1Size - intersectionSize);
double similarity = 1.0 / (1.0 + distance);
////// System.out.println( prefs1Size + ", " + prefs2Size + ", " + intersectionSize + ", " +
// similarity);
return similarity;
}
public CachingUserNeighborhood(UserNeighborhood neighborhood, DataModel dataModel)
throws TasteException {
Preconditions.checkArgument(neighborhood != null, "neighborhood is null");
this.neighborhood = neighborhood;
int maxCacheSize = dataModel.getNumUsers(); // just a dumb heuristic for sizing
this.neighborhoodCache =
new Cache<Long, long[]>(new NeighborhoodRetriever(neighborhood), maxCacheSize);
}
public static void main(String[] args) throws IOException, TasteException {
String file = "datafile/item.csv";
DataModel model = new FileDataModel(new File(file));
UserSimilarity user = new EuclideanDistanceSimilarity(model);
NearestNUserNeighborhood neighbor = new NearestNUserNeighborhood(NEIGHBORHOOD_NUM, user, model);
Recommender r = new GenericUserBasedRecommender(model, neighbor, user);
LongPrimitiveIterator iter = model.getUserIDs();
while (iter.hasNext()) {
long uid = iter.nextLong();
List<RecommendedItem> list = r.recommend(uid, RECOMMENDER_NUM);
System.out.printf("uid:%s", uid);
for (RecommendedItem ritem : list) {
System.out.printf("(%s,%f)", ritem.getItemID(), ritem.getValue());
}
System.out.println();
}
}
@Override
public double userSimilarity(long userID1, long userID2) throws TasteException {
FastIDSet prefs1 = dataModel.getItemIDsFromUser(userID1);
FastIDSet prefs2 = dataModel.getItemIDsFromUser(userID2);
int prefs1Size = prefs1.size();
int prefs2Size = prefs2.size();
int intersectionSize =
prefs1Size < prefs2Size ? prefs2.intersectionSize(prefs1) : prefs1.intersectionSize(prefs2);
if (intersectionSize == 0) {
return Double.NaN;
}
int numItems = dataModel.getNumItems();
double logLikelihood =
twoLogLambda(
intersectionSize, prefs1Size - intersectionSize, prefs2Size, numItems - prefs2Size);
return 1.0 - 1.0 / (1.0 + logLikelihood);
}
private void buildAverageDiffs() throws TasteException {
try {
buildAveragesLock.writeLock().lock();
DataModel dataModel = getDataModel();
LongPrimitiveIterator it = dataModel.getUserIDs();
while (it.hasNext()) {
long userID = it.nextLong();
PreferenceArray prefs = dataModel.getPreferencesFromUser(userID);
int size = prefs.length();
for (int i = 0; i < size; i++) {
long itemID = prefs.getItemID(i);
float value = prefs.getValue(i);
addDatumAndCreateIfNeeded(itemID, value, itemAverages);
addDatumAndCreateIfNeeded(userID, value, userAverages);
overallAveragePrefValue.addDatum(value);
}
}
} finally {
buildAveragesLock.writeLock().unlock();
}
}
/** Creates a possibly weighted AbstractSimilarity. */
AbstractSimilarity(final DataModel dataModel, Weighting weighting, boolean centerData)
throws TasteException {
if (dataModel == null) {
throw new IllegalArgumentException("dataModel is null");
}
this.dataModel = dataModel;
this.weighted = weighting == Weighting.WEIGHTED;
this.centerData = centerData;
this.cachedNumItems = dataModel.getNumItems();
this.cachedNumUsers = dataModel.getNumUsers();
this.refreshHelper =
new RefreshHelper(
new Callable<Object>() {
@Override
public Object call() throws TasteException {
cachedNumItems = dataModel.getNumItems();
cachedNumUsers = dataModel.getNumUsers();
return null;
}
});
this.refreshHelper.addDependency(this.dataModel);
}
public static void main(String args[]) {
try {
// Loading the DATA;
DataModel dm =
new FileDataModel(
new File(
"C:\\Users\\bryce\\Course Work\\3. Full Summer\\Big Data\\Final Project\\Yelp\\FINAL CODE\\Mahout\\data\\busirec_new.csv"));
// We use the below line to relate businesses.
// ItemSimilarity sim = new LogLikelihoodSimilarity(dm);
TanimotoCoefficientSimilarity sim = new TanimotoCoefficientSimilarity((dm));
// Using the below line get recommendations
GenericItemBasedRecommender recommender = new GenericItemBasedRecommender(dm, sim);
// Looping through every business.
for (LongPrimitiveIterator items = dm.getItemIDs(); items.hasNext(); ) {
long itemId = items.nextLong();
// For each business we recommend 3 businesses.
List<RecommendedItem> recommendations = recommender.mostSimilarItems(itemId, 2);
for (RecommendedItem recommendation : recommendations) {
System.out.println(
itemId + "," + recommendation.getItemID() + "," + recommendation.getValue());
}
}
} catch (IOException | TasteException e) {
System.out.println(e);
}
}
@Override
public double userSimilarity(long userID1, long userID2) throws TasteException {
DataModel dataModel = getDataModel();
FastIDSet xPrefs = dataModel.getItemIDsFromUser(userID1);
FastIDSet yPrefs = dataModel.getItemIDsFromUser(userID2);
int xPrefsSize = xPrefs.size();
int yPrefsSize = yPrefs.size();
if (xPrefsSize == 0 && yPrefsSize == 0) {
return Double.NaN;
}
if (xPrefsSize == 0 || yPrefsSize == 0) {
return 0.0;
}
double intersection = 0.0;
double union = 0.0;
for (LongPrimitiveIterator it_item = xPrefs.iterator(); it_item.hasNext(); ) {
long itemID = (long) it_item.nextLong();
double weight = (double) getDataModel().getNumUsers() / mItemPrefNum.get(itemID);
if (yPrefs.contains(itemID)) {
intersection += weight;
union -= weight;
}
union += weight;
}
for (LongPrimitiveIterator it_item = yPrefs.iterator(); it_item.hasNext(); ) {
long itemID = (long) it_item.nextLong();
double weight = (double) getDataModel().getNumUsers() / mItemPrefNum.get(itemID);
union += weight;
}
return Math.log(intersection) / Math.log(union);
}
public static void main(String[] args)
throws FileNotFoundException, TasteException, IOException, OptionException {
DataModel model;
model = new FileDataModel(new File("datasets/ratingsForMahout.dat"));
File movieMapFile = new File("datasets/moviesForMahout.dat");
HashMap<Long, String> movieMap = new HashMap<Long, String>();
Scanner scan = new Scanner(movieMapFile);
while (scan.hasNextLine()) {
String[] line = scan.nextLine().split("\\|");
movieMap.put(Long.parseLong(line[0]), line[1]);
}
scan.close();
UserSimilarity userSimilarity = new PearsonCorrelationSimilarity(model);
UserNeighborhood neighborhood = new NearestNUserNeighborhood(3, userSimilarity, model);
Recommender recommender = new GenericUserBasedRecommender(model, neighborhood, userSimilarity);
Recommender cachingRecommender = new CachingRecommender(recommender);
for (LongPrimitiveIterator it = model.getUserIDs(); it.hasNext(); ) {
long userId = it.nextLong();
List<RecommendedItem> recommendations = cachingRecommender.recommend(userId, 10);
if (recommendations.size() == 0) {
System.out.println("User " + userId + ": no recommendations");
}
for (RecommendedItem recommendedItem : recommendations) {
System.out.println(
"User "
+ userId
+ ": "
+ movieMap.get(recommendedItem.getItemID())
+ "; value="
+ recommendedItem.getValue());
}
}
}
public void testIUF() throws Exception {
DataModel dataModel =
getDataModel(
new long[] {1, 2, 3, 4, 5},
new Double[][] {
{0.1}, {0.2, 0.3}, {0.4, 0.5, 0.6}, {0.7, 0.8, 0.9, 1.0}, {1.0, 1.0, 1.0, 1.0, 1.0},
});
InverseUserFrequency iuf = new InverseUserFrequency(dataModel, 10.0);
PreferenceArray user5Prefs = dataModel.getPreferencesFromUser(5);
for (int i = 0; i < 5; i++) {
Preference pref = user5Prefs.get(i);
assertNotNull(pref);
assertEquals(
Math.log(5.0 / (double) (5 - i)) / Math.log(iuf.getLogBase()),
iuf.getTransformedValue(pref),
EPSILON);
}
// Make sure this doesn't throw an exception
iuf.refresh(null);
}
@Override
public void removePreference(long userID, long itemID) throws TasteException {
DataModel dataModel = getDataModel();
Float oldPref = dataModel.getPreferenceValue(userID, itemID);
super.removePreference(userID, itemID);
if (oldPref != null) {
try {
buildAveragesLock.writeLock().lock();
RunningAverage itemAverage = itemAverages.get(itemID);
if (itemAverage == null) {
throw new IllegalStateException("No preferences exist for item ID: " + itemID);
}
itemAverage.removeDatum(oldPref);
RunningAverage userAverage = userAverages.get(userID);
if (userAverage == null) {
throw new IllegalStateException("No preferences exist for user ID: " + userID);
}
userAverage.removeDatum(oldPref);
overallAveragePrefValue.removeDatum(oldPref);
} finally {
buildAveragesLock.writeLock().unlock();
}
}
}
/** 对用户性别进行过滤 */
public static void filterGender(
long uid, RecommenderBuilder recommenderBuilder, DataModel dataModel, String gender)
throws TasteException, IOException {
Set<Long> userids = getByGender("datafile/book/user.csv", gender);
// 计算指定性别用户打分过的图书
Set<Long> bookids = new HashSet<Long>();
for (long uids : userids) {
LongPrimitiveIterator iter = dataModel.getItemIDsFromUser(uids).iterator();
while (iter.hasNext()) {
long bookid = iter.next();
bookids.add(bookid);
}
}
IDRescorer rescorer = new FilterRescorer(bookids);
List<RecommendedItem> list =
recommenderBuilder.buildRecommender(dataModel).recommend(uid, RECOMMENDER_NUM, rescorer);
RecommendFactory.showItems(uid, list, false);
}
private void splitOneUsersPrefs(
double trainingPercentage,
FastByIDMap<PreferenceArray> trainingPrefs,
FastByIDMap<PreferenceArray> testPrefs,
long userID,
DataModel dataModel)
throws TasteException {
List<Preference> oneUserTrainingPrefs = null;
List<Preference> oneUserTestPrefs = null;
PreferenceArray prefs = dataModel.getPreferencesFromUser(userID);
int size = prefs.length();
boolean isInstanceOfContextualUserPreferenceArray =
prefs instanceof ContextualUserPreferenceArray;
for (int i = 0; i < size; i++) {
Preference newPref =
isInstanceOfContextualUserPreferenceArray
? new ContextualPreference(
userID,
prefs.getItemID(i),
prefs.getValue(i),
((ContextualUserPreferenceArray) prefs).getContextualPreferences(i))
: new GenericPreference(userID, prefs.getItemID(i), prefs.getValue(i));
if (this.idrescorer != null && this.idrescorer.isFiltered(newPref.getItemID())) { // adiciona
// ratings
// de
// source
// domain
// sempre
// em
// training
// set
if (oneUserTrainingPrefs == null) {
oneUserTrainingPrefs = Lists.newArrayListWithCapacity(3);
}
oneUserTrainingPrefs.add(newPref);
totalOfTrainingRatingsFromSource++;
continue;
}
if (this.contextualCriteria != null
&& isInstanceOfContextualUserPreferenceArray) { // adiciona
// ratings
// de outro
// contexto
// sempre em
// training
// set
ContextualPreference contextualPref = (ContextualPreference) newPref;
if (!this.contextualCriteria.containsAllContextualAttributes(
contextualPref.getContextualPreferences())) {
if (oneUserTrainingPrefs == null) {
oneUserTrainingPrefs = Lists.newArrayListWithCapacity(3);
}
oneUserTrainingPrefs.add(newPref);
totalOfTrainingRatingsFromTargetWithoutContext++;
continue;
}
}
// para ratings do target e do contexto, fazer proporcao definida
if (random.nextDouble() < trainingPercentage) {
if (oneUserTrainingPrefs == null) {
oneUserTrainingPrefs = Lists.newArrayListWithCapacity(3);
}
oneUserTrainingPrefs.add(newPref);
totalOfTrainingRatingsFromTargetWithContext++;
} else {
if (oneUserTestPrefs == null) {
oneUserTestPrefs = Lists.newArrayListWithCapacity(3);
}
oneUserTestPrefs.add(newPref);
totalOfTestRatings++;
}
// OLD training/test set
/*
* if (random.nextDouble() < trainingPercentage) { if
* (oneUserTrainingPrefs == null) { oneUserTrainingPrefs =
* Lists.newArrayListWithCapacity(3); }
* oneUserTrainingPrefs.add(newPref); totalOfTrainingRatings++; }
* else { if (oneUserTestPrefs == null) { oneUserTestPrefs =
* Lists.newArrayListWithCapacity(3); } //testa somente com um tipo
* de rating (rescorer e/ou context) if(this.idrescorer == null &&
* this.contextualCriteria == null){ oneUserTestPrefs.add(newPref);
* totalOfTestRatings++; }else{ if(this.idrescorer != null &&
* !this.idrescorer.isFiltered(newPref.getItemID())){
* if(this.contextualCriteria != null &&
* isInstanceOfContextualUserPreferenceArray){ ContextualPreference
* contextualPref = (ContextualPreference) newPref;
* if(this.contextualCriteria
* .containsAllContextualAttributes(contextualPref
* .getContextualPreferences())){ oneUserTestPrefs.add(newPref);
* totalOfTestRatings++; } }else{ oneUserTestPrefs.add(newPref);
* totalOfTestRatings++; } }else if(this.idrescorer == null &&
* this.contextualCriteria != null &&
* isInstanceOfContextualUserPreferenceArray){ ContextualPreference
* contextualPref = (ContextualPreference) newPref;
* if(this.contextualCriteria
* .containsAllContextualAttributes(contextualPref
* .getContextualPreferences())){ oneUserTestPrefs.add(newPref);
* totalOfTestRatings++; } } } }
*/
}
if (oneUserTrainingPrefs != null) {
trainingPrefs.put(
userID,
isInstanceOfContextualUserPreferenceArray
? new ContextualUserPreferenceArray(oneUserTrainingPrefs)
: new GenericUserPreferenceArray(oneUserTrainingPrefs));
if (oneUserTestPrefs != null) {
testPrefs.put(
userID,
isInstanceOfContextualUserPreferenceArray
? new ContextualUserPreferenceArray(oneUserTestPrefs)
: new GenericUserPreferenceArray(oneUserTestPrefs));
}
}
}
@Override
public double userSimilarity(long userID1, long userID2) throws TasteException {
PreferenceArray xPrefs = dataModel.getPreferencesFromUser(userID1);
PreferenceArray yPrefs = dataModel.getPreferencesFromUser(userID2);
int xLength = xPrefs.length();
int yLength = yPrefs.length();
if ((xLength == 0) || (yLength == 0)) {
return Double.NaN;
}
long xIndex = xPrefs.getItemID(0);
long yIndex = yPrefs.getItemID(0);
int xPrefIndex = 0;
int yPrefIndex = 0;
double sumX = 0.0;
double sumX2 = 0.0;
double sumY = 0.0;
double sumY2 = 0.0;
double sumXY = 0.0;
double sumXYdiff2 = 0.0;
int count = 0;
boolean hasInferrer = inferrer != null;
boolean hasPrefTransform = prefTransform != null;
while (true) {
int compare = xIndex < yIndex ? -1 : xIndex > yIndex ? 1 : 0;
if (hasInferrer || compare == 0) {
double x;
double y;
if (xIndex == yIndex) {
// Both users expressed a preference for the item
if (hasPrefTransform) {
x = prefTransform.getTransformedValue(xPrefs.get(xPrefIndex));
y = prefTransform.getTransformedValue(yPrefs.get(yPrefIndex));
} else {
x = xPrefs.getValue(xPrefIndex);
y = yPrefs.getValue(yPrefIndex);
}
} else {
// Only one user expressed a preference, but infer the other one's preference and tally
// as if the other user expressed that preference
if (compare < 0) {
// X has a value; infer Y's
x =
hasPrefTransform
? prefTransform.getTransformedValue(xPrefs.get(xPrefIndex))
: xPrefs.getValue(xPrefIndex);
y = inferrer.inferPreference(userID2, xIndex);
} else {
// compare > 0
// Y has a value; infer X's
x = inferrer.inferPreference(userID1, yIndex);
y =
hasPrefTransform
? prefTransform.getTransformedValue(yPrefs.get(yPrefIndex))
: yPrefs.getValue(yPrefIndex);
}
}
sumXY += x * y;
sumX += x;
sumX2 += x * x;
sumY += y;
sumY2 += y * y;
double diff = x - y;
sumXYdiff2 += diff * diff;
count++;
}
if (compare <= 0) {
if (++xPrefIndex >= xLength) {
if (hasInferrer) {
// Must count other Ys; pretend next X is far away
if (yIndex == Long.MAX_VALUE) {
// ... but stop if both are done!
break;
}
xIndex = Long.MAX_VALUE;
} else {
break;
}
} else {
xIndex = xPrefs.getItemID(xPrefIndex);
}
}
if (compare >= 0) {
if (++yPrefIndex >= yLength) {
if (hasInferrer) {
// Must count other Xs; pretend next Y is far away
if (xIndex == Long.MAX_VALUE) {
// ... but stop if both are done!
break;
}
yIndex = Long.MAX_VALUE;
} else {
break;
}
} else {
yIndex = yPrefs.getItemID(yPrefIndex);
}
}
}
// "Center" the data. If my math is correct, this'll do it.
double result;
if (centerData) {
double n = count;
double meanX = sumX / n;
double meanY = sumY / n;
// double centeredSumXY = sumXY - meanY * sumX - meanX * sumY + n * meanX * meanY;
double centeredSumXY = sumXY - meanY * sumX;
// double centeredSumX2 = sumX2 - 2.0 * meanX * sumX + n * meanX * meanX;
double centeredSumX2 = sumX2 - meanX * sumX;
// double centeredSumY2 = sumY2 - 2.0 * meanY * sumY + n * meanY * meanY;
double centeredSumY2 = sumY2 - meanY * sumY;
result = computeResult(count, centeredSumXY, centeredSumX2, centeredSumY2, sumXYdiff2);
} else {
result = computeResult(count, sumXY, sumX2, sumY2, sumXYdiff2);
}
if (similarityTransform != null) {
result = similarityTransform.transformSimilarity(userID1, userID2, result);
}
if (!Double.isNaN(result)) {
result = normalizeWeightResult(result, count, cachedNumItems);
}
return result;
}
@Override
public final double itemSimilarity(long itemID1, long itemID2) throws TasteException {
PreferenceArray xPrefs = dataModel.getPreferencesForItem(itemID1);
PreferenceArray yPrefs = dataModel.getPreferencesForItem(itemID2);
int xLength = xPrefs.length();
int yLength = yPrefs.length();
if ((xLength == 0) || (yLength == 0)) {
return Double.NaN;
}
long xIndex = xPrefs.getUserID(0);
long yIndex = yPrefs.getUserID(0);
int xPrefIndex = 0;
int yPrefIndex = 0;
double sumX = 0.0;
double sumX2 = 0.0;
double sumY = 0.0;
double sumY2 = 0.0;
double sumXY = 0.0;
double sumXYdiff2 = 0.0;
int count = 0;
// No, pref inferrers and transforms don't appy here. I think.
while (true) {
int compare = xIndex < yIndex ? -1 : xIndex > yIndex ? 1 : 0;
if (compare == 0) {
// Both users expressed a preference for the item
double x = xPrefs.getValue(xPrefIndex);
double y = yPrefs.getValue(yPrefIndex);
sumXY += x * y;
sumX += x;
sumX2 += x * x;
sumY += y;
sumY2 += y * y;
double diff = x - y;
sumXYdiff2 += diff * diff;
count++;
}
if (compare <= 0) {
if (++xPrefIndex == xLength) {
break;
}
xIndex = xPrefs.getUserID(xPrefIndex);
}
if (compare >= 0) {
if (++yPrefIndex == yLength) {
break;
}
yIndex = yPrefs.getUserID(yPrefIndex);
}
}
double result;
if (centerData) {
// See comments above on these computations
double n = (double) count;
double meanX = sumX / n;
double meanY = sumY / n;
// double centeredSumXY = sumXY - meanY * sumX - meanX * sumY + n * meanX * meanY;
double centeredSumXY = sumXY - meanY * sumX;
// double centeredSumX2 = sumX2 - 2.0 * meanX * sumX + n * meanX * meanX;
double centeredSumX2 = sumX2 - meanX * sumX;
// double centeredSumY2 = sumY2 - 2.0 * meanY * sumY + n * meanY * meanY;
double centeredSumY2 = sumY2 - meanY * sumY;
result = computeResult(count, centeredSumXY, centeredSumX2, centeredSumY2, sumXYdiff2);
} else {
result = computeResult(count, sumXY, sumX2, sumY2, sumXYdiff2);
}
if (similarityTransform != null) {
result = similarityTransform.transformSimilarity(itemID1, itemID2, result);
}
if (!Double.isNaN(result)) {
result = normalizeWeightResult(result, count, cachedNumUsers);
}
return result;
}
/** @throws IllegalArgumentException if {@link DataModel} does not have preference values */
public EuclideanDistanceSimilarity(DataModel dataModel, Weighting weighting)
throws TasteException {
super(dataModel, weighting, false);
Preconditions.checkArgument(
dataModel.hasPreferenceValues(), "DataModel doesn't have preference values");
}
@Override
public double evaluate(
RecommenderBuilder recommenderBuilder,
DataModelBuilder dataModelBuilder,
DataModel dataModel,
double trainingPercentage,
double evaluationPercentage)
throws TasteException {
Preconditions.checkNotNull(recommenderBuilder);
Preconditions.checkNotNull(dataModel);
Preconditions.checkArgument(
trainingPercentage >= 0.0 && trainingPercentage <= 1.0,
"Invalid trainingPercentage: "
+ trainingPercentage
+ ". Must be: 0.0 <= trainingPercentage <= 1.0");
Preconditions.checkArgument(
evaluationPercentage >= 0.0 && evaluationPercentage <= 1.0,
"Invalid evaluationPercentage: "
+ evaluationPercentage
+ ". Must be: 0.0 <= evaluationPercentage <= 1.0");
log.info("Beginning evaluation using {} of {}", trainingPercentage, dataModel);
int numUsers = dataModel.getNumUsers();
FastByIDMap<PreferenceArray> trainingPrefs =
new FastByIDMap<PreferenceArray>(1 + (int) (evaluationPercentage * numUsers));
FastByIDMap<PreferenceArray> testPrefs =
new FastByIDMap<PreferenceArray>(1 + (int) (evaluationPercentage * numUsers));
totalOfTrainingRatingsFromSource = 0;
totalOfTrainingRatingsFromTargetWithContext = 0;
totalOfTrainingRatingsFromTargetWithoutContext = 0;
totalOfTestRatings = 0;
LongPrimitiveIterator it = dataModel.getUserIDs();
while (it.hasNext()) {
long userID = it.nextLong();
if (random.nextDouble() < evaluationPercentage) {
splitOneUsersPrefs(trainingPercentage, trainingPrefs, testPrefs, userID, dataModel);
}
}
// System.out.println("Training (Source, TargetWithoutContext, TargetWithContext):
// "+totalOfTrainingRatingsFromSource+"/"+totalOfTrainingRatingsFromTargetWithoutContext+"/"+totalOfTrainingRatingsFromTargetWithContext);
// int totalTraining =
// (totalOfTrainingRatingsFromSource+totalOfTrainingRatingsFromTargetWithContext+totalOfTrainingRatingsFromTargetWithoutContext);
// System.out.println("Training/Test: "+totalTraining+"/"+totalOfTestRatings);
DataModel newDataModel =
dataModel instanceof ContextualDataModel
? new ContextualDataModel(trainingPrefs)
: new GenericDataModel(trainingPrefs);
DataModel trainingModel =
dataModelBuilder == null ? newDataModel : dataModelBuilder.buildDataModel(trainingPrefs);
Recommender recommender = recommenderBuilder.buildRecommender(trainingModel);
double result = getEvaluation(testPrefs, recommender);
log.info("Evaluation result: {}", result);
return result;
}
