public static FastByIDMap<FastIDSet> toDataMap(FastByIDMap<PreferenceArray> data) {
for (Map.Entry<Long, Object> entry : ((FastByIDMap<Object>) (FastByIDMap<?>) data).entrySet()) {
PreferenceArray prefArray = (PreferenceArray) entry.getValue();
int size = prefArray.length();
FastIDSet itemIDs = new FastIDSet(size);
for (int i = 0; i < size; i++) {
itemIDs.add(prefArray.getItemID(i));
}
entry.setValue(itemIDs);
}
return (FastByIDMap<FastIDSet>) (FastByIDMap<?>) data;
}
@Override
public PreferenceArray getPreferencesForItem(long itemID) throws NoSuchItemException {
FastIDSet userIDs = preferenceForItems.get(itemID);
if (userIDs == null) {
throw new NoSuchItemException(itemID);
}
PreferenceArray prefArray = new BooleanItemPreferenceArray(userIDs.size());
int i = 0;
LongPrimitiveIterator it = userIDs.iterator();
while (it.hasNext()) {
prefArray.setUserID(i, it.nextLong());
prefArray.setItemID(i, itemID);
i++;
}
return prefArray;
}
private void buildAverageDiffs() throws TasteException {
try {
buildAveragesLock.writeLock().lock();
DataModel dataModel = getDataModel();
LongPrimitiveIterator it = dataModel.getUserIDs();
while (it.hasNext()) {
long userID = it.nextLong();
PreferenceArray prefs = dataModel.getPreferencesFromUser(userID);
int size = prefs.length();
for (int i = 0; i < size; i++) {
long itemID = prefs.getItemID(i);
float value = prefs.getValue(i);
addDatumAndCreateIfNeeded(itemID, value, itemAverages);
addDatumAndCreateIfNeeded(userID, value, userAverages);
overallAveragePrefValue.addDatum(value);
}
}
} finally {
buildAveragesLock.writeLock().unlock();
}
}
public void testIUF() throws Exception {
DataModel dataModel =
getDataModel(
new long[] {1, 2, 3, 4, 5},
new Double[][] {
{0.1}, {0.2, 0.3}, {0.4, 0.5, 0.6}, {0.7, 0.8, 0.9, 1.0}, {1.0, 1.0, 1.0, 1.0, 1.0},
});
InverseUserFrequency iuf = new InverseUserFrequency(dataModel, 10.0);
PreferenceArray user5Prefs = dataModel.getPreferencesFromUser(5);
for (int i = 0; i < 5; i++) {
Preference pref = user5Prefs.get(i);
assertNotNull(pref);
assertEquals(
Math.log(5.0 / (double) (5 - i)) / Math.log(iuf.getLogBase()),
iuf.getTransformedValue(pref),
EPSILON);
}
// Make sure this doesn't throw an exception
iuf.refresh(null);
}
@Override
public final double itemSimilarity(long itemID1, long itemID2) throws TasteException {
PreferenceArray xPrefs = dataModel.getPreferencesForItem(itemID1);
PreferenceArray yPrefs = dataModel.getPreferencesForItem(itemID2);
int xLength = xPrefs.length();
int yLength = yPrefs.length();
if ((xLength == 0) || (yLength == 0)) {
return Double.NaN;
}
long xIndex = xPrefs.getUserID(0);
long yIndex = yPrefs.getUserID(0);
int xPrefIndex = 0;
int yPrefIndex = 0;
double sumX = 0.0;
double sumX2 = 0.0;
double sumY = 0.0;
double sumY2 = 0.0;
double sumXY = 0.0;
double sumXYdiff2 = 0.0;
int count = 0;
// No, pref inferrers and transforms don't appy here. I think.
while (true) {
int compare = xIndex < yIndex ? -1 : xIndex > yIndex ? 1 : 0;
if (compare == 0) {
// Both users expressed a preference for the item
double x = xPrefs.getValue(xPrefIndex);
double y = yPrefs.getValue(yPrefIndex);
sumXY += x * y;
sumX += x;
sumX2 += x * x;
sumY += y;
sumY2 += y * y;
double diff = x - y;
sumXYdiff2 += diff * diff;
count++;
}
if (compare <= 0) {
if (++xPrefIndex == xLength) {
break;
}
xIndex = xPrefs.getUserID(xPrefIndex);
}
if (compare >= 0) {
if (++yPrefIndex == yLength) {
break;
}
yIndex = yPrefs.getUserID(yPrefIndex);
}
}
double result;
if (centerData) {
// See comments above on these computations
double n = (double) count;
double meanX = sumX / n;
double meanY = sumY / n;
// double centeredSumXY = sumXY - meanY * sumX - meanX * sumY + n * meanX * meanY;
double centeredSumXY = sumXY - meanY * sumX;
// double centeredSumX2 = sumX2 - 2.0 * meanX * sumX + n * meanX * meanX;
double centeredSumX2 = sumX2 - meanX * sumX;
// double centeredSumY2 = sumY2 - 2.0 * meanY * sumY + n * meanY * meanY;
double centeredSumY2 = sumY2 - meanY * sumY;
result = computeResult(count, centeredSumXY, centeredSumX2, centeredSumY2, sumXYdiff2);
} else {
result = computeResult(count, sumXY, sumX2, sumY2, sumXYdiff2);
}
if (similarityTransform != null) {
result = similarityTransform.transformSimilarity(itemID1, itemID2, result);
}
if (!Double.isNaN(result)) {
result = normalizeWeightResult(result, count, cachedNumUsers);
}
return result;
}
@Override
public double userSimilarity(long userID1, long userID2) throws TasteException {
PreferenceArray xPrefs = dataModel.getPreferencesFromUser(userID1);
PreferenceArray yPrefs = dataModel.getPreferencesFromUser(userID2);
int xLength = xPrefs.length();
int yLength = yPrefs.length();
if ((xLength == 0) || (yLength == 0)) {
return Double.NaN;
}
long xIndex = xPrefs.getItemID(0);
long yIndex = yPrefs.getItemID(0);
int xPrefIndex = 0;
int yPrefIndex = 0;
double sumX = 0.0;
double sumX2 = 0.0;
double sumY = 0.0;
double sumY2 = 0.0;
double sumXY = 0.0;
double sumXYdiff2 = 0.0;
int count = 0;
boolean hasInferrer = inferrer != null;
boolean hasPrefTransform = prefTransform != null;
while (true) {
int compare = xIndex < yIndex ? -1 : xIndex > yIndex ? 1 : 0;
if (hasInferrer || compare == 0) {
double x;
double y;
if (xIndex == yIndex) {
// Both users expressed a preference for the item
if (hasPrefTransform) {
x = prefTransform.getTransformedValue(xPrefs.get(xPrefIndex));
y = prefTransform.getTransformedValue(yPrefs.get(yPrefIndex));
} else {
x = xPrefs.getValue(xPrefIndex);
y = yPrefs.getValue(yPrefIndex);
}
} else {
// Only one user expressed a preference, but infer the other one's preference and tally
// as if the other user expressed that preference
if (compare < 0) {
// X has a value; infer Y's
x =
hasPrefTransform
? prefTransform.getTransformedValue(xPrefs.get(xPrefIndex))
: xPrefs.getValue(xPrefIndex);
y = inferrer.inferPreference(userID2, xIndex);
} else {
// compare > 0
// Y has a value; infer X's
x = inferrer.inferPreference(userID1, yIndex);
y =
hasPrefTransform
? prefTransform.getTransformedValue(yPrefs.get(yPrefIndex))
: yPrefs.getValue(yPrefIndex);
}
}
sumXY += x * y;
sumX += x;
sumX2 += x * x;
sumY += y;
sumY2 += y * y;
double diff = x - y;
sumXYdiff2 += diff * diff;
count++;
}
if (compare <= 0) {
if (++xPrefIndex >= xLength) {
if (hasInferrer) {
// Must count other Ys; pretend next X is far away
if (yIndex == Long.MAX_VALUE) {
// ... but stop if both are done!
break;
}
xIndex = Long.MAX_VALUE;
} else {
break;
}
} else {
xIndex = xPrefs.getItemID(xPrefIndex);
}
}
if (compare >= 0) {
if (++yPrefIndex >= yLength) {
if (hasInferrer) {
// Must count other Xs; pretend next Y is far away
if (xIndex == Long.MAX_VALUE) {
// ... but stop if both are done!
break;
}
yIndex = Long.MAX_VALUE;
} else {
break;
}
} else {
yIndex = yPrefs.getItemID(yPrefIndex);
}
}
}
// "Center" the data. If my math is correct, this'll do it.
double result;
if (centerData) {
double n = count;
double meanX = sumX / n;
double meanY = sumY / n;
// double centeredSumXY = sumXY - meanY * sumX - meanX * sumY + n * meanX * meanY;
double centeredSumXY = sumXY - meanY * sumX;
// double centeredSumX2 = sumX2 - 2.0 * meanX * sumX + n * meanX * meanX;
double centeredSumX2 = sumX2 - meanX * sumX;
// double centeredSumY2 = sumY2 - 2.0 * meanY * sumY + n * meanY * meanY;
double centeredSumY2 = sumY2 - meanY * sumY;
result = computeResult(count, centeredSumXY, centeredSumX2, centeredSumY2, sumXYdiff2);
} else {
result = computeResult(count, sumXY, sumX2, sumY2, sumXYdiff2);
}
if (similarityTransform != null) {
result = similarityTransform.transformSimilarity(userID1, userID2, result);
}
if (!Double.isNaN(result)) {
result = normalizeWeightResult(result, count, cachedNumItems);
}
return result;
}
private void splitOneUsersPrefs(
double trainingPercentage,
FastByIDMap<PreferenceArray> trainingPrefs,
FastByIDMap<PreferenceArray> testPrefs,
long userID,
DataModel dataModel)
throws TasteException {
List<Preference> oneUserTrainingPrefs = null;
List<Preference> oneUserTestPrefs = null;
PreferenceArray prefs = dataModel.getPreferencesFromUser(userID);
int size = prefs.length();
boolean isInstanceOfContextualUserPreferenceArray =
prefs instanceof ContextualUserPreferenceArray;
for (int i = 0; i < size; i++) {
Preference newPref =
isInstanceOfContextualUserPreferenceArray
? new ContextualPreference(
userID,
prefs.getItemID(i),
prefs.getValue(i),
((ContextualUserPreferenceArray) prefs).getContextualPreferences(i))
: new GenericPreference(userID, prefs.getItemID(i), prefs.getValue(i));
if (this.idrescorer != null && this.idrescorer.isFiltered(newPref.getItemID())) { // adiciona
// ratings
// de
// source
// domain
// sempre
// em
// training
// set
if (oneUserTrainingPrefs == null) {
oneUserTrainingPrefs = Lists.newArrayListWithCapacity(3);
}
oneUserTrainingPrefs.add(newPref);
totalOfTrainingRatingsFromSource++;
continue;
}
if (this.contextualCriteria != null
&& isInstanceOfContextualUserPreferenceArray) { // adiciona
// ratings
// de outro
// contexto
// sempre em
// training
// set
ContextualPreference contextualPref = (ContextualPreference) newPref;
if (!this.contextualCriteria.containsAllContextualAttributes(
contextualPref.getContextualPreferences())) {
if (oneUserTrainingPrefs == null) {
oneUserTrainingPrefs = Lists.newArrayListWithCapacity(3);
}
oneUserTrainingPrefs.add(newPref);
totalOfTrainingRatingsFromTargetWithoutContext++;
continue;
}
}
// para ratings do target e do contexto, fazer proporcao definida
if (random.nextDouble() < trainingPercentage) {
if (oneUserTrainingPrefs == null) {
oneUserTrainingPrefs = Lists.newArrayListWithCapacity(3);
}
oneUserTrainingPrefs.add(newPref);
totalOfTrainingRatingsFromTargetWithContext++;
} else {
if (oneUserTestPrefs == null) {
oneUserTestPrefs = Lists.newArrayListWithCapacity(3);
}
oneUserTestPrefs.add(newPref);
totalOfTestRatings++;
}
// OLD training/test set
/*
* if (random.nextDouble() < trainingPercentage) { if
* (oneUserTrainingPrefs == null) { oneUserTrainingPrefs =
* Lists.newArrayListWithCapacity(3); }
* oneUserTrainingPrefs.add(newPref); totalOfTrainingRatings++; }
* else { if (oneUserTestPrefs == null) { oneUserTestPrefs =
* Lists.newArrayListWithCapacity(3); } //testa somente com um tipo
* de rating (rescorer e/ou context) if(this.idrescorer == null &&
* this.contextualCriteria == null){ oneUserTestPrefs.add(newPref);
* totalOfTestRatings++; }else{ if(this.idrescorer != null &&
* !this.idrescorer.isFiltered(newPref.getItemID())){
* if(this.contextualCriteria != null &&
* isInstanceOfContextualUserPreferenceArray){ ContextualPreference
* contextualPref = (ContextualPreference) newPref;
* if(this.contextualCriteria
* .containsAllContextualAttributes(contextualPref
* .getContextualPreferences())){ oneUserTestPrefs.add(newPref);
* totalOfTestRatings++; } }else{ oneUserTestPrefs.add(newPref);
* totalOfTestRatings++; } }else if(this.idrescorer == null &&
* this.contextualCriteria != null &&
* isInstanceOfContextualUserPreferenceArray){ ContextualPreference
* contextualPref = (ContextualPreference) newPref;
* if(this.contextualCriteria
* .containsAllContextualAttributes(contextualPref
* .getContextualPreferences())){ oneUserTestPrefs.add(newPref);
* totalOfTestRatings++; } } } }
*/
}
if (oneUserTrainingPrefs != null) {
trainingPrefs.put(
userID,
isInstanceOfContextualUserPreferenceArray
? new ContextualUserPreferenceArray(oneUserTrainingPrefs)
: new GenericUserPreferenceArray(oneUserTrainingPrefs));
if (oneUserTestPrefs != null) {
testPrefs.put(
userID,
isInstanceOfContextualUserPreferenceArray
? new ContextualUserPreferenceArray(oneUserTestPrefs)
: new GenericUserPreferenceArray(oneUserTestPrefs));
}
}
}