private double doItemSimilarity(long itemID1, long itemID2, FastIDSet preferring1)
throws TasteException {
double intersection = 0.0;
double union = 0.0;
for (Preference pref : getDataModel().getPreferencesForItem(itemID2)) {
long userID = pref.getUserID();
double weight = (double) getDataModel().getNumItems() / mUserPrefNum.get(userID);
if (preferring1.contains(userID)) {
intersection += weight;
union -= weight;
}
union += weight;
}
for (LongPrimitiveIterator it_user = preferring1.iterator(); it_user.hasNext(); ) {
long userID = (long) it_user.nextLong();
double weight = (double) getDataModel().getNumItems() / mUserPrefNum.get(userID);
union += weight;
}
if (intersection == 0) {
return Double.NaN;
}
return Math.log(intersection) / Math.log(union);
}
/**
* tests {@link UserVectorSplitterMapper} in the special case that some userIDs shall be excluded
*/
@Test
public void testUserVectorSplitterMapperUserExclusion() throws Exception {
Mapper<VarLongWritable, VectorWritable, VarIntWritable, VectorOrPrefWritable>.Context context =
EasyMock.createMock(Mapper.Context.class);
context.write(
EasyMock.eq(new VarIntWritable(34)), prefOfVectorOrPrefWritableMatches(123L, 0.5f));
context.write(
EasyMock.eq(new VarIntWritable(56)), prefOfVectorOrPrefWritableMatches(123L, 0.7f));
EasyMock.replay(context);
FastIDSet usersToRecommendFor = new FastIDSet();
usersToRecommendFor.add(123L);
UserVectorSplitterMapper mapper = new UserVectorSplitterMapper();
setField(mapper, "maxPrefsPerUserConsidered", 10);
setField(mapper, "usersToRecommendFor", usersToRecommendFor);
RandomAccessSparseVector vector = new RandomAccessSparseVector(Integer.MAX_VALUE, 100);
vector.set(34, 0.5);
vector.set(56, 0.7);
mapper.map(new VarLongWritable(123L), new VectorWritable(vector), context);
mapper.map(new VarLongWritable(456L), new VectorWritable(vector), context);
EasyMock.verify(context);
}
private FastIDSet toUserFastIDSet(PreferenceArray array) {
FastIDSet fastIDSet = new FastIDSet();
for (Preference preference : array) {
fastIDSet.add(preference.getUserID());
}
return fastIDSet;
}
@Override
public Float getPreferenceValue(long userID, long itemID) throws NoSuchUserException {
FastIDSet itemIDs = preferenceFromUsers.get(userID);
if (itemIDs == null) {
throw new NoSuchUserException(userID);
}
if (itemIDs.contains(itemID)) {
return 1.0f;
}
return null;
}
public static FastByIDMap<FastIDSet> toDataMap(FastByIDMap<PreferenceArray> data) {
for (Map.Entry<Long, Object> entry : ((FastByIDMap<Object>) (FastByIDMap<?>) data).entrySet()) {
PreferenceArray prefArray = (PreferenceArray) entry.getValue();
int size = prefArray.length();
FastIDSet itemIDs = new FastIDSet(size);
for (int i = 0; i < size; i++) {
itemIDs.add(prefArray.getItemID(i));
}
entry.setValue(itemIDs);
}
return (FastByIDMap<FastIDSet>) (FastByIDMap<?>) data;
}
@Override
public PreferenceArray getPreferencesForItem(long itemID) throws NoSuchItemException {
FastIDSet userIDs = preferenceForItems.get(itemID);
if (userIDs == null) {
throw new NoSuchItemException(itemID);
}
PreferenceArray prefArray = new BooleanItemPreferenceArray(userIDs.size());
int i = 0;
LongPrimitiveIterator it = userIDs.iterator();
while (it.hasNext()) {
prefArray.setUserID(i, it.nextLong());
prefArray.setItemID(i, itemID);
i++;
}
return prefArray;
}
@Test
public void testStrategy() throws TasteException {
FastIDSet itemIDsFromUser123 = new FastIDSet();
itemIDsFromUser123.add(1L);
FastIDSet itemIDsFromUser456 = new FastIDSet();
itemIDsFromUser456.add(1L);
itemIDsFromUser456.add(2L);
List<Preference> prefs = new ArrayList<Preference>();
prefs.add(new GenericPreference(123L, 1L, 1.0f));
prefs.add(new GenericPreference(456L, 1L, 1.0f));
PreferenceArray preferencesForItem1 = new GenericItemPreferenceArray(prefs);
DataModel dataModel = EasyMock.createMock(DataModel.class);
EasyMock.expect(dataModel.getPreferencesForItem(1L)).andReturn(preferencesForItem1);
EasyMock.expect(dataModel.getItemIDsFromUser(123L)).andReturn(itemIDsFromUser123);
EasyMock.expect(dataModel.getItemIDsFromUser(456L)).andReturn(itemIDsFromUser456);
PreferenceArray prefArrayOfUser123 =
new GenericUserPreferenceArray(Arrays.asList(new GenericPreference(123L, 1L, 1.0f)));
CandidateItemsStrategy strategy = new PreferredItemsNeighborhoodCandidateItemsStrategy();
EasyMock.replay(dataModel);
FastIDSet candidateItems = strategy.getCandidateItems(123L, prefArrayOfUser123, dataModel);
assertEquals(1, candidateItems.size());
assertTrue(candidateItems.contains(2L));
EasyMock.verify(dataModel);
}
@Override
public List<RecommendedItem> recommend(
long userID, int howMany, IDRescorer rescorer, boolean includeKnownItems)
throws TasteException {
Preconditions.checkArgument(howMany >= 1, "howMany must be at least 1");
log.debug("Recommending items for user ID '{}'", userID);
PreferenceArray preferencesFromUser = getDataModel().getPreferencesFromUser(userID);
FastIDSet possibleItemIDs = getAllOtherItems(userID, preferencesFromUser, includeKnownItems);
TopItems.Estimator<Long> estimator = new Estimator(userID);
List<RecommendedItem> topItems =
TopItems.getTopItems(howMany, possibleItemIDs.iterator(), rescorer, estimator);
log.debug("Recommendations are: {}", topItems);
return topItems;
}
@Override
protected void reduce(Vertex vertex, Iterable<Vertex> vertices, Context ctx)
throws IOException, InterruptedException {
int howmany = 0;
FastIDSet bufferedVertexIDs = new FastIDSet();
for (Vertex firstVertexOfMissingEdge : vertices) {
LongPrimitiveIterator bufferedVertexIdsIterator = bufferedVertexIDs.iterator();
while (bufferedVertexIdsIterator.hasNext()) {
Vertex secondVertexOfMissingEdge = new Vertex(bufferedVertexIdsIterator.nextLong());
UndirectedEdge missingEdge =
new UndirectedEdge(firstVertexOfMissingEdge, secondVertexOfMissingEdge);
JoinableUndirectedEdge key = new JoinableUndirectedEdge(missingEdge, false);
VertexOrMarker value = new VertexOrMarker(vertex);
log.trace("{} -> {}", key, value);
howmany++;
ctx.write(key, value);
}
bufferedVertexIDs.add(firstVertexOfMissingEdge.getId());
}
log.debug("{} triads on vertex {}", howmany, vertex);
}
@Override
public double userSimilarity(long userID1, long userID2) throws TasteException {
FastIDSet prefs1 = dataModel.getItemIDsFromUser(userID1);
FastIDSet prefs2 = dataModel.getItemIDsFromUser(userID2);
int prefs1Size = prefs1.size();
int prefs2Size = prefs2.size();
int intersectionSize =
prefs1Size < prefs2Size ? prefs2.intersectionSize(prefs1) : prefs1.intersectionSize(prefs2);
if (intersectionSize == 0) {
return Double.NaN;
}
// int numItems = dataModel.getNumItems();
// int numItems = Math.max(prefs1Size, prefs2Size);
// int numItems = prefs1Size;
// double distance1 = (double) (prefs1Size - intersectionSize)/(double) 2;
// double distance2 = (double) (prefs2Size - intersectionSize)/(double) 2;
// double distance = (distance1 + distance2);
double distance = (double) (prefs1Size - intersectionSize);
double similarity = 1.0 / (1.0 + distance);
////// System.out.println( prefs1Size + ", " + prefs2Size + ", " + intersectionSize + ", " +
// similarity);
return similarity;
}
@Override
public int getNumUsersWithPreferenceFor(long itemID1, long itemID2) {
FastIDSet userIDs1 = preferenceForItems.get(itemID1);
if (userIDs1 == null) {
return 0;
}
FastIDSet userIDs2 = preferenceForItems.get(itemID2);
if (userIDs2 == null) {
return 0;
}
return userIDs1.size() < userIDs2.size()
? userIDs2.intersectionSize(userIDs1)
: userIDs1.intersectionSize(userIDs2);
}
@Override
public double userSimilarity(long userID1, long userID2) throws TasteException {
FastIDSet prefs1 = dataModel.getItemIDsFromUser(userID1);
FastIDSet prefs2 = dataModel.getItemIDsFromUser(userID2);
int prefs1Size = prefs1.size();
int prefs2Size = prefs2.size();
int intersectionSize =
prefs1Size < prefs2Size ? prefs2.intersectionSize(prefs1) : prefs1.intersectionSize(prefs2);
if (intersectionSize == 0) {
return Double.NaN;
}
int numItems = dataModel.getNumItems();
double logLikelihood =
twoLogLambda(
intersectionSize, prefs1Size - intersectionSize, prefs2Size, numItems - prefs2Size);
return 1.0 - 1.0 / (1.0 + logLikelihood);
}
/**
* Creates a new {@link GenericDataModel} from the given users (and their preferences). This
* {@link DataModel} retains all this information in memory and is effectively immutable.
*
* @param userData users to include
* @param timestamps optionally, provided timestamps of preferences as milliseconds since the
* epoch. User IDs are mapped to maps of item IDs to Long timestamps.
*/
public GenericBooleanPrefDataModel(
FastByIDMap<FastIDSet> userData, FastByIDMap<FastByIDMap<Long>> timestamps) {
Preconditions.checkArgument(userData != null, "userData is null");
this.preferenceFromUsers = userData;
this.preferenceForItems = new FastByIDMap<>();
FastIDSet itemIDSet = new FastIDSet();
for (Map.Entry<Long, FastIDSet> entry : preferenceFromUsers.entrySet()) {
long userID = entry.getKey();
FastIDSet itemIDs = entry.getValue();
itemIDSet.addAll(itemIDs);
LongPrimitiveIterator it = itemIDs.iterator();
while (it.hasNext()) {
long itemID = it.nextLong();
FastIDSet userIDs = preferenceForItems.get(itemID);
if (userIDs == null) {
userIDs = new FastIDSet(2);
preferenceForItems.put(itemID, userIDs);
}
userIDs.add(userID);
}
}
this.itemIDs = itemIDSet.toArray();
itemIDSet = null; // Might help GC -- this is big
Arrays.sort(itemIDs);
this.userIDs = new long[userData.size()];
int i = 0;
LongPrimitiveIterator it = userData.keySetIterator();
while (it.hasNext()) {
userIDs[i++] = it.next();
}
Arrays.sort(userIDs);
this.timestamps = timestamps;
}
@Override
public double userSimilarity(long userID1, long userID2) throws TasteException {
DataModel dataModel = getDataModel();
FastIDSet xPrefs = dataModel.getItemIDsFromUser(userID1);
FastIDSet yPrefs = dataModel.getItemIDsFromUser(userID2);
int xPrefsSize = xPrefs.size();
int yPrefsSize = yPrefs.size();
if (xPrefsSize == 0 && yPrefsSize == 0) {
return Double.NaN;
}
if (xPrefsSize == 0 || yPrefsSize == 0) {
return 0.0;
}
double intersection = 0.0;
double union = 0.0;
for (LongPrimitiveIterator it_item = xPrefs.iterator(); it_item.hasNext(); ) {
long itemID = (long) it_item.nextLong();
double weight = (double) getDataModel().getNumUsers() / mItemPrefNum.get(itemID);
if (yPrefs.contains(itemID)) {
intersection += weight;
union -= weight;
}
union += weight;
}
for (LongPrimitiveIterator it_item = yPrefs.iterator(); it_item.hasNext(); ) {
long itemID = (long) it_item.nextLong();
double weight = (double) getDataModel().getNumUsers() / mItemPrefNum.get(itemID);
union += weight;
}
return Math.log(intersection) / Math.log(union);
}
@Override
public int getNumUsersWithPreferenceFor(long itemID) {
FastIDSet userIDs1 = preferenceForItems.get(itemID);
return userIDs1 == null ? 0 : userIDs1.size();
}