@Test
public void testMultiBLANC() {
final BLANCScorer multiBlancScorer = BLANCScorers.getMultiBLANCScorer();
final ImmutableSet<ImmutableSet<Integer>> reference = parseClustering("(1,2) (3,4,5,6) (7)");
final ImmutableSet<ImmutableSet<Integer>> system = parseClustering("(1,2,3) (4,5,8) (9)");
final BLANCResult score = multiBlancScorer.score(system, reference);
assertEquals(2 / 7.0, score.corefLinkRecall().get(), .0001);
assertEquals(1 / 3.0, score.corefLinkPrecision().get(), .0001);
assertEquals(2 / 7.0, score.nonCorefLinkRecall().get(), .0001);
assertEquals(4 / 15.0, score.nonCorefLinkPrecision().get(), .0001);
assertEquals(.30769, score.corefLinkF1().get(), .0001);
assertEquals(.27586, score.nonCorefLinkF1().get(), .0001);
assertEquals((.30769 + .27586) / 2.0, score.blancScore(), .0001);
}
private BLANCResult scoreSets(
final Iterable<Set<Object>> predicted, final Iterable<Set<Object>> gold) {
final Multimap<Object, Set<Object>> predictedItemToGroup =
CollectionUtils.makeSetElementsToContainersMultimap(predicted);
final Multimap<Object, Set<Object>> goldItemToGroup =
CollectionUtils.makeSetElementsToContainersMultimap(gold);
final Set<Object> keyItems = goldItemToGroup.keySet();
final Set<Object> responseItems = predictedItemToGroup.keySet();
final ImmutableSet<Object> itemsInBoth =
Sets.intersection(keyItems, responseItems).immutableCopy();
// |C_k \cap C_r|
int corefLinksInBoth = 0;
// |C_k|
int corefLinksInKey = 0;
// |C_r|
int corefLinksInResponse = 0;
// |N_K \cap N_r|
int nonCorefInBoth = 0;
// |N_k|
int nonCorefLinksInKey = 0;
// |N_r|
int nonCorefLinksInResponse = 0;
final Set<Object> allItems = Sets.union(responseItems, keyItems).immutableCopy();
for (final Object item : allItems) {
final boolean inKey = keyItems.contains(item);
final boolean inResponse = responseItems.contains(item);
final Collection<Set<Object>> predictedClusters = predictedItemToGroup.get(item);
final Collection<Set<Object>> goldClusters = goldItemToGroup.get(item);
final Predicate<Object> SELF_ADJUSTMENT_FILTER;
if (useSelfEdges) {
SELF_ADJUSTMENT_FILTER = Predicates.alwaysTrue();
} else {
SELF_ADJUSTMENT_FILTER = not(equalTo(item));
}
final int selfAdjustment = useSelfEdges ? 0 : -1;
final ImmutableSet<Object> predictedNeighbors =
FluentIterable.from(concat(predictedClusters)).filter(SELF_ADJUSTMENT_FILTER).toSet();
final ImmutableSet<Object> goldNeighbors =
FluentIterable.from(concat(goldClusters)).filter(SELF_ADJUSTMENT_FILTER).toSet();
// The contribution for this item is the size of the intersection
// of the gold and predicted neighbor sets.
corefLinksInBoth += Sets.intersection(predictedNeighbors, goldNeighbors).size();
corefLinksInResponse += predictedNeighbors.size();
corefLinksInKey += goldNeighbors.size();
if (inKey) {
nonCorefLinksInKey += keyItems.size() - goldNeighbors.size() + selfAdjustment;
}
if (inResponse) {
nonCorefLinksInResponse +=
responseItems.size() - predictedNeighbors.size() + selfAdjustment;
}
if (inKey && inResponse) {
final ImmutableSet<Object> neighborsInEither =
Sets.union(predictedNeighbors, goldNeighbors).immutableCopy();
// -1 = don't count this item itself as a link
nonCorefInBoth += Sets.difference(itemsInBoth, neighborsInEither).size() + selfAdjustment;
}
}
return BLANCResult.fromSetCounts(
keyItems.equals(responseItems),
corefLinksInBoth,
corefLinksInKey,
corefLinksInResponse,
nonCorefInBoth,
nonCorefLinksInKey,
nonCorefLinksInResponse);
}
