@Value.Check
protected void checkValidity() {
// no incomplete response may appear in any response set
final ImmutableSet<Response> allResponsesInSets = ImmutableSet.copyOf(concat(responseSets()));
for (final Response incompleteResponse : incompleteResponses()) {
checkArgument(
!allResponsesInSets.contains(incompleteResponse),
"A response may not be both completed and incomplete");
}
if (responseSetIds().isPresent()) {
for (final String id : responseSetIds().get().keySet()) {
checkArgument(!id.contains("-"), "Event frame IDs may not contain -s");
checkArgument(!id.contains("\t"), "Event frame IDs may not contain tabs");
}
// we can have an empty output file, verify that all the responseSets have an id
checkArgument(
responseSets().size() == responseSetIds().get().size(), "Response set IDs are missing");
checkArgument(
responseSetIds().get().keySet().size() == responseSets().size(),
"All response set IDs must be unique");
CollectionUtils.assertSameElementsOrIllegalArgument(
responseSets(),
responseSetIds().get().values(),
"Response sets did not match IDs",
"Response sets in list",
"Response sets in ID map");
}
}
private BLANCResult scoreSets(
final Iterable<Set<Object>> predicted, final Iterable<Set<Object>> gold) {
final Multimap<Object, Set<Object>> predictedItemToGroup =
CollectionUtils.makeSetElementsToContainersMultimap(predicted);
final Multimap<Object, Set<Object>> goldItemToGroup =
CollectionUtils.makeSetElementsToContainersMultimap(gold);
final Set<Object> keyItems = goldItemToGroup.keySet();
final Set<Object> responseItems = predictedItemToGroup.keySet();
final ImmutableSet<Object> itemsInBoth =
Sets.intersection(keyItems, responseItems).immutableCopy();
// |C_k \cap C_r|
int corefLinksInBoth = 0;
// |C_k|
int corefLinksInKey = 0;
// |C_r|
int corefLinksInResponse = 0;
// |N_K \cap N_r|
int nonCorefInBoth = 0;
// |N_k|
int nonCorefLinksInKey = 0;
// |N_r|
int nonCorefLinksInResponse = 0;
final Set<Object> allItems = Sets.union(responseItems, keyItems).immutableCopy();
for (final Object item : allItems) {
final boolean inKey = keyItems.contains(item);
final boolean inResponse = responseItems.contains(item);
final Collection<Set<Object>> predictedClusters = predictedItemToGroup.get(item);
final Collection<Set<Object>> goldClusters = goldItemToGroup.get(item);
final Predicate<Object> SELF_ADJUSTMENT_FILTER;
if (useSelfEdges) {
SELF_ADJUSTMENT_FILTER = Predicates.alwaysTrue();
} else {
SELF_ADJUSTMENT_FILTER = not(equalTo(item));
}
final int selfAdjustment = useSelfEdges ? 0 : -1;
final ImmutableSet<Object> predictedNeighbors =
FluentIterable.from(concat(predictedClusters)).filter(SELF_ADJUSTMENT_FILTER).toSet();
final ImmutableSet<Object> goldNeighbors =
FluentIterable.from(concat(goldClusters)).filter(SELF_ADJUSTMENT_FILTER).toSet();
// The contribution for this item is the size of the intersection
// of the gold and predicted neighbor sets.
corefLinksInBoth += Sets.intersection(predictedNeighbors, goldNeighbors).size();
corefLinksInResponse += predictedNeighbors.size();
corefLinksInKey += goldNeighbors.size();
if (inKey) {
nonCorefLinksInKey += keyItems.size() - goldNeighbors.size() + selfAdjustment;
}
if (inResponse) {
nonCorefLinksInResponse +=
responseItems.size() - predictedNeighbors.size() + selfAdjustment;
}
if (inKey && inResponse) {
final ImmutableSet<Object> neighborsInEither =
Sets.union(predictedNeighbors, goldNeighbors).immutableCopy();
// -1 = don't count this item itself as a link
nonCorefInBoth += Sets.difference(itemsInBoth, neighborsInEither).size() + selfAdjustment;
}
}
return BLANCResult.fromSetCounts(
keyItems.equals(responseItems),
corefLinksInBoth,
corefLinksInKey,
corefLinksInResponse,
nonCorefInBoth,
nonCorefLinksInKey,
nonCorefLinksInResponse);
}
