/**
* Evaluates relations.
*
* @param gold
* @param system
* @param labelClass
* @param evalOption
* @param matchLabel TODO
* @param strictSpan TODO
* @param outcomes
*/
private static <T extends WithIdentifier> void evaluateRelations(
SpRL gold,
SpRL system,
Class<T> labelClass,
String evalOption,
Outcome outcome,
boolean matchLabel,
boolean strictSpan) {
if (!gold.getTEXT().getContent().equalsIgnoreCase(system.getTEXT().getContent())) {
if (!gold.getTEXT().getContent().contains(system.getTEXT().getContent()))
// throw new IllegalArgumentException("The content of the data is not equal.");
System.err.println(
system.getFilename()
+ ": The content of the data is not equal. Gold text length: "
+ gold.getTEXT().getContent().length()
+ " System text length:"
+ system.getTEXT().getContent().length()
+ " EVALUATING ANYWAY!");
}
// else{
List<T> goldAnnotations = null;
List<T> systemAnnotations = null;
if (labelClass == null) {
throw new NullPointerException("Annotation class is not defined.");
} else {
goldAnnotations = JAXBUtil.getAnnotations(gold, labelClass);
systemAnnotations = JAXBUtil.getAnnotations(system, labelClass);
}
List<T> goldRelations = JAXBUtil.getAnnotations(gold, labelClass); // all gold relations
String goldLabel = "";
String systemLabel = "";
if (systemAnnotations != null) {
for (Iterator<T> iterator = systemAnnotations.iterator(); iterator.hasNext(); ) {
T t = iterator.next();
RELATION systemRelation = (RELATION) t;
if (systemRelation.getGeneralType() != null
&& systemRelation.getGeneralType().trim().length() == 0)
System.err.println("ERROR in " + system.getFilename());
if (systemRelation.getRCC8Value() != null
&& systemRelation.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue;
Map<Class<? extends Markable>, List<? extends Markable>> args =
JAXBUtil.getArgs(systemRelation, system);
RELATION goldMatch =
(RELATION)
evaluateRelation(
args, goldRelations, gold, labelClass, evalOption, outcome, strictSpan);
if (evalOption.equalsIgnoreCase(TASK_E)) {
goldLabel = goldMatch != null ? goldMatch.getGeneralType() : NULL;
systemLabel = systemRelation.getGeneralType();
} else {
goldLabel = labelClass.getSimpleName();
systemLabel = labelClass.getSimpleName();
}
if (systemLabel == null) systemLabel = NULL; // if no rel found
if (goldMatch != null) {
outcome.evaluate(
truncateTo(goldLabel, MAX_LEN),
truncateTo(systemLabel, MAX_LEN)); // exact match for markables
goldRelations.remove(goldMatch);
} else {
outcome.evaluate(NULL, truncateTo(systemLabel, MAX_LEN));
goldMatch = null;
}
/*
System.out.println("SYSTEM OUTPUT:\t"+r.toString(args));
if(goldMatch != null){
System.out.println("MATCH FOUND:\t"+goldMatch.toString(JAXBUtil.getArgs(goldMatch, gold)));
}else
System.out.println("NO MATCH FOUND");
System.out.println();
*/
iterator.remove();
}
}
if (goldRelations != null) {
// for the rest of gold annotation which were not matched by markables
for (Iterator<T> iterator = goldRelations.iterator(); iterator.hasNext(); ) {
T t = iterator.next();
RELATION r = (RELATION) t;
// ignore coreferential relations
if (r.getRCC8Value() != null && r.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue;
if (evalOption.equalsIgnoreCase(TASK_E)) {
goldLabel = r.getGeneralType();
systemLabel = NULL;
} else {
goldLabel = labelClass.getSimpleName();
systemLabel = NULL;
}
outcome.evaluate(truncateTo(goldLabel, MAX_LEN), NULL);
/*
System.out.println("SYSTEM OUTPUT:\t"+null);
System.out.println("GOLD ANN:\t"+r.toString(JAXBUtil.getArgs((RELATION) r, gold)));
System.out.println();
*/
}
}
// }
}
