/**
* Evaluates one relation and returns a gold relation which corresponds to the specified system
* relation arguments.
*
* @param args - system relation agruments.
* @param goldRelations - list of all gold relations
* @param gold - JAXB-representation of the entire document.
* @param label - label class used for the evaluation (depricated)
* @param evalOption - evaluation task
* @param outcome - outcome to store results.
* @param strict TODO
*/
private static <T extends WithIdentifier> T evaluateRelation(
Map<Class<? extends Markable>, List<? extends Markable>> args,
List<T> goldRelations,
SpRL gold,
Class<T> label,
String evalOption,
Outcome outcome,
boolean strict) {
// List<T> goldRelations = JAXBUtil.getAnnotations(gold, label);
int maxMatch = Integer.MIN_VALUE;
T found = null;
if (goldRelations != null) {
for (Iterator<T> iterator = goldRelations.iterator(); iterator.hasNext(); ) {
T r = iterator.next();
RELATION goldRelation = (RELATION) r;
// skip if it is an EQ = coreference
if (goldRelation.getRCC8Value() != null
&& goldRelation.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue;
Map<Class<? extends Markable>, List<? extends Markable>> goldArgs =
JAXBUtil.getArgs(goldRelation, gold);
// matches is a map of labels and matches (true, false) between system and gold annotations.
// The span and the label have to be the same.
Map<Class<Markable>, Boolean> matches = JAXBUtil.getMarkableMatches(goldArgs, args, strict);
int matchesFound =
JAXBUtil.getCntNonEmptyArgs(
matches,
EVAL_MAP.get(evalOption).toArray(new Class[EVAL_MAP.get(evalOption).size()]));
/*
// span-based matches
Map<Class<Markable>, Boolean> spanMatches =
JAXBUtil.getSpanMatches(goldArgs, args);
int spanMatchesFound = JAXBUtil.getCntNonEmptyArgs(spanMatches,
EVAL_MAP.get(evalOption).toArray(new Class[EVAL_MAP.get(evalOption).size()]));
*/
// swap swappable arguments
Map<Class<? extends Markable>, List<? extends Markable>> swappedArgs =
swapSymmetricalArgs(args);
// find matches
Map<Class<Markable>, Boolean> swappedMatches =
JAXBUtil.getMarkableMatches(goldArgs, swappedArgs, strict);
// count matches
int swappedMatchesFound =
JAXBUtil.getCntNonEmptyArgs(
swappedMatches,
EVAL_MAP.get(evalOption).toArray(new Class[EVAL_MAP.get(evalOption).size()]));
// take the max
matchesFound = Math.max(matchesFound, swappedMatchesFound);
int toMatch = EVAL_MAP.get(evalOption).size();
if (matchesFound - toMatch > maxMatch) {
found = r;
maxMatch = matchesFound - toMatch;
}
// if the max match number is found
if (maxMatch >= 0) break;
}
}
if (maxMatch >= 0 && found != null) {
// outcome.evaluate(label.getSimpleName(), label.getSimpleName());
// goldRelations.remove(found);
} else {
// outcome.evaluate(NULL, label.getSimpleName());
found = null;
}
return found;
}
/**
* Evaluates relations.
*
* @param gold
* @param system
* @param labelClass
* @param evalOption
* @param matchLabel TODO
* @param strictSpan TODO
* @param outcomes
*/
private static <T extends WithIdentifier> void evaluateRelations(
SpRL gold,
SpRL system,
Class<T> labelClass,
String evalOption,
Outcome outcome,
boolean matchLabel,
boolean strictSpan) {
if (!gold.getTEXT().getContent().equalsIgnoreCase(system.getTEXT().getContent())) {
if (!gold.getTEXT().getContent().contains(system.getTEXT().getContent()))
// throw new IllegalArgumentException("The content of the data is not equal.");
System.err.println(
system.getFilename()
+ ": The content of the data is not equal. Gold text length: "
+ gold.getTEXT().getContent().length()
+ " System text length:"
+ system.getTEXT().getContent().length()
+ " EVALUATING ANYWAY!");
}
// else{
List<T> goldAnnotations = null;
List<T> systemAnnotations = null;
if (labelClass == null) {
throw new NullPointerException("Annotation class is not defined.");
} else {
goldAnnotations = JAXBUtil.getAnnotations(gold, labelClass);
systemAnnotations = JAXBUtil.getAnnotations(system, labelClass);
}
List<T> goldRelations = JAXBUtil.getAnnotations(gold, labelClass); // all gold relations
String goldLabel = "";
String systemLabel = "";
if (systemAnnotations != null) {
for (Iterator<T> iterator = systemAnnotations.iterator(); iterator.hasNext(); ) {
T t = iterator.next();
RELATION systemRelation = (RELATION) t;
if (systemRelation.getGeneralType() != null
&& systemRelation.getGeneralType().trim().length() == 0)
System.err.println("ERROR in " + system.getFilename());
if (systemRelation.getRCC8Value() != null
&& systemRelation.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue;
Map<Class<? extends Markable>, List<? extends Markable>> args =
JAXBUtil.getArgs(systemRelation, system);
RELATION goldMatch =
(RELATION)
evaluateRelation(
args, goldRelations, gold, labelClass, evalOption, outcome, strictSpan);
if (evalOption.equalsIgnoreCase(TASK_E)) {
goldLabel = goldMatch != null ? goldMatch.getGeneralType() : NULL;
systemLabel = systemRelation.getGeneralType();
} else {
goldLabel = labelClass.getSimpleName();
systemLabel = labelClass.getSimpleName();
}
if (systemLabel == null) systemLabel = NULL; // if no rel found
if (goldMatch != null) {
outcome.evaluate(
truncateTo(goldLabel, MAX_LEN),
truncateTo(systemLabel, MAX_LEN)); // exact match for markables
goldRelations.remove(goldMatch);
} else {
outcome.evaluate(NULL, truncateTo(systemLabel, MAX_LEN));
goldMatch = null;
}
/*
System.out.println("SYSTEM OUTPUT:\t"+r.toString(args));
if(goldMatch != null){
System.out.println("MATCH FOUND:\t"+goldMatch.toString(JAXBUtil.getArgs(goldMatch, gold)));
}else
System.out.println("NO MATCH FOUND");
System.out.println();
*/
iterator.remove();
}
}
if (goldRelations != null) {
// for the rest of gold annotation which were not matched by markables
for (Iterator<T> iterator = goldRelations.iterator(); iterator.hasNext(); ) {
T t = iterator.next();
RELATION r = (RELATION) t;
// ignore coreferential relations
if (r.getRCC8Value() != null && r.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue;
if (evalOption.equalsIgnoreCase(TASK_E)) {
goldLabel = r.getGeneralType();
systemLabel = NULL;
} else {
goldLabel = labelClass.getSimpleName();
systemLabel = NULL;
}
outcome.evaluate(truncateTo(goldLabel, MAX_LEN), NULL);
/*
System.out.println("SYSTEM OUTPUT:\t"+null);
System.out.println("GOLD ANN:\t"+r.toString(JAXBUtil.getArgs((RELATION) r, gold)));
System.out.println();
*/
}
}
// }
}