/**
* Evaluates markable annotations for one specified label (provided as a JAXB annotation class)
* and saves the results into the list of outcomes.
*
* @param gold - JAXB-representation of a gold standard file
* @param system - JAXB-representation of a system annotated file
* @param labels - JAXB-class for which the evaluation is performed. If <code>null</code> is
* provided the evaluation is performed for all markables.
* @param outcome - list of outcomes. It will be updated with the evaluation results for this
* class.
*/
private static <T extends Comparable<? super T>> void evaluateMarkables(
SpRL gold, SpRL system, List<Class<T>> labels, Outcome outcome) {
if (!gold.getTEXT().getContent().equalsIgnoreCase(system.getTEXT().getContent())) {
if (!gold.getTEXT().getContent().contains(system.getTEXT().getContent()))
// throw new IllegalArgumentException("The content of the data is not equal.");
System.err.println(
"The content of the data is not equal. Gold text length: "
+ gold.getTEXT().getContent().length()
+ " System text length:"
+ system.getTEXT().getContent().length()
+ " EVALUATING ANYWAY!");
}
// else{
List<T> goldAnnotations = null;
List<T> systemAnnotations = null;
if (labels == null) {
goldAnnotations = (List<T>) JAXBUtil.getAllMarkables(gold);
systemAnnotations = (List<T>) JAXBUtil.getAllMarkables(system);
} else {
goldAnnotations = JAXBUtil.getAnnotations(gold, labels);
systemAnnotations = JAXBUtil.getAnnotations(system, labels);
}
if (systemAnnotations == null) systemAnnotations = new ArrayList<T>();
if (goldAnnotations == null) goldAnnotations = new ArrayList<T>();
validate(systemAnnotations, system.getFilename());
Collections.sort(goldAnnotations);
Collections.sort(systemAnnotations);
String text = gold.getTEXT().getContent();
int length = text.length();
for (int i = 0; i < length; i++) {
String[] goldLabels = getLabels(i, i, goldAnnotations);
String[] systemLabels = getLabels(i, i, systemAnnotations);
evaluate(goldLabels, systemLabels, outcome);
}
// }
}
/**
* Swaps eligible arguments in relations to address symmetry.
*
* @param newArgs
* @return
*/
private static Map<Class<? extends Markable>, List<? extends Markable>> swapSymmetricalArgs(
Map<Class<? extends Markable>, List<? extends Markable>> newArgs) {
Map<Class<? extends Markable>, List<? extends Markable>> result =
new HashMap<Class<? extends Markable>, List<? extends Markable>>(newArgs);
Set<Class<? extends Markable>> labels = SYMMETRIC_CLASSES.keySet();
for (Class<? extends Markable> label : labels) {
Class<? extends Markable> swapWithLabel = SYMMETRIC_CLASSES.get(label);
List<? extends Markable> origMarkables = newArgs.get(label);
Markable firstMarkable = JAXBUtil.unwrapFromList(origMarkables); // first element to swap
if (firstMarkable == null) firstMarkable = createMarkable(label);
Markable firstClone = JAXBUtil.copyObj(firstMarkable);
origMarkables = newArgs.get(swapWithLabel);
Markable secondMarkable = JAXBUtil.unwrapFromList(origMarkables); // second element to swap
if (secondMarkable == null) secondMarkable = createMarkable(swapWithLabel);
Markable secondClone = JAXBUtil.copyObj(secondMarkable);
swap(firstClone, secondClone);
// set those which received value from nulls to nulls.
if (firstClone.getStart() == null) firstClone = null;
if (secondClone.getStart() == null) secondClone = null;
result.put(label, JAXBUtil.wrapToList(firstClone));
result.put(swapWithLabel, JAXBUtil.wrapToList(secondClone));
}
return result;
}
/**
* Updates the outcome with respect to the labels found.
*
* @param goldLabels - gold labels
* @param systemLabels - system labels
* @param outcome -outcome
*/
private static void evaluate(String[] goldLabels, String[] systemLabels, Outcome outcome) {
List<String> goldList = new ArrayList<String>();
if (goldLabels != null) goldList = new ArrayList<String>(Arrays.asList(goldLabels));
List<String> systemList = new ArrayList<String>();
if (systemLabels != null) systemList = new ArrayList<String>(Arrays.asList(systemLabels));
JAXBUtil.normalizeLists(
goldList, systemList, NULL); // make lists of the same size by adding additional nulls
//
for (Iterator<String> iterator = systemList.iterator(); iterator.hasNext(); ) {
String systemLabel = iterator.next();
int goldLabelIndex =
goldList.indexOf(systemLabel); // look for the same label in the gold list
// if not found, try to find the Null
if (goldLabelIndex == -1) goldLabelIndex = goldList.indexOf(NULL);
// if NULL label is not a gold label either, pick the first one
if (goldLabelIndex == -1 && goldList.size() > 0) goldLabelIndex = 0;
/*
else{
System.err.println(new IllegalArgumentException(String.format("Ups! No label pair is found. System label: %s Gold labels: %s",
systemLabel, goldList)));
}
*/
String goldLabel = goldList.get(goldLabelIndex);
// store the results
outcome.evaluate(goldLabel, systemLabel);
// remove labels
iterator.remove();
goldList.remove(goldLabelIndex);
}
}
/**
* Evaluates one relation and returns a gold relation which corresponds to the specified system
* relation arguments.
*
* @param args - system relation agruments.
* @param goldRelations - list of all gold relations
* @param gold - JAXB-representation of the entire document.
* @param label - label class used for the evaluation (depricated)
* @param evalOption - evaluation task
* @param outcome - outcome to store results.
* @param strict TODO
*/
private static <T extends WithIdentifier> T evaluateRelation(
Map<Class<? extends Markable>, List<? extends Markable>> args,
List<T> goldRelations,
SpRL gold,
Class<T> label,
String evalOption,
Outcome outcome,
boolean strict) {
// List<T> goldRelations = JAXBUtil.getAnnotations(gold, label);
int maxMatch = Integer.MIN_VALUE;
T found = null;
if (goldRelations != null) {
for (Iterator<T> iterator = goldRelations.iterator(); iterator.hasNext(); ) {
T r = iterator.next();
RELATION goldRelation = (RELATION) r;
// skip if it is an EQ = coreference
if (goldRelation.getRCC8Value() != null
&& goldRelation.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue;
Map<Class<? extends Markable>, List<? extends Markable>> goldArgs =
JAXBUtil.getArgs(goldRelation, gold);
// matches is a map of labels and matches (true, false) between system and gold annotations.
// The span and the label have to be the same.
Map<Class<Markable>, Boolean> matches = JAXBUtil.getMarkableMatches(goldArgs, args, strict);
int matchesFound =
JAXBUtil.getCntNonEmptyArgs(
matches,
EVAL_MAP.get(evalOption).toArray(new Class[EVAL_MAP.get(evalOption).size()]));
/*
// span-based matches
Map<Class<Markable>, Boolean> spanMatches =
JAXBUtil.getSpanMatches(goldArgs, args);
int spanMatchesFound = JAXBUtil.getCntNonEmptyArgs(spanMatches,
EVAL_MAP.get(evalOption).toArray(new Class[EVAL_MAP.get(evalOption).size()]));
*/
// swap swappable arguments
Map<Class<? extends Markable>, List<? extends Markable>> swappedArgs =
swapSymmetricalArgs(args);
// find matches
Map<Class<Markable>, Boolean> swappedMatches =
JAXBUtil.getMarkableMatches(goldArgs, swappedArgs, strict);
// count matches
int swappedMatchesFound =
JAXBUtil.getCntNonEmptyArgs(
swappedMatches,
EVAL_MAP.get(evalOption).toArray(new Class[EVAL_MAP.get(evalOption).size()]));
// take the max
matchesFound = Math.max(matchesFound, swappedMatchesFound);
int toMatch = EVAL_MAP.get(evalOption).size();
if (matchesFound - toMatch > maxMatch) {
found = r;
maxMatch = matchesFound - toMatch;
}
// if the max match number is found
if (maxMatch >= 0) break;
}
}
if (maxMatch >= 0 && found != null) {
// outcome.evaluate(label.getSimpleName(), label.getSimpleName());
// goldRelations.remove(found);
} else {
// outcome.evaluate(NULL, label.getSimpleName());
found = null;
}
return found;
}
/**
* Evaluates relations.
*
* @param gold
* @param system
* @param labelClass
* @param evalOption
* @param matchLabel TODO
* @param strictSpan TODO
* @param outcomes
*/
private static <T extends WithIdentifier> void evaluateRelations(
SpRL gold,
SpRL system,
Class<T> labelClass,
String evalOption,
Outcome outcome,
boolean matchLabel,
boolean strictSpan) {
if (!gold.getTEXT().getContent().equalsIgnoreCase(system.getTEXT().getContent())) {
if (!gold.getTEXT().getContent().contains(system.getTEXT().getContent()))
// throw new IllegalArgumentException("The content of the data is not equal.");
System.err.println(
system.getFilename()
+ ": The content of the data is not equal. Gold text length: "
+ gold.getTEXT().getContent().length()
+ " System text length:"
+ system.getTEXT().getContent().length()
+ " EVALUATING ANYWAY!");
}
// else{
List<T> goldAnnotations = null;
List<T> systemAnnotations = null;
if (labelClass == null) {
throw new NullPointerException("Annotation class is not defined.");
} else {
goldAnnotations = JAXBUtil.getAnnotations(gold, labelClass);
systemAnnotations = JAXBUtil.getAnnotations(system, labelClass);
}
List<T> goldRelations = JAXBUtil.getAnnotations(gold, labelClass); // all gold relations
String goldLabel = "";
String systemLabel = "";
if (systemAnnotations != null) {
for (Iterator<T> iterator = systemAnnotations.iterator(); iterator.hasNext(); ) {
T t = iterator.next();
RELATION systemRelation = (RELATION) t;
if (systemRelation.getGeneralType() != null
&& systemRelation.getGeneralType().trim().length() == 0)
System.err.println("ERROR in " + system.getFilename());
if (systemRelation.getRCC8Value() != null
&& systemRelation.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue;
Map<Class<? extends Markable>, List<? extends Markable>> args =
JAXBUtil.getArgs(systemRelation, system);
RELATION goldMatch =
(RELATION)
evaluateRelation(
args, goldRelations, gold, labelClass, evalOption, outcome, strictSpan);
if (evalOption.equalsIgnoreCase(TASK_E)) {
goldLabel = goldMatch != null ? goldMatch.getGeneralType() : NULL;
systemLabel = systemRelation.getGeneralType();
} else {
goldLabel = labelClass.getSimpleName();
systemLabel = labelClass.getSimpleName();
}
if (systemLabel == null) systemLabel = NULL; // if no rel found
if (goldMatch != null) {
outcome.evaluate(
truncateTo(goldLabel, MAX_LEN),
truncateTo(systemLabel, MAX_LEN)); // exact match for markables
goldRelations.remove(goldMatch);
} else {
outcome.evaluate(NULL, truncateTo(systemLabel, MAX_LEN));
goldMatch = null;
}
/*
System.out.println("SYSTEM OUTPUT:\t"+r.toString(args));
if(goldMatch != null){
System.out.println("MATCH FOUND:\t"+goldMatch.toString(JAXBUtil.getArgs(goldMatch, gold)));
}else
System.out.println("NO MATCH FOUND");
System.out.println();
*/
iterator.remove();
}
}
if (goldRelations != null) {
// for the rest of gold annotation which were not matched by markables
for (Iterator<T> iterator = goldRelations.iterator(); iterator.hasNext(); ) {
T t = iterator.next();
RELATION r = (RELATION) t;
// ignore coreferential relations
if (r.getRCC8Value() != null && r.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue;
if (evalOption.equalsIgnoreCase(TASK_E)) {
goldLabel = r.getGeneralType();
systemLabel = NULL;
} else {
goldLabel = labelClass.getSimpleName();
systemLabel = NULL;
}
outcome.evaluate(truncateTo(goldLabel, MAX_LEN), NULL);
/*
System.out.println("SYSTEM OUTPUT:\t"+null);
System.out.println("GOLD ANN:\t"+r.toString(JAXBUtil.getArgs((RELATION) r, gold)));
System.out.println();
*/
}
}
// }
}
