/** * Updates the outcome with respect to the labels found. * * @param goldLabels - gold labels * @param systemLabels - system labels * @param outcome -outcome */ private static void evaluate(String[] goldLabels, String[] systemLabels, Outcome outcome) { List<String> goldList = new ArrayList<String>(); if (goldLabels != null) goldList = new ArrayList<String>(Arrays.asList(goldLabels)); List<String> systemList = new ArrayList<String>(); if (systemLabels != null) systemList = new ArrayList<String>(Arrays.asList(systemLabels)); JAXBUtil.normalizeLists( goldList, systemList, NULL); // make lists of the same size by adding additional nulls // for (Iterator<String> iterator = systemList.iterator(); iterator.hasNext(); ) { String systemLabel = iterator.next(); int goldLabelIndex = goldList.indexOf(systemLabel); // look for the same label in the gold list // if not found, try to find the Null if (goldLabelIndex == -1) goldLabelIndex = goldList.indexOf(NULL); // if NULL label is not a gold label either, pick the first one if (goldLabelIndex == -1 && goldList.size() > 0) goldLabelIndex = 0; /* else{ System.err.println(new IllegalArgumentException(String.format("Ups! No label pair is found. System label: %s Gold labels: %s", systemLabel, goldList))); } */ String goldLabel = goldList.get(goldLabelIndex); // store the results outcome.evaluate(goldLabel, systemLabel); // remove labels iterator.remove(); goldList.remove(goldLabelIndex); } }
/** * Evaluates relations. * * @param gold * @param system * @param labelClass * @param evalOption * @param matchLabel TODO * @param strictSpan TODO * @param outcomes */ private static <T extends WithIdentifier> void evaluateRelations( SpRL gold, SpRL system, Class<T> labelClass, String evalOption, Outcome outcome, boolean matchLabel, boolean strictSpan) { if (!gold.getTEXT().getContent().equalsIgnoreCase(system.getTEXT().getContent())) { if (!gold.getTEXT().getContent().contains(system.getTEXT().getContent())) // throw new IllegalArgumentException("The content of the data is not equal."); System.err.println( system.getFilename() + ": The content of the data is not equal. Gold text length: " + gold.getTEXT().getContent().length() + " System text length:" + system.getTEXT().getContent().length() + " EVALUATING ANYWAY!"); } // else{ List<T> goldAnnotations = null; List<T> systemAnnotations = null; if (labelClass == null) { throw new NullPointerException("Annotation class is not defined."); } else { goldAnnotations = JAXBUtil.getAnnotations(gold, labelClass); systemAnnotations = JAXBUtil.getAnnotations(system, labelClass); } List<T> goldRelations = JAXBUtil.getAnnotations(gold, labelClass); // all gold relations String goldLabel = ""; String systemLabel = ""; if (systemAnnotations != null) { for (Iterator<T> iterator = systemAnnotations.iterator(); iterator.hasNext(); ) { T t = iterator.next(); RELATION systemRelation = (RELATION) t; if (systemRelation.getGeneralType() != null && systemRelation.getGeneralType().trim().length() == 0) System.err.println("ERROR in " + system.getFilename()); if (systemRelation.getRCC8Value() != null && systemRelation.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue; Map<Class<? extends Markable>, List<? extends Markable>> args = JAXBUtil.getArgs(systemRelation, system); RELATION goldMatch = (RELATION) evaluateRelation( args, goldRelations, gold, labelClass, evalOption, outcome, strictSpan); if (evalOption.equalsIgnoreCase(TASK_E)) { goldLabel = goldMatch != null ? goldMatch.getGeneralType() : NULL; systemLabel = systemRelation.getGeneralType(); } else { goldLabel = labelClass.getSimpleName(); systemLabel = labelClass.getSimpleName(); } if (systemLabel == null) systemLabel = NULL; // if no rel found if (goldMatch != null) { outcome.evaluate( truncateTo(goldLabel, MAX_LEN), truncateTo(systemLabel, MAX_LEN)); // exact match for markables goldRelations.remove(goldMatch); } else { outcome.evaluate(NULL, truncateTo(systemLabel, MAX_LEN)); goldMatch = null; } /* System.out.println("SYSTEM OUTPUT:\t"+r.toString(args)); if(goldMatch != null){ System.out.println("MATCH FOUND:\t"+goldMatch.toString(JAXBUtil.getArgs(goldMatch, gold))); }else System.out.println("NO MATCH FOUND"); System.out.println(); */ iterator.remove(); } } if (goldRelations != null) { // for the rest of gold annotation which were not matched by markables for (Iterator<T> iterator = goldRelations.iterator(); iterator.hasNext(); ) { T t = iterator.next(); RELATION r = (RELATION) t; // ignore coreferential relations if (r.getRCC8Value() != null && r.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue; if (evalOption.equalsIgnoreCase(TASK_E)) { goldLabel = r.getGeneralType(); systemLabel = NULL; } else { goldLabel = labelClass.getSimpleName(); systemLabel = NULL; } outcome.evaluate(truncateTo(goldLabel, MAX_LEN), NULL); /* System.out.println("SYSTEM OUTPUT:\t"+null); System.out.println("GOLD ANN:\t"+r.toString(JAXBUtil.getArgs((RELATION) r, gold))); System.out.println(); */ } } // } }