Example #1
0
  /**
   * Evaluates markable annotations for one specified label (provided as a JAXB annotation class)
   * and saves the results into the list of outcomes.
   *
   * @param gold - JAXB-representation of a gold standard file
   * @param system - JAXB-representation of a system annotated file
   * @param labels - JAXB-class for which the evaluation is performed. If <code>null</code> is
   *     provided the evaluation is performed for all markables.
   * @param outcome - list of outcomes. It will be updated with the evaluation results for this
   *     class.
   */
  private static <T extends Comparable<? super T>> void evaluateMarkables(
      SpRL gold, SpRL system, List<Class<T>> labels, Outcome outcome) {

    if (!gold.getTEXT().getContent().equalsIgnoreCase(system.getTEXT().getContent())) {
      if (!gold.getTEXT().getContent().contains(system.getTEXT().getContent()))
        // throw new IllegalArgumentException("The content of the data is not equal.");
        System.err.println(
            "The content of the data is not equal. Gold text length: "
                + gold.getTEXT().getContent().length()
                + " System text length:"
                + system.getTEXT().getContent().length()
                + " EVALUATING ANYWAY!");
    }
    // else{
    List<T> goldAnnotations = null;
    List<T> systemAnnotations = null;

    if (labels == null) {
      goldAnnotations = (List<T>) JAXBUtil.getAllMarkables(gold);
      systemAnnotations = (List<T>) JAXBUtil.getAllMarkables(system);

    } else {
      goldAnnotations = JAXBUtil.getAnnotations(gold, labels);
      systemAnnotations = JAXBUtil.getAnnotations(system, labels);
    }
    if (systemAnnotations == null) systemAnnotations = new ArrayList<T>();
    if (goldAnnotations == null) goldAnnotations = new ArrayList<T>();

    validate(systemAnnotations, system.getFilename());

    Collections.sort(goldAnnotations);
    Collections.sort(systemAnnotations);

    String text = gold.getTEXT().getContent();
    int length = text.length();
    for (int i = 0; i < length; i++) {
      String[] goldLabels = getLabels(i, i, goldAnnotations);
      String[] systemLabels = getLabels(i, i, systemAnnotations);
      evaluate(goldLabels, systemLabels, outcome);
    }
    // }
  }
Example #2
0
  /**
   * Swaps eligible arguments in relations to address symmetry.
   *
   * @param newArgs
   * @return
   */
  private static Map<Class<? extends Markable>, List<? extends Markable>> swapSymmetricalArgs(
      Map<Class<? extends Markable>, List<? extends Markable>> newArgs) {

    Map<Class<? extends Markable>, List<? extends Markable>> result =
        new HashMap<Class<? extends Markable>, List<? extends Markable>>(newArgs);

    Set<Class<? extends Markable>> labels = SYMMETRIC_CLASSES.keySet();
    for (Class<? extends Markable> label : labels) {
      Class<? extends Markable> swapWithLabel = SYMMETRIC_CLASSES.get(label);

      List<? extends Markable> origMarkables = newArgs.get(label);
      Markable firstMarkable = JAXBUtil.unwrapFromList(origMarkables); // first element to swap

      if (firstMarkable == null) firstMarkable = createMarkable(label);
      Markable firstClone = JAXBUtil.copyObj(firstMarkable);

      origMarkables = newArgs.get(swapWithLabel);
      Markable secondMarkable = JAXBUtil.unwrapFromList(origMarkables); // second element to swap
      if (secondMarkable == null) secondMarkable = createMarkable(swapWithLabel);
      Markable secondClone = JAXBUtil.copyObj(secondMarkable);

      swap(firstClone, secondClone);

      // set those which received value from nulls to nulls.
      if (firstClone.getStart() == null) firstClone = null;
      if (secondClone.getStart() == null) secondClone = null;

      result.put(label, JAXBUtil.wrapToList(firstClone));
      result.put(swapWithLabel, JAXBUtil.wrapToList(secondClone));
    }
    return result;
  }
Example #3
0
  /**
   * Updates the outcome with respect to the labels found.
   *
   * @param goldLabels - gold labels
   * @param systemLabels - system labels
   * @param outcome -outcome
   */
  private static void evaluate(String[] goldLabels, String[] systemLabels, Outcome outcome) {
    List<String> goldList = new ArrayList<String>();
    if (goldLabels != null) goldList = new ArrayList<String>(Arrays.asList(goldLabels));

    List<String> systemList = new ArrayList<String>();
    if (systemLabels != null) systemList = new ArrayList<String>(Arrays.asList(systemLabels));

    JAXBUtil.normalizeLists(
        goldList, systemList, NULL); // make lists of the same size by adding additional nulls

    //
    for (Iterator<String> iterator = systemList.iterator(); iterator.hasNext(); ) {
      String systemLabel = iterator.next();
      int goldLabelIndex =
          goldList.indexOf(systemLabel); // look for the same label in the gold list

      // if not found, try to find the Null
      if (goldLabelIndex == -1) goldLabelIndex = goldList.indexOf(NULL);

      // if NULL label is not a gold label either, pick the first one
      if (goldLabelIndex == -1 && goldList.size() > 0) goldLabelIndex = 0;
      /*
      else{
      	System.err.println(new IllegalArgumentException(String.format("Ups! No label pair is found. System label: %s Gold labels: %s",
      			systemLabel, goldList)));
      }
      */

      String goldLabel = goldList.get(goldLabelIndex);
      // store the results
      outcome.evaluate(goldLabel, systemLabel);
      // remove labels
      iterator.remove();
      goldList.remove(goldLabelIndex);
    }
  }
Example #4
0
  /**
   * Evaluates one relation and returns a gold relation which corresponds to the specified system
   * relation arguments.
   *
   * @param args - system relation agruments.
   * @param goldRelations - list of all gold relations
   * @param gold - JAXB-representation of the entire document.
   * @param label - label class used for the evaluation (depricated)
   * @param evalOption - evaluation task
   * @param outcome - outcome to store results.
   * @param strict TODO
   */
  private static <T extends WithIdentifier> T evaluateRelation(
      Map<Class<? extends Markable>, List<? extends Markable>> args,
      List<T> goldRelations,
      SpRL gold,
      Class<T> label,
      String evalOption,
      Outcome outcome,
      boolean strict) {

    // List<T> goldRelations = JAXBUtil.getAnnotations(gold, label);

    int maxMatch = Integer.MIN_VALUE;

    T found = null;

    if (goldRelations != null) {
      for (Iterator<T> iterator = goldRelations.iterator(); iterator.hasNext(); ) {
        T r = iterator.next();
        RELATION goldRelation = (RELATION) r;

        // skip if it is an EQ = coreference
        if (goldRelation.getRCC8Value() != null
            && goldRelation.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue;

        Map<Class<? extends Markable>, List<? extends Markable>> goldArgs =
            JAXBUtil.getArgs(goldRelation, gold);

        // matches is a map of labels and matches (true, false) between system and gold annotations.
        // The span and the label have to be the same.
        Map<Class<Markable>, Boolean> matches = JAXBUtil.getMarkableMatches(goldArgs, args, strict);

        int matchesFound =
            JAXBUtil.getCntNonEmptyArgs(
                matches,
                EVAL_MAP.get(evalOption).toArray(new Class[EVAL_MAP.get(evalOption).size()]));

        /*
        // span-based matches
        Map<Class<Markable>, Boolean> spanMatches =
        		JAXBUtil.getSpanMatches(goldArgs, args);

        int spanMatchesFound = JAXBUtil.getCntNonEmptyArgs(spanMatches,
        		EVAL_MAP.get(evalOption).toArray(new Class[EVAL_MAP.get(evalOption).size()]));
        */

        // swap swappable arguments
        Map<Class<? extends Markable>, List<? extends Markable>> swappedArgs =
            swapSymmetricalArgs(args);
        // find matches
        Map<Class<Markable>, Boolean> swappedMatches =
            JAXBUtil.getMarkableMatches(goldArgs, swappedArgs, strict);
        // count matches
        int swappedMatchesFound =
            JAXBUtil.getCntNonEmptyArgs(
                swappedMatches,
                EVAL_MAP.get(evalOption).toArray(new Class[EVAL_MAP.get(evalOption).size()]));

        // take the max
        matchesFound = Math.max(matchesFound, swappedMatchesFound);

        int toMatch = EVAL_MAP.get(evalOption).size();

        if (matchesFound - toMatch > maxMatch) {
          found = r;
          maxMatch = matchesFound - toMatch;
        }

        // if the max match number is found
        if (maxMatch >= 0) break;
      }
    }

    if (maxMatch >= 0 && found != null) {
      // outcome.evaluate(label.getSimpleName(), label.getSimpleName());
      // goldRelations.remove(found);
    } else {
      // outcome.evaluate(NULL, label.getSimpleName());
      found = null;
    }

    return found;
  }
Example #5
0
  /**
   * Evaluates relations.
   *
   * @param gold
   * @param system
   * @param labelClass
   * @param evalOption
   * @param matchLabel TODO
   * @param strictSpan TODO
   * @param outcomes
   */
  private static <T extends WithIdentifier> void evaluateRelations(
      SpRL gold,
      SpRL system,
      Class<T> labelClass,
      String evalOption,
      Outcome outcome,
      boolean matchLabel,
      boolean strictSpan) {

    if (!gold.getTEXT().getContent().equalsIgnoreCase(system.getTEXT().getContent())) {
      if (!gold.getTEXT().getContent().contains(system.getTEXT().getContent()))
        // throw new IllegalArgumentException("The content of the data is not equal.");
        System.err.println(
            system.getFilename()
                + ": The content of the data is not equal. Gold text length: "
                + gold.getTEXT().getContent().length()
                + " System text length:"
                + system.getTEXT().getContent().length()
                + " EVALUATING ANYWAY!");
    }
    // else{

    List<T> goldAnnotations = null;
    List<T> systemAnnotations = null;

    if (labelClass == null) {
      throw new NullPointerException("Annotation class is not defined.");
    } else {
      goldAnnotations = JAXBUtil.getAnnotations(gold, labelClass);
      systemAnnotations = JAXBUtil.getAnnotations(system, labelClass);
    }

    List<T> goldRelations = JAXBUtil.getAnnotations(gold, labelClass); // all gold relations

    String goldLabel = "";
    String systemLabel = "";
    if (systemAnnotations != null) {
      for (Iterator<T> iterator = systemAnnotations.iterator(); iterator.hasNext(); ) {
        T t = iterator.next();
        RELATION systemRelation = (RELATION) t;
        if (systemRelation.getGeneralType() != null
            && systemRelation.getGeneralType().trim().length() == 0)
          System.err.println("ERROR in " + system.getFilename());
        if (systemRelation.getRCC8Value() != null
            && systemRelation.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue;

        Map<Class<? extends Markable>, List<? extends Markable>> args =
            JAXBUtil.getArgs(systemRelation, system);
        RELATION goldMatch =
            (RELATION)
                evaluateRelation(
                    args, goldRelations, gold, labelClass, evalOption, outcome, strictSpan);

        if (evalOption.equalsIgnoreCase(TASK_E)) {
          goldLabel = goldMatch != null ? goldMatch.getGeneralType() : NULL;
          systemLabel = systemRelation.getGeneralType();
        } else {
          goldLabel = labelClass.getSimpleName();
          systemLabel = labelClass.getSimpleName();
        }

        if (systemLabel == null) systemLabel = NULL; // if no rel found

        if (goldMatch != null) {
          outcome.evaluate(
              truncateTo(goldLabel, MAX_LEN),
              truncateTo(systemLabel, MAX_LEN)); // exact match for markables
          goldRelations.remove(goldMatch);
        } else {
          outcome.evaluate(NULL, truncateTo(systemLabel, MAX_LEN));
          goldMatch = null;
        }

        /*
        System.out.println("SYSTEM OUTPUT:\t"+r.toString(args));
        if(goldMatch != null){
        	System.out.println("MATCH FOUND:\t"+goldMatch.toString(JAXBUtil.getArgs(goldMatch, gold)));
        }else
        	System.out.println("NO MATCH FOUND");
        System.out.println();
        */

        iterator.remove();
      }
    }

    if (goldRelations != null) {
      // for the rest of gold annotation which were not matched by markables
      for (Iterator<T> iterator = goldRelations.iterator(); iterator.hasNext(); ) {
        T t = iterator.next();
        RELATION r = (RELATION) t;

        // ignore coreferential relations
        if (r.getRCC8Value() != null && r.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue;

        if (evalOption.equalsIgnoreCase(TASK_E)) {
          goldLabel = r.getGeneralType();
          systemLabel = NULL;
        } else {
          goldLabel = labelClass.getSimpleName();
          systemLabel = NULL;
        }

        outcome.evaluate(truncateTo(goldLabel, MAX_LEN), NULL);

        /*
        System.out.println("SYSTEM OUTPUT:\t"+null);
        System.out.println("GOLD ANN:\t"+r.toString(JAXBUtil.getArgs((RELATION) r, gold)));
        System.out.println();
        */
      }
    }
    // }
  }