Example #1
0
  /**
   * Evaluates one relation and returns a gold relation which corresponds to the specified system
   * relation arguments.
   *
   * @param args - system relation agruments.
   * @param goldRelations - list of all gold relations
   * @param gold - JAXB-representation of the entire document.
   * @param label - label class used for the evaluation (depricated)
   * @param evalOption - evaluation task
   * @param outcome - outcome to store results.
   * @param strict TODO
   */
  private static <T extends WithIdentifier> T evaluateRelation(
      Map<Class<? extends Markable>, List<? extends Markable>> args,
      List<T> goldRelations,
      SpRL gold,
      Class<T> label,
      String evalOption,
      Outcome outcome,
      boolean strict) {

    // List<T> goldRelations = JAXBUtil.getAnnotations(gold, label);

    int maxMatch = Integer.MIN_VALUE;

    T found = null;

    if (goldRelations != null) {
      for (Iterator<T> iterator = goldRelations.iterator(); iterator.hasNext(); ) {
        T r = iterator.next();
        RELATION goldRelation = (RELATION) r;

        // skip if it is an EQ = coreference
        if (goldRelation.getRCC8Value() != null
            && goldRelation.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue;

        Map<Class<? extends Markable>, List<? extends Markable>> goldArgs =
            JAXBUtil.getArgs(goldRelation, gold);

        // matches is a map of labels and matches (true, false) between system and gold annotations.
        // The span and the label have to be the same.
        Map<Class<Markable>, Boolean> matches = JAXBUtil.getMarkableMatches(goldArgs, args, strict);

        int matchesFound =
            JAXBUtil.getCntNonEmptyArgs(
                matches,
                EVAL_MAP.get(evalOption).toArray(new Class[EVAL_MAP.get(evalOption).size()]));

        /*
        // span-based matches
        Map<Class<Markable>, Boolean> spanMatches =
        		JAXBUtil.getSpanMatches(goldArgs, args);

        int spanMatchesFound = JAXBUtil.getCntNonEmptyArgs(spanMatches,
        		EVAL_MAP.get(evalOption).toArray(new Class[EVAL_MAP.get(evalOption).size()]));
        */

        // swap swappable arguments
        Map<Class<? extends Markable>, List<? extends Markable>> swappedArgs =
            swapSymmetricalArgs(args);
        // find matches
        Map<Class<Markable>, Boolean> swappedMatches =
            JAXBUtil.getMarkableMatches(goldArgs, swappedArgs, strict);
        // count matches
        int swappedMatchesFound =
            JAXBUtil.getCntNonEmptyArgs(
                swappedMatches,
                EVAL_MAP.get(evalOption).toArray(new Class[EVAL_MAP.get(evalOption).size()]));

        // take the max
        matchesFound = Math.max(matchesFound, swappedMatchesFound);

        int toMatch = EVAL_MAP.get(evalOption).size();

        if (matchesFound - toMatch > maxMatch) {
          found = r;
          maxMatch = matchesFound - toMatch;
        }

        // if the max match number is found
        if (maxMatch >= 0) break;
      }
    }

    if (maxMatch >= 0 && found != null) {
      // outcome.evaluate(label.getSimpleName(), label.getSimpleName());
      // goldRelations.remove(found);
    } else {
      // outcome.evaluate(NULL, label.getSimpleName());
      found = null;
    }

    return found;
  }
Example #2
0
  /**
   * Evaluates relations.
   *
   * @param gold
   * @param system
   * @param labelClass
   * @param evalOption
   * @param matchLabel TODO
   * @param strictSpan TODO
   * @param outcomes
   */
  private static <T extends WithIdentifier> void evaluateRelations(
      SpRL gold,
      SpRL system,
      Class<T> labelClass,
      String evalOption,
      Outcome outcome,
      boolean matchLabel,
      boolean strictSpan) {

    if (!gold.getTEXT().getContent().equalsIgnoreCase(system.getTEXT().getContent())) {
      if (!gold.getTEXT().getContent().contains(system.getTEXT().getContent()))
        // throw new IllegalArgumentException("The content of the data is not equal.");
        System.err.println(
            system.getFilename()
                + ": The content of the data is not equal. Gold text length: "
                + gold.getTEXT().getContent().length()
                + " System text length:"
                + system.getTEXT().getContent().length()
                + " EVALUATING ANYWAY!");
    }
    // else{

    List<T> goldAnnotations = null;
    List<T> systemAnnotations = null;

    if (labelClass == null) {
      throw new NullPointerException("Annotation class is not defined.");
    } else {
      goldAnnotations = JAXBUtil.getAnnotations(gold, labelClass);
      systemAnnotations = JAXBUtil.getAnnotations(system, labelClass);
    }

    List<T> goldRelations = JAXBUtil.getAnnotations(gold, labelClass); // all gold relations

    String goldLabel = "";
    String systemLabel = "";
    if (systemAnnotations != null) {
      for (Iterator<T> iterator = systemAnnotations.iterator(); iterator.hasNext(); ) {
        T t = iterator.next();
        RELATION systemRelation = (RELATION) t;
        if (systemRelation.getGeneralType() != null
            && systemRelation.getGeneralType().trim().length() == 0)
          System.err.println("ERROR in " + system.getFilename());
        if (systemRelation.getRCC8Value() != null
            && systemRelation.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue;

        Map<Class<? extends Markable>, List<? extends Markable>> args =
            JAXBUtil.getArgs(systemRelation, system);
        RELATION goldMatch =
            (RELATION)
                evaluateRelation(
                    args, goldRelations, gold, labelClass, evalOption, outcome, strictSpan);

        if (evalOption.equalsIgnoreCase(TASK_E)) {
          goldLabel = goldMatch != null ? goldMatch.getGeneralType() : NULL;
          systemLabel = systemRelation.getGeneralType();
        } else {
          goldLabel = labelClass.getSimpleName();
          systemLabel = labelClass.getSimpleName();
        }

        if (systemLabel == null) systemLabel = NULL; // if no rel found

        if (goldMatch != null) {
          outcome.evaluate(
              truncateTo(goldLabel, MAX_LEN),
              truncateTo(systemLabel, MAX_LEN)); // exact match for markables
          goldRelations.remove(goldMatch);
        } else {
          outcome.evaluate(NULL, truncateTo(systemLabel, MAX_LEN));
          goldMatch = null;
        }

        /*
        System.out.println("SYSTEM OUTPUT:\t"+r.toString(args));
        if(goldMatch != null){
        	System.out.println("MATCH FOUND:\t"+goldMatch.toString(JAXBUtil.getArgs(goldMatch, gold)));
        }else
        	System.out.println("NO MATCH FOUND");
        System.out.println();
        */

        iterator.remove();
      }
    }

    if (goldRelations != null) {
      // for the rest of gold annotation which were not matched by markables
      for (Iterator<T> iterator = goldRelations.iterator(); iterator.hasNext(); ) {
        T t = iterator.next();
        RELATION r = (RELATION) t;

        // ignore coreferential relations
        if (r.getRCC8Value() != null && r.getRCC8Value().equalsIgnoreCase(RCC8_EQUAL)) continue;

        if (evalOption.equalsIgnoreCase(TASK_E)) {
          goldLabel = r.getGeneralType();
          systemLabel = NULL;
        } else {
          goldLabel = labelClass.getSimpleName();
          systemLabel = NULL;
        }

        outcome.evaluate(truncateTo(goldLabel, MAX_LEN), NULL);

        /*
        System.out.println("SYSTEM OUTPUT:\t"+null);
        System.out.println("GOLD ANN:\t"+r.toString(JAXBUtil.getArgs((RELATION) r, gold)));
        System.out.println();
        */
      }
    }
    // }
  }