Example #1
0
  public static <TK, FV> double[][] incontextMetricScores(
      NBestListContainer<TK, FV> nbest,
      List<ScoredFeaturizedTranslation<TK, FV>> context,
      EvaluationMetric<TK, FV> emetric) {
    IncrementalEvaluationMetric<TK, FV> incMetric = emetric.getIncrementalMetric();

    for (ScoredFeaturizedTranslation<TK, FV> trans : context) {
      incMetric.add(trans);
    }

    List<List<ScoredFeaturizedTranslation<TK, FV>>> nbestLists = nbest.nbestLists();
    double[][] incontextScores = new double[nbestLists.size()][];

    for (int i = 0; i < incontextScores.length; i++) {
      List<ScoredFeaturizedTranslation<TK, FV>> nbestList = nbestLists.get(i);
      incontextScores[i] = new double[nbestList.size()];
      for (int j = 0; j < incontextScores[i].length; j++) {
        ScoredFeaturizedTranslation<TK, FV> trans = nbestList.get(j);
        incMetric.replace(i, trans);
        incontextScores[i][j] = incMetric.score();
      }
      incMetric.replace(i, context.get(i));
    }
    return incontextScores;
  }
Example #2
0
 public static <TK, FV> double[][] calcDeltaMetric(
     NBestListContainer<TK, FV> nbest,
     List<ScoredFeaturizedTranslation<TK, FV>> base,
     EvaluationMetric<TK, FV> emetric) {
   double baseScore = emetric.score(base);
   double[][] incontextScores = incontextMetricScores(nbest, base, emetric);
   double[][] deltaScores = new double[incontextScores.length][];
   for (int i = 0; i < incontextScores.length; i++) {
     deltaScores[i] = new double[incontextScores[i].length];
     for (int j = 0; j < incontextScores[i].length; j++) {
       deltaScores[i][j] = baseScore - incontextScores[i][j];
     }
   }
   return deltaScores;
 }
  @Override
  public List<ScoredFeaturizedTranslation<TK, FV>> maximize(NBestListContainer<TK, FV> nbest) {

    List<ScoredFeaturizedTranslation<TK, FV>> selected = greedyMetrixMax.maximize(nbest);
    List<List<ScoredFeaturizedTranslation<TK, FV>>> nbestLists = nbest.nbestLists();

    IncrementalEvaluationMetric<TK, FV> incrementalMetric = metric.getIncrementalMetric();
    IncrementalEvaluationMetric<TK, FV> incrementalSubMetric =
        (subMetric != null ? subMetric.getIncrementalMetric() : null);
    for (ScoredFeaturizedTranslation<TK, FV> featurizedTranslation : selected) {
      incrementalMetric.add(featurizedTranslation);
      if (incrementalSubMetric != null) incrementalSubMetric.add(featurizedTranslation);
    }

    int nbestListsSize = nbestLists.size();
    int iter = 0;
    for (int changes = nbestListsSize; changes != 0 && iter < 25; iter++) { // XXX
      changes = 0;
      for (int i = 0; i < nbestListsSize; i++) {
        List<ScoredFeaturizedTranslation<TK, FV>> nbestList = nbestLists.get(i);
        ScoredFeaturizedTranslation<TK, FV> bestFTrans = null;
        double bestScore = Double.NaN;
        double bestScoreSub = Double.NaN;
        // int tI = -1;
        for (ScoredFeaturizedTranslation<TK, FV> ftrans : nbestList) { // tI++;
          incrementalMetric.replace(i, ftrans);
          if (subMetric != null) {
            assert (incrementalSubMetric != null);
            incrementalSubMetric.replace(i, ftrans);
          }
          double score = incrementalMetric.score();
          // System.err.printf("bestScore(%d): %f score: %f \n", bestI,
          // bestScore, score);
          if (bestScore != bestScore || bestScore < score) {
            bestFTrans = ftrans;
            bestScore = score;
            if (subMetric != null) {
              bestScoreSub = incrementalSubMetric.score();
            }
          } else if (bestScore == score && subMetric != null) {
            double subMetricScore = incrementalSubMetric.score();
            if (bestScoreSub < subMetricScore) {
              bestScoreSub = subMetricScore;
              bestFTrans = ftrans;
            }
          }
        }
        incrementalMetric.replace(i, bestFTrans);
        if (incrementalSubMetric != null) incrementalSubMetric.replace(i, bestFTrans);
        if (selected.get(i) != bestFTrans) {
          changes++;
          selected.set(i, bestFTrans);
        }
      }
      if (DEBUG) {
        /*
         * IncrementalEvaluationMetric<TK,FV> iMetric =
         * metric.getIncrementalMetric(); for (ScoredFeaturizedTranslation<TK,
         * FV> trans : selected) { iMetric.add(trans); } if (iMetric.score() !=
         * incrementalMetric.score()) {
         *
         * /*System.err.printf("cnt: %d null cnt: %d\n",
         * ((TERMetric.TERIncrementalMetric)incrementalMetric).cnt,
         * ((TERMetric.TERIncrementalMetric)incrementalMetric).nullCnt);
         * System.err.printf("cnt: %d null cnt: %d\n",
         * ((TERMetric.TERIncrementalMetric)iMetric).cnt,
         * ((TERMetric.TERIncrementalMetric)iMetric).nullCnt); / throw new
         * RuntimeException(String.format("%f!=%f\n", incrementalMetric.score(),
         * iMetric.score())); }
         */

        System.err.printf(
            "%d: score: %.5f changes: %d\n", iter, incrementalMetric.score(), changes);
        /*
         * for (ScoredFeaturizedTranslation<TK,FV> t : selected) {
         * System.err.printf("%s\n", (t == null ? t : t.translation)); }
         */
      }
    }

    return selected;
  }