Пример #1
0
 public Evaluator(RANKER_TYPE rType, String trainMetric, String testMetric) {
   this.type = rType;
   trainScorer = mFact.createScorer(trainMetric);
   testScorer = mFact.createScorer(testMetric);
   if (qrelFile.compareTo("") != 0) {
     trainScorer.loadExternalRelevanceJudgment(qrelFile);
     testScorer.loadExternalRelevanceJudgment(qrelFile);
   }
 }
Пример #2
0
  /**
   * Evaluate the currently selected ranking algorithm using percenTrain% of the training samples
   * for training the rest as validation data. Test data is specified separately.
   *
   * @param trainFile
   * @param percentTrain
   * @param testFile Empty string for "no test data"
   * @param featureDefFile
   */
  public void evaluate(
      String trainFile, double percentTrain, String testFile, String featureDefFile) {
    List<RankList> train = new ArrayList<RankList>();
    List<RankList> validation = new ArrayList<RankList>();
    int[] features =
        prepareSplit(trainFile, featureDefFile, percentTrain, normalize, train, validation);
    List<RankList> test = null;
    if (testFile.compareTo("") != 0) test = readInput(testFile);

    Ranker ranker = rFact.createRanker(type, train, features);
    ranker.set(trainScorer);
    ranker.setValidationSet(validation);
    ranker.init();
    ranker.learn();

    if (test != null) {
      double rankScore = evaluate(ranker, test);
      System.out.println(testScorer.name() + " on test data: " + SimpleMath.round(rankScore, 4));
    }
    if (modelFile.compareTo("") != 0) {
      System.out.println("");
      ranker.save(modelFile);
      System.out.println("Model saved to: " + modelFile);
    }
  }
Пример #3
0
  /**
   * Evaluate the currently selected ranking algorithm using <training data, validation data,
   * testing data and the defined features>.
   *
   * @param trainFile
   * @param validationFile
   * @param testFile
   * @param featureDefFile
   */
  public void evaluate(
      String trainFile, String validationFile, String testFile, String featureDefFile) {
    List<RankList> train = readInput(trainFile); // read input
    List<RankList> validation = null;
    if (validationFile.compareTo("") != 0) validation = readInput(validationFile);
    List<RankList> test = null;
    if (testFile.compareTo("") != 0) test = readInput(testFile);
    int[] features = readFeature(featureDefFile); // read features
    if (features == null) // no features specified ==> use all features in the training file
    features = getFeatureFromSampleVector(train);

    if (normalize) {
      normalize(train, features);
      if (validation != null) normalize(validation, features);
      if (test != null) normalize(test, features);
    }
    /*if(newFeatureFile.compareTo("")!=0)
    {
    	System.out.print("Loading new feature description file... ");
    	List<String> descriptions = FileUtils.readLine(newFeatureFile, "ASCII");
    	int taken = 0;
    	for(int i=0;i<descriptions.size();i++)
    	{
    		if(descriptions.get(i).indexOf("##")==0)
    			continue;
    		LinearComputer lc = new LinearComputer("", descriptions.get(i));
    		//if we keep the orig. features ==> discard size-1 linear computer
    		if(!keepOrigFeatures || lc.size()>1)
    		{
    			lcList.add(lc);
    			taken++;
    			if(taken == topNew)
    				break;
    		}
    		//System.out.println(lc.toString());
    	}
    	applyNewFeatures(train, features);
    	applyNewFeatures(validation, features);
    	features = applyNewFeatures(test, features);
    	System.out.println("[Done]");//0.1195
    								//0.071
    }*/

    Ranker ranker = rFact.createRanker(type, train, features);
    ranker.set(trainScorer);
    ranker.setValidationSet(validation);
    ranker.init();
    ranker.learn();

    if (test != null) {
      double rankScore = evaluate(ranker, test);
      System.out.println(testScorer.name() + " on test data: " + SimpleMath.round(rankScore, 4));
    }
    if (modelFile.compareTo("") != 0) {
      System.out.println("");
      ranker.save(modelFile);
      System.out.println("Model saved to: " + modelFile);
    }
  }
Пример #4
0
  public void test(String modelFile, String testFile) {
    Ranker ranker = rFact.loadRanker(modelFile);
    int[] features = ranker.getFeatures();
    List<RankList> test = readInput(testFile);
    if (normalize) normalize(test, features);

    double rankScore = evaluate(ranker, test);
    System.out.println(testScorer.name() + " on test data: " + SimpleMath.round(rankScore, 4));
  }
Пример #5
0
  public void test(String modelFile, String testFile, boolean printIndividual) {
    Ranker ranker = rFact.loadRanker(modelFile);
    int[] features = ranker.getFeatures();
    List<RankList> test = readInput(testFile);
    if (normalize) normalize(test, features);

    double rankScore = 0.0;
    double score = 0.0;
    for (int i = 0; i < test.size(); i++) {
      RankList l = ranker.rank(test.get(i));
      score = testScorer.score(l);
      if (printIndividual)
        System.out.println(
            testScorer.name() + "   " + l.getID() + "   " + SimpleMath.round(score, 4));
      rankScore += score;
    }
    rankScore /= test.size();
    if (printIndividual)
      System.out.println(testScorer.name() + "   all   " + SimpleMath.round(rankScore, 4));
    else System.out.println(testScorer.name() + " on test data: " + SimpleMath.round(rankScore, 4));
  }
Пример #6
0
 public void test(String testFile) {
   List<RankList> test = readInput(testFile);
   double rankScore = evaluate(null, test);
   System.out.println(testScorer.name() + " on test data: " + SimpleMath.round(rankScore, 4));
 }
Пример #7
0
 public double evaluate(Ranker ranker, List<RankList> rl) {
   List<RankList> l = rl;
   if (ranker != null) l = ranker.rank(rl);
   return testScorer.score(l);
 }
Пример #8
0
 public Evaluator(RANKER_TYPE rType, METRIC metric, int k) {
   this.type = rType;
   trainScorer = mFact.createScorer(metric, k);
   if (qrelFile.compareTo("") != 0) trainScorer.loadExternalRelevanceJudgment(qrelFile);
   testScorer = trainScorer;
 }