@Before
public void setUp() throws Exception {
task = new NemaTask();
task.setId(-1);
task.setName("test name");
task.setDescription("test description");
task.setDatasetId(-1);
task.setSubjectTrackMetadataId(-1);
task.setSubjectTrackMetadataName(NemaDataConstants.CHORD_LABEL_SEQUENCE);
dataset = new NemaDataset();
dataset.setId(task.getDatasetId());
dataset.setName("dataset name");
dataset.setDescription("some description");
// read ground-truth here as we'll use it a few times
File groundTruthDirectory = new File("src/test/resources/chord/groundtruth");
SingleTrackEvalFileType reader = new ChordShortHandTextFile();
groundTruth = reader.readDirectory(groundTruthDirectory, null);
ArrayList<NemaTrack> trackList = new ArrayList<NemaTrack>(groundTruth.size());
for (Iterator<NemaData> iterator = groundTruth.iterator(); iterator.hasNext(); ) {
trackList.add(new NemaTrack(iterator.next().getId()));
}
testSets = new ArrayList<NemaTrackList>(1);
int id = 0;
testSets.add(new NemaTrackList(id, task.getDatasetId(), 3, "test", id, trackList));
id++;
}
@Test
public void testEvaluateShortHandBasedSystem()
throws FileNotFoundException, IOException, IllegalArgumentException, IOException,
InstantiationException, IllegalAccessException {
File resultsDirectory = new File("src/test/resources/chord/CH");
String systemName = "CH-System";
Evaluator evaluator = null;
// test reader and setup for evaluation
// evaluator = new ChordEvaluator(task, dataset, outputDirectory, workingDirectory, testSets,
// false, null);
evaluator =
EvaluatorFactory.getEvaluator(
task.getSubjectTrackMetadataName(),
task,
dataset,
outputDirectory,
workingDirectory,
null,
testSets,
false,
null);
SingleTrackEvalFileType reader = new ChordShortHandTextFile();
evaluator.setGroundTruth(groundTruth);
List<NemaData> resultsForAllTracks = reader.readDirectory(resultsDirectory, null);
evaluator.addResults(systemName, systemName, testSets.get(0), resultsForAllTracks);
// test evaluation
NemaEvaluationResultSet results = evaluator.evaluate();
assertTrue(results != null);
}