public static void main(String[] args) throws IOException {
Map<Integer, String> relID2rel = new HashMap<Integer, String>();
/*
// read relID to rel mapping
Mappings mapping = new Mappings();
mapping.read(mappingFile);
for (Map.Entry<String,Integer> e : mapping.getRel2RelID().entrySet())
relID2rel.put(e.getValue(), e.getKey());
*/
List<Example> predictions = new ArrayList<Example>();
{
BufferedReader r =
new BufferedReader(new InputStreamReader(new FileInputStream(resultsFile), "utf-8"));
String[] h = r.readLine().split(" ");
for (int i = 0; i < h.length; i++) relID2rel.put(i, h[i]);
String l = null;
while ((l = r.readLine()) != null) {
String[] c = l.split("\t");
// final String[] rels = c[3].split(" ");
final String[] scores = c[5].split(" ");
Integer[] pos = new Integer[scores.length];
for (int i = 0; i < pos.length; i++) pos[i] = i;
Arrays.sort(
pos,
new Comparator<Integer>() {
public int compare(Integer i1, Integer i2) {
double d1 = Double.parseDouble(scores[i1]);
double d2 = Double.parseDouble(scores[i2]);
if (d1 > d2) return -1;
else return 1;
}
});
for (int i = 0; i < scores.length; i++) {
Example e = new Example();
e.arg1 = c[0];
e.arg2 = c[1];
e.mentionID = Integer.parseInt(c[2]);
int p = pos[i];
// e.predRelation = rels[p];
e.predRelation = relID2rel.get(p);
// e.predScore = Double.parseDouble(scores[i]) - Double.parseDouble(scores[0]);
e.predScore = Double.parseDouble(scores[p]);
e.rank = i;
// HACK; diff rel - NA
// e.predScore = Double.parseDouble(scores[p]) - Double.parseDouble(scores[0]);
// e.rank = 0;
// if (e.rank == 0)
predictions.add(e);
}
}
r.close();
}
// List<Example> labeled = new ArrayList<Example>();
Map<String, List<Label>> labels = new HashMap<String, List<Label>>();
{
BufferedReader r =
new BufferedReader(new InputStreamReader(new FileInputStream(labelsFile), "utf-8"));
String l = null;
while ((l = r.readLine()) != null) {
String[] c = l.split("\t");
String key = c[6] + "\t" + c[7] + "\t" + c[0]; // arg1, arg2, mentionID
Label label = new Label();
label.relation = c[5];
if (label.relation.equals("/location/country/administrative_divisions")
|| label.relation.equals("/location/administrative_division/country")
|| label.relation.equals("/film/film/featured_film_locations")) continue;
// only measure performance on location/contains
// if (!label.relation.equals("/location/location/contains")) continue;
/*
if (!label.relation.equals("/location/location/contains") &&
!label.relation.equals("/people/person/nationality") &&
!label.relation.equals("/location/neighborhood/neighborhood_of") &&
!label.relation.equals("/people/person/children"))
continue;
*/
// if (!targetsSet.contains(label.relation)) continue;
label.tf = c[2].equals("y") || c[2].equals("indirect");
label.name1 = c[3];
label.name2 = c[4];
label.sentence = c[8];
List<Label> ll = labels.get(key);
if (ll == null) {
ll = new ArrayList<Label>();
labels.put(key, ll);
}
ll.add(label);
}
r.close();
}
// sort predictions by decreasing score
Collections.sort(
predictions,
new Comparator<Example>() {
public int compare(Example e1, Example e2) {
if (e1.rank < e2.rank) return -1;
else if (e1.rank > e2.rank) return 1;
else if (e1.predScore > e2.predScore) return -1;
else return 1;
}
});
// max recall
int MAX_TP = 0;
for (List<Label> ll : labels.values()) {
for (Label l : ll) if (!l.relation.equals("NA") && l.tf) MAX_TP++;
}
List<double[]> curve = new ArrayList<double[]>();
int TP = 0, FP = 0, FN = 0;
double lastPrecision = -1, lastRecall = -1;
for (Example e : predictions) {
String key = e.arg1 + "\t" + e.arg2 + "\t" + e.mentionID;
List<Label> ll = labels.get(key);
if (ll != null) {
for (Label l : ll) {
if (l.relation.equals(e.predRelation)) {
if (l.tf) TP++;
else FP++;
} else {
if (l.tf) FN++; // && e.predRelation.equals("NA")) FN++;
// else TN++;
}
}
double precision = TP / (double) (TP + FP);
double recall = TP / (double) (MAX_TP);
if (precision != lastPrecision || recall != lastRecall) {
curve.add(new double[] {precision, recall});
lastPrecision = precision;
lastRecall = recall;
}
}
}
// print to file
{
BufferedWriter w =
new BufferedWriter(new OutputStreamWriter(new FileOutputStream(curveFile)));
for (double[] d : curve) {
w.write(d[1] + "\t" + d[0] + "\n");
}
w.close();
}
}
