@Override
public void close() {
System.err.println(
"Target: " + vocE.size() + " types. Writing to " + job_.get("root", null) + "/vocab.E");
System.err.println(
"Source: " + vocF.size() + " types .Writing to " + job_.get("root", null) + "/vocab.F");
// write out vocabulary to file
try {
FileSystem fs = FileSystem.get(job_);
DataOutputStream dos =
new DataOutputStream(
new BufferedOutputStream(fs.create(new Path(job_.get("root", null) + "/vocab.E"))));
((VocabularyWritable) vocE).write(dos);
dos.close();
DataOutputStream dos2 =
new DataOutputStream(
new BufferedOutputStream(fs.create(new Path(job_.get("root", null) + "/vocab.F"))));
((VocabularyWritable) vocF).write(dos2);
dos2.close();
} catch (IOException e) {
throw new RuntimeException("Vocab couldn't be written to disk.\n" + e.toString());
}
}
public int[] convertStrings(String[] s, Vocab v) {
int[] res = new int[s.length];
for (int i = 0; i < s.length; ++i) {
res[i] = v.addOrGet(s[i]);
sLogger.info(s[i] + "-->" + res[i]);
}
return res;
}
