public static ArrayList<TaggedWord> StopWordRemoval(ArrayList<TaggedWord> taggedWords) {
ArrayList<TaggedWord> newList = new ArrayList<TaggedWord>();
try {
String path = "data/nltk_stoplist.txt";
File textFile = new File(path);
BufferedReader br = new BufferedReader(new FileReader(textFile));
String stopwordsLine = br.readLine();
br.close();
String[] stopwords = stopwordsLine.split(",");
HashMap<String, String> stopwordsDict = new HashMap<String, String>();
for (int i = 0; i < stopwords.length; i++) {
stopwordsDict.put(stopwords[i], stopwords[i]);
}
for (int i = 0; i < taggedWords.size(); i++) {
String word = taggedWords.get(i).word();
String posTag = taggedWords.get(i).tag();
if (!stopwordsDict.containsKey(word.toLowerCase())) {
String newWord, newPosTag;
newWord = word;
newPosTag = posTag;
newList.add(new TaggedWord(newWord, newPosTag));
}
}
} catch (Exception ex) {
ex.printStackTrace();
}
return newList;
}
public static double LexicalSimilarityScoreMin(
ArrayList<TaggedWord> taggedWords1,
ArrayList<TaggedWord> taggedWords2,
DISCOSimilarity discoRAM,
LexicalizedParser lp) {
// System.out.println(taggedWords1.size() + "," + taggedWords2.size());
// array of edge weights with default weight 0
int length1 = taggedWords1.size();
int length2 = taggedWords2.size();
int arrSize = Math.max(length1, length2);
double[][] array = new double[arrSize][arrSize];
for (int i = 0; i < arrSize; i++) {
for (int j = 0; j < arrSize; j++) {
array[i][j] = 0;
}
}
for (int i = 0; i < length1; i++) {
for (int j = 0; j < length2; j++) {
String word1 = taggedWords1.get(i).word();
String word2 = taggedWords2.get(j).word();
double edgeWeight = 0;
// LSA Similarity
// edgeWeight = LSASimilarity.LSAWordSimilarity(word1, word2);
// DISCO Similarity
// DISCOSimilarity discoObj = new DISCOSimilarity();
try {
if (word1.compareToIgnoreCase(word2) == 0) edgeWeight = 1;
else {
edgeWeight = discoRAM.similarity2(word1, word2);
// edgeWeight = LSASimilarity.LSAWordSimilarity(word1, word2);
}
} catch (Exception ex) {
ex.printStackTrace();
}
array[i][j] = edgeWeight;
}
}
// System.out.println("Hungarian starts " + arrSize);
double finalScore;
String sumType = "max";
int minLength = Math.min(length1, length2);
finalScore = HungarianAlgorithm.hgAlgorithm(array, sumType) / minLength * 5;
// finalScore = HungarianAlgorithm.hgAlgorithm(array, sumType)/arrSize * 5;
return finalScore;
}
public static double LexicalSimilarityScoreWordNet(
String sentence1, String sentence2, LeskWSD tm, LexicalizedParser lp, WordNetSimilarity ws) {
ArrayList<TaggedWord> taggedWordsPrev1 = Preprocess(StanfordParse(sentence1, lp));
ArrayList<TaggedWord> taggedWordsPrev2 = Preprocess(StanfordParse(sentence2, lp));
ArrayList<TaggedWord> taggedWords1 = new ArrayList<TaggedWord>();
ArrayList<TaggedWord> taggedWords2 = new ArrayList<TaggedWord>();
WordNetSense[] sensesPrev1 = tm.LeskJWI(sentence1);
WordNetSense[] sensesPrev2 = tm.LeskJWI(sentence2);
// System.out.println("Senses found!");
ArrayList<WordNetSense> senses1 = new ArrayList<WordNetSense>();
ArrayList<WordNetSense> senses2 = new ArrayList<WordNetSense>();
for (int i = 0; i < taggedWordsPrev1.size(); i++) {
String word = taggedWordsPrev1.get(i).word();
String posTag = taggedWordsPrev1.get(i).tag();
if (posTag.length() >= 2 && posTag.substring(0, 2).equals("NN")) {
taggedWords1.add(new TaggedWord(word, "NN"));
senses1.add(sensesPrev1[i]);
} else if (posTag.length() >= 2 && posTag.substring(0, 2).equals("VB")) {
taggedWords1.add(new TaggedWord(word, "VB"));
senses1.add(sensesPrev1[i]);
}
}
for (int i = 0; i < taggedWordsPrev2.size(); i++) {
String word = taggedWordsPrev2.get(i).word();
String posTag = taggedWordsPrev2.get(i).tag();
if (posTag.length() >= 2 && posTag.substring(0, 2).equals("NN")) {
taggedWords2.add(new TaggedWord(word, "NN"));
senses2.add(sensesPrev2[i]);
} else if (posTag.length() >= 2 && posTag.substring(0, 2).equals("VB")) {
taggedWords2.add(new TaggedWord(word, "VB"));
senses2.add(sensesPrev2[i]);
}
}
// System.out.println(taggedWords1.size() + "," + taggedWords2.size());
// array of edge weights with default weight 0
int length1 = taggedWords1.size();
int length2 = taggedWords2.size();
int arrSize = Math.max(length1, length2);
double[][] array = new double[arrSize][arrSize];
for (int i = 0; i < arrSize; i++) {
for (int j = 0; j < arrSize; j++) {
array[i][j] = 0;
}
}
for (int i = 0; i < length1; i++) {
for (int j = 0; j < length2; j++) {
String word1 = taggedWords1.get(i).word();
String posTag1 = taggedWords1.get(i).tag();
String word2 = taggedWords2.get(j).word();
String posTag2 = taggedWords2.get(j).tag();
double edgeWeight = 0;
// LSA Similarity
// edgeWeight = LSASimilarity.LSAWordSimilarity(word1, word2);
// DISCO Similarity
// DISCOSimilarity discoObj = new DISCOSimilarity();
try {
if (word1.compareToIgnoreCase(word2) == 0) edgeWeight = 1;
else {
// edgeWeight = ws.wuPalmerSimilarity(senses1.get(i), senses2.get(j));
edgeWeight = ws.linSimilarity(senses1.get(i), senses2.get(j));
}
} catch (Exception ex) {
ex.printStackTrace();
}
array[i][j] = edgeWeight;
}
}
// System.out.println("Hungarian starts " + arrSize);
double finalScore;
String sumType = "max";
int minLength = Math.min(length1, length2);
// finalScore = HungarianAlgorithm.hgAlgorithm(array, sumType)/minLength * 5;
if (arrSize == 0) finalScore = 0;
else finalScore = HungarianAlgorithm.hgAlgorithm(array, sumType) / arrSize * 5;
return finalScore;
}
/**
* for testing -- CURRENTLY BROKEN!!!
*
* @param args input dir and output filename
* @throws IOException
*/
public static void main(String[] args) throws IOException {
if (args.length != 3) {
throw new RuntimeException("args: treebankPath trainNums testNums");
}
ChineseTreebankParserParams ctpp = new ChineseTreebankParserParams();
ctpp.charTags = true;
// TODO: these options are getting clobbered by reading in the
// parser object (unless it's a text file parser?)
Options op = new Options(ctpp);
op.doDep = false;
op.testOptions.maxLength = 90;
LexicalizedParser lp;
try {
FileFilter trainFilt = new NumberRangesFileFilter(args[1], false);
lp = LexicalizedParser.trainFromTreebank(args[0], trainFilt, op);
try {
String filename = "chineseCharTagPCFG.ser.gz";
System.err.println("Writing parser in serialized format to file " + filename + ' ');
System.err.flush();
ObjectOutputStream out = IOUtils.writeStreamFromString(filename);
out.writeObject(lp);
out.close();
System.err.println("done.");
} catch (IOException ioe) {
ioe.printStackTrace();
}
} catch (IllegalArgumentException e) {
lp = LexicalizedParser.loadModel(args[1], op);
}
FileFilter testFilt = new NumberRangesFileFilter(args[2], false);
MemoryTreebank testTreebank = ctpp.memoryTreebank();
testTreebank.loadPath(new File(args[0]), testFilt);
PrintWriter pw =
new PrintWriter(new OutputStreamWriter(new FileOutputStream("out.chi"), "GB18030"), true);
WordCatEquivalenceClasser eqclass = new WordCatEquivalenceClasser();
WordCatEqualityChecker eqcheck = new WordCatEqualityChecker();
EquivalenceClassEval eval = new EquivalenceClassEval(eqclass, eqcheck);
// System.out.println("Preterminals:" + preterminals);
System.out.println("Testing...");
for (Tree gold : testTreebank) {
Tree tree;
try {
tree = lp.parseTree(gold.yieldHasWord());
if (tree == null) {
System.out.println("Failed to parse " + gold.yieldHasWord());
continue;
}
} catch (Exception e) {
e.printStackTrace();
continue;
}
gold = gold.firstChild();
pw.println(Sentence.listToString(gold.preTerminalYield()));
pw.println(Sentence.listToString(gold.yield()));
gold.pennPrint(pw);
pw.println(tree.preTerminalYield());
pw.println(tree.yield());
tree.pennPrint(pw);
// Collection allBrackets = WordCatConstituent.allBrackets(tree);
// Collection goldBrackets = WordCatConstituent.allBrackets(gold);
// eval.eval(allBrackets, goldBrackets);
eval.displayLast();
}
System.out.println();
System.out.println();
eval.display();
}