public static ArrayList<TaggedWord> StopWordRemoval(ArrayList<TaggedWord> taggedWords) {
ArrayList<TaggedWord> newList = new ArrayList<TaggedWord>();
try {
String path = "data/nltk_stoplist.txt";
File textFile = new File(path);
BufferedReader br = new BufferedReader(new FileReader(textFile));
String stopwordsLine = br.readLine();
br.close();
String[] stopwords = stopwordsLine.split(",");
HashMap<String, String> stopwordsDict = new HashMap<String, String>();
for (int i = 0; i < stopwords.length; i++) {
stopwordsDict.put(stopwords[i], stopwords[i]);
}
for (int i = 0; i < taggedWords.size(); i++) {
String word = taggedWords.get(i).word();
String posTag = taggedWords.get(i).tag();
if (!stopwordsDict.containsKey(word.toLowerCase())) {
String newWord, newPosTag;
newWord = word;
newPosTag = posTag;
newList.add(new TaggedWord(newWord, newPosTag));
}
}
} catch (Exception ex) {
ex.printStackTrace();
}
return newList;
}
public static double LexicalSimilarityScoreMin(
ArrayList<TaggedWord> taggedWords1,
ArrayList<TaggedWord> taggedWords2,
DISCOSimilarity discoRAM,
LexicalizedParser lp) {
// System.out.println(taggedWords1.size() + "," + taggedWords2.size());
// array of edge weights with default weight 0
int length1 = taggedWords1.size();
int length2 = taggedWords2.size();
int arrSize = Math.max(length1, length2);
double[][] array = new double[arrSize][arrSize];
for (int i = 0; i < arrSize; i++) {
for (int j = 0; j < arrSize; j++) {
array[i][j] = 0;
}
}
for (int i = 0; i < length1; i++) {
for (int j = 0; j < length2; j++) {
String word1 = taggedWords1.get(i).word();
String word2 = taggedWords2.get(j).word();
double edgeWeight = 0;
// LSA Similarity
// edgeWeight = LSASimilarity.LSAWordSimilarity(word1, word2);
// DISCO Similarity
// DISCOSimilarity discoObj = new DISCOSimilarity();
try {
if (word1.compareToIgnoreCase(word2) == 0) edgeWeight = 1;
else {
edgeWeight = discoRAM.similarity2(word1, word2);
// edgeWeight = LSASimilarity.LSAWordSimilarity(word1, word2);
}
} catch (Exception ex) {
ex.printStackTrace();
}
array[i][j] = edgeWeight;
}
}
// System.out.println("Hungarian starts " + arrSize);
double finalScore;
String sumType = "max";
int minLength = Math.min(length1, length2);
finalScore = HungarianAlgorithm.hgAlgorithm(array, sumType) / minLength * 5;
// finalScore = HungarianAlgorithm.hgAlgorithm(array, sumType)/arrSize * 5;
return finalScore;
}
public static double LexicalSimilarityScoreWordNet(
String sentence1, String sentence2, LeskWSD tm, LexicalizedParser lp, WordNetSimilarity ws) {
ArrayList<TaggedWord> taggedWordsPrev1 = Preprocess(StanfordParse(sentence1, lp));
ArrayList<TaggedWord> taggedWordsPrev2 = Preprocess(StanfordParse(sentence2, lp));
ArrayList<TaggedWord> taggedWords1 = new ArrayList<TaggedWord>();
ArrayList<TaggedWord> taggedWords2 = new ArrayList<TaggedWord>();
WordNetSense[] sensesPrev1 = tm.LeskJWI(sentence1);
WordNetSense[] sensesPrev2 = tm.LeskJWI(sentence2);
// System.out.println("Senses found!");
ArrayList<WordNetSense> senses1 = new ArrayList<WordNetSense>();
ArrayList<WordNetSense> senses2 = new ArrayList<WordNetSense>();
for (int i = 0; i < taggedWordsPrev1.size(); i++) {
String word = taggedWordsPrev1.get(i).word();
String posTag = taggedWordsPrev1.get(i).tag();
if (posTag.length() >= 2 && posTag.substring(0, 2).equals("NN")) {
taggedWords1.add(new TaggedWord(word, "NN"));
senses1.add(sensesPrev1[i]);
} else if (posTag.length() >= 2 && posTag.substring(0, 2).equals("VB")) {
taggedWords1.add(new TaggedWord(word, "VB"));
senses1.add(sensesPrev1[i]);
}
}
for (int i = 0; i < taggedWordsPrev2.size(); i++) {
String word = taggedWordsPrev2.get(i).word();
String posTag = taggedWordsPrev2.get(i).tag();
if (posTag.length() >= 2 && posTag.substring(0, 2).equals("NN")) {
taggedWords2.add(new TaggedWord(word, "NN"));
senses2.add(sensesPrev2[i]);
} else if (posTag.length() >= 2 && posTag.substring(0, 2).equals("VB")) {
taggedWords2.add(new TaggedWord(word, "VB"));
senses2.add(sensesPrev2[i]);
}
}
// System.out.println(taggedWords1.size() + "," + taggedWords2.size());
// array of edge weights with default weight 0
int length1 = taggedWords1.size();
int length2 = taggedWords2.size();
int arrSize = Math.max(length1, length2);
double[][] array = new double[arrSize][arrSize];
for (int i = 0; i < arrSize; i++) {
for (int j = 0; j < arrSize; j++) {
array[i][j] = 0;
}
}
for (int i = 0; i < length1; i++) {
for (int j = 0; j < length2; j++) {
String word1 = taggedWords1.get(i).word();
String posTag1 = taggedWords1.get(i).tag();
String word2 = taggedWords2.get(j).word();
String posTag2 = taggedWords2.get(j).tag();
double edgeWeight = 0;
// LSA Similarity
// edgeWeight = LSASimilarity.LSAWordSimilarity(word1, word2);
// DISCO Similarity
// DISCOSimilarity discoObj = new DISCOSimilarity();
try {
if (word1.compareToIgnoreCase(word2) == 0) edgeWeight = 1;
else {
// edgeWeight = ws.wuPalmerSimilarity(senses1.get(i), senses2.get(j));
edgeWeight = ws.linSimilarity(senses1.get(i), senses2.get(j));
}
} catch (Exception ex) {
ex.printStackTrace();
}
array[i][j] = edgeWeight;
}
}
// System.out.println("Hungarian starts " + arrSize);
double finalScore;
String sumType = "max";
int minLength = Math.min(length1, length2);
// finalScore = HungarianAlgorithm.hgAlgorithm(array, sumType)/minLength * 5;
if (arrSize == 0) finalScore = 0;
else finalScore = HungarianAlgorithm.hgAlgorithm(array, sumType) / arrSize * 5;
return finalScore;
}