public class Parser {
private String grammar = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz";
private String[] options = {"-maxLength", "80", "-retainTmpSubcategories"};
private LexicalizedParser lp = LexicalizedParser.loadModel(grammar, options);
private TreebankLanguagePack tlp = lp.getOp().langpack();
private GrammaticalStructureFactory gsf = tlp.grammaticalStructureFactory();
public Parser() {}
public LinkedList<String> getKeyWrodsFromSentence(String string) {
LinkedList<String> list = new LinkedList<String>();
String[] sent = string.split(" ");
List<HasWord> sentence = new ArrayList<HasWord>();
for (String word : sent) sentence.add(new Word(word));
Tree parse = lp.parse(sentence);
GrammaticalStructure gs = gsf.newGrammaticalStructure(parse);
List<TypedDependency> tdl = gs.typedDependenciesCCprocessed();
String[] current;
String type, key;
List<CoreLabel> labelsList = parse.taggedLabeledYield();
for (Label l : labelsList) {
current = l.toString().split("-");
type = current[0];
if (type.equals("NN") || type.equals("NNS")) {
key = sent[Integer.parseInt(current[1])];
list.add(key);
}
}
return list;
}
public LinkedList<String> getKeyWrodsFromSentenceTest(String string) {
LinkedList<String> list = new LinkedList<String>();
String[] sent = string.split(" ");
List<HasWord> sentence = new ArrayList<HasWord>();
for (String word : sent) {
sentence.add(new Word(word));
}
Tree parse = lp.parse(sentence);
parse.pennPrint();
GrammaticalStructure gs = gsf.newGrammaticalStructure(parse);
List<TypedDependency> tdl = gs.typedDependenciesCCprocessed();
System.out.println(tdl);
System.out.println();
System.out.println("The words of the sentence:");
for (Label lab : parse.yield()) {
if (lab instanceof CoreLabel) {
System.out.println(((CoreLabel) lab).toString(CoreLabel.OutputFormat.VALUE_MAP));
} else {
System.out.println(lab);
}
}
System.out.println();
System.out.println("tagged");
System.out.println(parse.taggedYield());
List<CoreLabel> temp = parse.taggedLabeledYield();
for (Label l : temp) {
String[] sss = l.toString().split("-");
String type = sss[0];
System.out.println(sss[0] + " " + sss[1] + " " + sent[Integer.parseInt(sss[1])]);
}
for (Iterator<String> ite = list.iterator(); ite.hasNext(); ) System.out.println(ite.next());
return list;
}
public static void main(String[] args) throws IOException {
Parser parser = new Parser();
parser.getKeyWrodsFromSentence(
"When athletes begin to exercise, their heart rates and respiration rates increase. At what level of organization does the human body coordinate these functions?");
parser.getKeyWrodsFromSentenceTest(
"When athletes begin to exercise, their heart rates and respiration rates increase. At what level of organization does the human body coordinate these functions?");
// main2();
}
}
/**
* parse sentence and generate .trees file
*
* @param en
* @param align
* @param out
*/
public static void parse(String en, String align, String out, boolean verbose) {
// use alignments?
boolean use_alignments = true;
if (align.startsWith("no_align")) {
use_alignments = false;
System.err.println("Not using alignments.");
} else {
System.err.println("Using alignments from " + align);
}
// setup stanfordparser
String grammar = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz";
String[] options = {"-outputFormat", "wordsAndTags, typedDependencies"};
LexicalizedParser lp = LexicalizedParser.loadModel(grammar, options);
TreebankLanguagePack tlp = lp.getOp().langpack();
java.util.function.Predicate<java.lang.String> punctuationFilter = x -> true;
GrammaticalStructureFactory gsf =
new edu.stanford.nlp.trees.EnglishGrammaticalStructureFactory(punctuationFilter);
// read document
Iterable<List<? extends HasWord>> sentences;
Reader r = new Reader(en);
String line = null;
List<List<? extends HasWord>> tmp = new ArrayList<List<? extends HasWord>>();
while ((line = r.getNext()) != null) {
Tokenizer<? extends HasWord> token =
tlp.getTokenizerFactory().getTokenizer(new StringReader(line));
List<? extends HasWord> sentence = token.tokenize();
tmp.add(sentence);
}
sentences = tmp;
// set up alignment file reader
Reader alignment = new Reader();
if (use_alignments) {
alignment = new Reader(align);
}
// set up tree file writer
Writer treeWriter = new Writer(out);
// parse
long start = System.currentTimeMillis();
// System.err.print("Parsing sentences ");
int sentID = 0;
for (List<? extends HasWord> sentence : sentences) {
Tree t = new Tree();
// t.setSentID(++sentID);
System.err.println("parse Sentence :" + sentence + "...");
// System.err.print(".");
System.err.println("-----------------------------------------------------------------------");
edu.stanford.nlp.trees.Tree parse = lp.parse(sentence);
// parse.pennPrint();
// List for root node and lexical nodes
List<Node> loneNodes = new LinkedList<Node>();
List<Node> governingNodes = new LinkedList<Node>();
// ROOT node
Node root = new Node(true, true);
root.setTag("ROOT");
t.setRoot(root);
loneNodes.add(root);
governingNodes.add(root);
// tagging
int counter = 0;
String surface = "";
String tag = "";
for (TaggedWord tw : parse.taggedYield()) {
Node n = new Node();
Node governingNode = new Node();
n.setNodeID(++counter);
surface = tw.value();
tag = tw.tag();
if (surface.startsWith("-LRB-")) {
surface = "(";
} else if (surface.startsWith("-RRB-")) {
surface = ")";
// } else if (surface.startsWith("-LSB-")){
// surface = "[";
// } else if (surface.startsWith("-RSB-")){
// surface = "]";
// } else if (surface.startsWith("-LCB-")){
// surface = "{";
// } else if (surface.startsWith("-RCB-")){
// surface = "}";
} else if (surface.startsWith("''")) {
surface = "\"";
}
tag = tag.replaceAll("#", "-NUM-");
surface = surface.replaceAll("&", "-AMP-");
surface = surface.replaceAll("#", "-NUM-");
surface = surface.replaceAll(">", "-GRE-");
surface = surface.replaceAll("=", "-EQU-");
n.setInitialLexicalIndex(counter);
governingNode.setInitialLexicalIndex(counter);
n.setSurface(surface);
// System.out.print("("+tw.value()+" : ");
n.setTag(tag);
governingNode.setTag("_" + tag);
governingNode.setLabel("_gov");
// System.out.print(tw.tag()+")");
loneNodes.add(n);
governingNodes.add(governingNode);
governingNode.setChild(n);
}
// System.out.println("");
// t.setSentLength(t.getNodes().size() - 1);
// List<Node> loneNodes = new LinkedList<Node>();
Node[] nodes = new Node[2000];
// labeling
int depIndex;
int govIndex;
String[] depInfo;
String[] govInfo;
GrammaticalStructure gs = gsf.newGrammaticalStructure(parse);
List<TypedDependency> tdl = gs.typedDependencies(false);
// List<TypedDependency> tdl = gs.typedDependenciesCCprocessed();
for (TypedDependency td : tdl) {
depIndex = td.dep().index();
govIndex = td.gov().index();
// System.out.println("Index1:"+depIndex);
// System.out.println("Index2:"+govIndex);
// if (nodes[depIndex] == null){
// System.out.println("Making node!");
// nodes[depIndex] = new Node();
// }
// if (nodes[govIndex] == null){
// System.out.println("Making node!");
// nodes[govIndex] = new Node();
// }
Node dep = loneNodes.get((depIndex));
Node gov = governingNodes.get((govIndex));
Node depcopy = governingNodes.get((depIndex));
Node govcopy = loneNodes.get((govIndex));
dep.setLabel(td.reln().toString());
depcopy.setLabel(td.reln().toString());
govcopy.setLabel("head");
// System.out.println(td.toString());
govInfo = td.gov().toString().split("/");
depInfo = td.dep().toString().split("/");
// System.out.println(td.gov().toString());
// System.out.println(td.dep().toString());
// dep.setSurface(depInfo[0]);
// dep.setTag(depInfo[1]);
gov.setChild(governingNodes.get(depIndex));
governingNodes.get(depIndex).setParent(gov);
// gov.setChild(dep);
dep.setParent(governingNodes.get(depIndex));
}
// t.setRoot(nodes[0]);
// Collapse tree to remove unneeded governing nodes:
Node gov;
Node dep;
Node parent;
List<Node> children;
for (int i = 1; i < governingNodes.size(); i++) { // start with index 1 to skip root
gov = governingNodes.get(i);
dep = loneNodes.get(i);
if (gov.getChildren().size() <= 1) {
int k = 0;
parent = gov.getParent();
children = parent.getChildren();
for (Node n : children) {
if (n == gov) {
gov.getParent().replaceChild(k, dep);
dep.setParent(gov.getParent());
}
k++;
}
}
}
// Mark head nodes with appropriate label:
int k = 0;
for (Node n : loneNodes) {
if (k != 0) {
if (n.getLabel() == n.getParent().getLabel()) {
n.setLabel("head");
}
} else {
n.setLabel("null");
}
k++;
}
// Sort lexical children of each governing node in lexical order
for (Node n : governingNodes) {
n.sortChildrenByInitialIndex();
}
// combine with alignment
if (use_alignments) {
t.initialize(alignment.readNextAlign());
} else {
t.initializeUnaligned();
}
// write tree to file
treeWriter.write(t);
// print tree to console
System.out.println(t.toSentence());
if (verbose) {
System.err.println(t.toString());
// t.recursivePrint();
}
System.err.println("#######################################################################");
}
long stop = System.currentTimeMillis();
System.err.println("...done! [" + (stop - start) / 1000 + " sec].");
treeWriter.close();
}