public static Tree processPatternsOnTree(List<Pair<TregexPattern, TsurgeonPattern>> ops, Tree t) {
matchedOnTree = false;
for (Pair<TregexPattern, TsurgeonPattern> op : ops) {
try {
if (DEBUG) {
System.err.println("Running pattern " + op.first());
}
TregexMatcher m = op.first().matcher(t);
while (m.find()) {
matchedOnTree = true;
t = op.second().evaluate(t, m);
if (t == null) {
return null;
}
m = op.first().matcher(t);
}
} catch (NullPointerException npe) {
throw new RuntimeException(
"Tsurgeon.processPatternsOnTree failed to match label for pattern: "
+ op.first()
+ ", "
+ op.second(),
npe);
}
}
return t;
}
/**
* Uses regexp matching to match month, day, and year fields TODO: Find a way to mark what;s
* already been handled in the string
*/
public boolean extractFields(String inputDate) {
if (tokens.size() < 2) {
tokenizeDate(inputDate);
}
if (DEBUG) {
System.err.println("Extracting date: " + inputDate);
}
// first we see if it's a hyphen and two parseable dates - if not, we treat it as one date
Pair<String, String> dateEndpoints = getRangeDates(inputDate);
if (dateEndpoints != null) {
ISODateInstance date1 = new ISODateInstance(dateEndpoints.first());
if (dateEndpoints.first().contains(" ") && !dateEndpoints.second().contains(" ")) {
// consider whether it's a leading modifier; e.g., "June 8-10" will be split into June 8,
// and 10 when really we'd like June 8 and June 10
String date =
dateEndpoints.first().substring(0, dateEndpoints.first().indexOf(' '))
+ ' '
+ dateEndpoints.second();
ISODateInstance date2 = new ISODateInstance(date);
if (!date1.isUnparseable() && !date2.isUnparseable()) {
isoDate = (new ISODateInstance(date1, date2)).getDateString();
return true;
}
}
ISODateInstance date2 = new ISODateInstance(dateEndpoints.second());
if (!date1.isUnparseable() && !date2.isUnparseable()) {
isoDate = (new ISODateInstance(date1, date2)).getDateString();
return true;
}
}
if (extractYYYYMMDD(inputDate)) {
return true;
}
if (extractMMDDYY(inputDate)) {
return true;
}
boolean passed = false;
passed = extractYear(inputDate) || passed;
passed = extractMonth(inputDate) || passed;
passed = extractDay(inputDate) || passed;
// slightly hacky, but check for some common modifiers that get grouped into the date
passed = addExtraRanges(inputDate) || passed;
if (!passed) { // couldn't parse
// try one more trick
unparseable = true;
boolean weekday = extractWeekday(inputDate);
if (!weekday) {
isoDate = inputDate;
}
}
return passed;
}
private String findNextParagraphSpeaker(
List<CoreMap> paragraph, int paragraphOffset, Dictionaries dict) {
CoreMap lastSent = paragraph.get(paragraph.size() - 1);
String speaker = "";
for (CoreLabel w : lastSent.get(CoreAnnotations.TokensAnnotation.class)) {
if (w.get(CoreAnnotations.LemmaAnnotation.class).equals("report")
|| w.get(CoreAnnotations.LemmaAnnotation.class).equals("say")) {
String word = w.get(CoreAnnotations.TextAnnotation.class);
SemanticGraph dependency =
lastSent.get(SemanticGraphCoreAnnotations.CollapsedDependenciesAnnotation.class);
IndexedWord t = dependency.getNodeByWordPattern(word);
for (Pair<GrammaticalRelation, IndexedWord> child : dependency.childPairs(t)) {
if (child.first().getShortName().equals("nsubj")) {
int subjectIndex = child.second().index(); // start from 1
IntTuple headPosition = new IntTuple(2);
headPosition.set(0, paragraph.size() - 1 + paragraphOffset);
headPosition.set(1, subjectIndex - 1);
if (mentionheadPositions.containsKey(headPosition)
&& mentionheadPositions.get(headPosition).nerString.startsWith("PER")) {
speaker = Integer.toString(mentionheadPositions.get(headPosition).mentionID);
}
}
}
}
}
return speaker;
}
/** Check one mention is the speaker of the other mention */
public static boolean isSpeaker(Mention m, Mention ant, Dictionaries dict) {
if (!dict.firstPersonPronouns.contains(ant.spanToString().toLowerCase())
|| ant.number == Number.PLURAL
|| ant.sentNum != m.sentNum) return false;
int countQuotationMark = 0;
for (int i = Math.min(m.headIndex, ant.headIndex) + 1;
i < Math.max(m.headIndex, ant.headIndex);
i++) {
String word = m.sentenceWords.get(i).get(CoreAnnotations.TextAnnotation.class);
if (word.equals("``") || word.equals("''")) countQuotationMark++;
}
if (countQuotationMark != 1) return false;
IndexedWord w =
m.dependency.getNodeByWordPattern(
m.sentenceWords.get(m.headIndex).get(CoreAnnotations.TextAnnotation.class));
if (w == null) return false;
for (Pair<GrammaticalRelation, IndexedWord> parent : m.dependency.parentPairs(w)) {
if (parent.first().getShortName().equals("nsubj")
&& dict.reportVerb.contains(parent.second().get(CoreAnnotations.LemmaAnnotation.class))) {
return true;
}
}
return false;
}
private void commitVariableGroups(Matcher m) {
committedVariables = true; // commit all my variable groups.
for (Pair<Integer, String> varGroup : myNode.variableGroups) {
String thisVarString = m.group(varGroup.first());
variableStrings.setVar(varGroup.second(), thisVarString);
}
}
private void findQuotationSpeaker(
int utterNum,
List<CoreMap> sentences,
Pair<Integer, Integer> beginQuotation,
Pair<Integer, Integer> endQuotation,
Dictionaries dict) {
if (findSpeaker(utterNum, beginQuotation.first(), sentences, 0, beginQuotation.second(), dict))
return;
if (findSpeaker(
utterNum,
endQuotation.first(),
sentences,
endQuotation.second(),
sentences.get(endQuotation.first()).get(CoreAnnotations.TokensAnnotation.class).size(),
dict)) return;
if (beginQuotation.second() <= 1 && beginQuotation.first() > 0) {
if (findSpeaker(
utterNum,
beginQuotation.first() - 1,
sentences,
0,
sentences
.get(beginQuotation.first() - 1)
.get(CoreAnnotations.TokensAnnotation.class)
.size(),
dict)) return;
}
if (endQuotation.second() == sentences.get(endQuotation.first()).size() - 1
&& sentences.size() > endQuotation.first() + 1) {
if (findSpeaker(
utterNum,
endQuotation.first() + 1,
sentences,
0,
sentences
.get(endQuotation.first() + 1)
.get(CoreAnnotations.TokensAnnotation.class)
.size(),
dict)) return;
}
}
public String expandStringRegex(String regex) {
// Replace all variables in regex
String expanded = regex;
for (String v : stringRegexVariables.keySet()) {
Pair<Pattern, String> p = stringRegexVariables.get(v);
expanded = p.first().matcher(expanded).replaceAll(p.second());
}
return expanded;
}
/**
* Compares this <code>Pair</code> to another object. If the object is a <code>Pair</code>, this
* function will work providing the elements of the <code>Pair</code> are themselves comparable.
* It will then return a value based on the pair of objects, where <code>
* p > q iff p.first() > q.first() ||
* (p.first().equals(q.first()) && p.second() > q.second())</code>. If the other object is not
* a <code>Pair</code>, it throws a <code>ClassCastException</code>.
*
* @param o the <code>Object</code> to be compared.
* @return the value <code>0</code> if the argument is a <code>Pair</code> equal to this <code>
* Pair</code>; a value less than <code>0</code> if the argument is a <code>Pair</code>
* greater than this <code>Pair</code>; and a value greater than <code>0</code> if the
* argument is a <code>Pair</code> less than this <code>Pair</code>.
* @throws ClassCastException if the argument is not a <code>Pair</code>.
* @see java.lang.Comparable
*/
public int compareTo(Object o) {
Pair another = (Pair) o;
int comp = ((Comparable) first()).compareTo(another.first());
if (comp != 0) {
return comp;
} else {
return ((Comparable) second()).compareTo(another.second());
}
}
/**
* Read a string representation of a Pair from a DataStream. This might not work correctly unless
* the pair of objects are of type <code>String</code>.
*/
public static Pair<String, String> readStringPair(DataInputStream in) {
Pair<String, String> p = new Pair<String, String>();
try {
p.first = in.readUTF();
p.second = in.readUTF();
} catch (Exception e) {
e.printStackTrace();
}
return p;
}
protected void updateKeepBids(Set<Integer> bids) {
// TODO: Is there a point when we don't need to keep these bids anymore?
for (int i = 0; i < reachableChildBids.length; i++) {
Set<Pair<Integer, Integer>> v = reachableChildBids[i];
if (v != null) {
for (Pair<Integer, Integer> p : v) {
bids.add(p.first());
}
}
}
}
/**
* Construct a new ISODate based on its relation to a referenceDate. relativeDate should be
* something like "today" or "tomorrow" or "last year" and the resulting ISODate will be the same
* as the referenceDate, a day later, or a year earlier, respectively.
*/
public ISODateInstance(ISODateInstance referenceDate, String relativeDate) {
Pair<DateField, Integer> relation = relativeDateMap.get(relativeDate.toLowerCase());
if (relation != null) {
switch (relation.first()) {
case DAY:
incrementDay(referenceDate, relation);
break;
case MONTH:
incrementMonth(referenceDate, relation);
break;
case YEAR:
incrementYear(referenceDate, relation);
break;
}
}
}
private static void extractSubtrees(List<String> codeStrings, String treeFile) {
List<Pair<Integer, Integer>> codes = new ArrayList<Pair<Integer, Integer>>();
for (String s : codeStrings) {
Matcher m = codePattern.matcher(s);
if (m.matches())
codes.add(
new Pair<Integer, Integer>(Integer.parseInt(m.group(1)), Integer.parseInt(m.group(2))));
else throw new RuntimeException("Error: illegal node code " + s);
}
TreeReaderFactory trf = new TRegexTreeReaderFactory();
MemoryTreebank treebank = new MemoryTreebank(trf);
treebank.loadPath(treeFile, null, true);
for (Pair<Integer, Integer> code : codes) {
Tree t = treebank.get(code.first() - 1);
t.getNodeNumber(code.second()).pennPrint();
}
}
private boolean findSpeaker(
int utterNum,
int sentNum,
List<CoreMap> sentences,
int startIndex,
int endIndex,
Dictionaries dict) {
List<CoreLabel> sent = sentences.get(sentNum).get(CoreAnnotations.TokensAnnotation.class);
for (int i = startIndex; i < endIndex; i++) {
if (sent.get(i).get(CoreAnnotations.UtteranceAnnotation.class) != 0) continue;
String lemma = sent.get(i).get(CoreAnnotations.LemmaAnnotation.class);
String word = sent.get(i).get(CoreAnnotations.TextAnnotation.class);
if (dict.reportVerb.contains(lemma)) {
// find subject
SemanticGraph dependency =
sentences
.get(sentNum)
.get(SemanticGraphCoreAnnotations.CollapsedDependenciesAnnotation.class);
IndexedWord w = dependency.getNodeByWordPattern(word);
if (w != null) {
for (Pair<GrammaticalRelation, IndexedWord> child : dependency.childPairs(w)) {
if (child.first().getShortName().equals("nsubj")) {
String subjectString = child.second().word();
int subjectIndex = child.second().index(); // start from 1
IntTuple headPosition = new IntTuple(2);
headPosition.set(0, sentNum);
headPosition.set(1, subjectIndex - 1);
String speaker;
if (mentionheadPositions.containsKey(headPosition)) {
speaker = Integer.toString(mentionheadPositions.get(headPosition).mentionID);
} else {
speaker = subjectString;
}
speakers.put(utterNum, speaker);
return true;
}
}
} else {
SieveCoreferenceSystem.logger.warning("Cannot find node in dependency for word " + word);
}
}
}
return false;
}
/**
* Returns array of child branch ids that causes all child expressions to be satisfied with
* respect to the specified child expression (assuming satisfiction with the specified branch
* and node index) For other child expressions to have a compatible satisfiable branch, that
* branch must also terminate with the same node index as this one.
*
* @param index - Index of the child expression
* @param bid - Branch id that causes the indexed child to be satisfied
* @param pos - Node index that causes the indexed child to be satisfied
* @return array of child branch ids if there is a valid combination null otherwise
*/
private int[] getAllChildMatchedBids(int index, int bid, int pos) {
int[] matchedBids = new int[reachableChildBids.length];
for (int i = 0; i < reachableChildBids.length; i++) {
Set<Pair<Integer, Integer>> v = reachableChildBids[i];
if (v == null || v.isEmpty()) return null;
if (i != index) {
boolean ok = false;
for (Pair<Integer, Integer> p : v) {
if (p.second() == pos) {
ok = true;
matchedBids[i] = p.first();
break;
}
}
if (!ok) {
return null;
}
} else {
matchedBids[i] = bid;
}
}
return matchedBids;
}
protected <T> boolean match(
int bid, SequenceMatcher.MatchedStates<T> matchedStates, boolean consume) {
// Try to match previous node/nodes exactly
if (consume) {
// First element is group that is matched, second is number of nodes matched so far
Pair<SequenceMatcher.MatchedGroup, Integer> backRefState =
(Pair<SequenceMatcher.MatchedGroup, Integer>)
matchedStates.getBranchStates().getMatchStateInfo(bid, this);
if (backRefState == null) {
// Haven't tried to match this node before, try now
// Get element and return if it matched or not
SequenceMatcher.MatchedGroup matchedGroup =
matchedStates.getBranchStates().getMatchedGroup(bid, captureGroupId);
if (matchedGroup != null) {
// See if the first node matches
if (matchedGroup.matchEnd > matchedGroup.matchBegin) {
boolean matched = match(bid, matchedStates, matchedGroup, 0);
return matched;
} else {
// TODO: Check handling of previous nodes that are zero elements?
return super.match(bid, matchedStates, consume);
}
}
return false;
} else {
SequenceMatcher.MatchedGroup matchedGroup = backRefState.first();
int matchedNodes = backRefState.second();
boolean matched = match(bid, matchedStates, matchedGroup, matchedNodes);
return matched;
}
} else {
// Not consuming, just add this state back to list of states to be processed
matchedStates.addState(bid, this);
return false;
}
}
/** @param args */
public static void main(String[] args) {
if (args.length != 3) {
System.err.printf(
"Usage: java %s language filename features%n",
TreebankFactoredLexiconStats.class.getName());
System.exit(-1);
}
Language language = Language.valueOf(args[0]);
TreebankLangParserParams tlpp = language.params;
if (language.equals(Language.Arabic)) {
String[] options = {"-arabicFactored"};
tlpp.setOptionFlag(options, 0);
} else {
String[] options = {"-frenchFactored"};
tlpp.setOptionFlag(options, 0);
}
Treebank tb = tlpp.diskTreebank();
tb.loadPath(args[1]);
MorphoFeatureSpecification morphoSpec =
language.equals(Language.Arabic)
? new ArabicMorphoFeatureSpecification()
: new FrenchMorphoFeatureSpecification();
String[] features = args[2].trim().split(",");
for (String feature : features) {
morphoSpec.activate(MorphoFeatureType.valueOf(feature));
}
// Counters
Counter<String> wordTagCounter = new ClassicCounter<>(30000);
Counter<String> morphTagCounter = new ClassicCounter<>(500);
// Counter<String> signatureTagCounter = new ClassicCounter<String>();
Counter<String> morphCounter = new ClassicCounter<>(500);
Counter<String> wordCounter = new ClassicCounter<>(30000);
Counter<String> tagCounter = new ClassicCounter<>(300);
Counter<String> lemmaCounter = new ClassicCounter<>(25000);
Counter<String> lemmaTagCounter = new ClassicCounter<>(25000);
Counter<String> richTagCounter = new ClassicCounter<>(1000);
Counter<String> reducedTagCounter = new ClassicCounter<>(500);
Counter<String> reducedTagLemmaCounter = new ClassicCounter<>(500);
Map<String, Set<String>> wordLemmaMap = Generics.newHashMap();
TwoDimensionalIntCounter<String, String> lemmaReducedTagCounter =
new TwoDimensionalIntCounter<>(30000);
TwoDimensionalIntCounter<String, String> reducedTagTagCounter =
new TwoDimensionalIntCounter<>(500);
TwoDimensionalIntCounter<String, String> tagReducedTagCounter =
new TwoDimensionalIntCounter<>(300);
int numTrees = 0;
for (Tree tree : tb) {
for (Tree subTree : tree) {
if (!subTree.isLeaf()) {
tlpp.transformTree(subTree, tree);
}
}
List<Label> pretermList = tree.preTerminalYield();
List<Label> yield = tree.yield();
assert yield.size() == pretermList.size();
int yieldLen = yield.size();
for (int i = 0; i < yieldLen; ++i) {
String tag = pretermList.get(i).value();
String word = yield.get(i).value();
String morph = ((CoreLabel) yield.get(i)).originalText();
// Note: if there is no lemma, then we use the surface form.
Pair<String, String> lemmaTag = MorphoFeatureSpecification.splitMorphString(word, morph);
String lemma = lemmaTag.first();
String richTag = lemmaTag.second();
// WSGDEBUG
if (tag.contains("MW")) lemma += "-MWE";
lemmaCounter.incrementCount(lemma);
lemmaTagCounter.incrementCount(lemma + tag);
richTagCounter.incrementCount(richTag);
String reducedTag = morphoSpec.strToFeatures(richTag).toString();
reducedTagCounter.incrementCount(reducedTag);
reducedTagLemmaCounter.incrementCount(reducedTag + lemma);
wordTagCounter.incrementCount(word + tag);
morphTagCounter.incrementCount(morph + tag);
morphCounter.incrementCount(morph);
wordCounter.incrementCount(word);
tagCounter.incrementCount(tag);
reducedTag = reducedTag.equals("") ? "NONE" : reducedTag;
if (wordLemmaMap.containsKey(word)) {
wordLemmaMap.get(word).add(lemma);
} else {
Set<String> lemmas = Generics.newHashSet(1);
wordLemmaMap.put(word, lemmas);
}
lemmaReducedTagCounter.incrementCount(lemma, reducedTag);
reducedTagTagCounter.incrementCount(lemma + reducedTag, tag);
tagReducedTagCounter.incrementCount(tag, reducedTag);
}
++numTrees;
}
// Barf...
System.out.println("Language: " + language.toString());
System.out.printf("#trees:\t%d%n", numTrees);
System.out.printf("#tokens:\t%d%n", (int) wordCounter.totalCount());
System.out.printf("#words:\t%d%n", wordCounter.keySet().size());
System.out.printf("#tags:\t%d%n", tagCounter.keySet().size());
System.out.printf("#wordTagPairs:\t%d%n", wordTagCounter.keySet().size());
System.out.printf("#lemmas:\t%d%n", lemmaCounter.keySet().size());
System.out.printf("#lemmaTagPairs:\t%d%n", lemmaTagCounter.keySet().size());
System.out.printf("#feattags:\t%d%n", reducedTagCounter.keySet().size());
System.out.printf("#feattag+lemmas:\t%d%n", reducedTagLemmaCounter.keySet().size());
System.out.printf("#richtags:\t%d%n", richTagCounter.keySet().size());
System.out.printf("#richtag+lemma:\t%d%n", morphCounter.keySet().size());
System.out.printf("#richtag+lemmaTagPairs:\t%d%n", morphTagCounter.keySet().size());
// Extra
System.out.println("==================");
StringBuilder sbNoLemma = new StringBuilder();
StringBuilder sbMultLemmas = new StringBuilder();
for (Map.Entry<String, Set<String>> wordLemmas : wordLemmaMap.entrySet()) {
String word = wordLemmas.getKey();
Set<String> lemmas = wordLemmas.getValue();
if (lemmas.size() == 0) {
sbNoLemma.append("NO LEMMAS FOR WORD: " + word + "\n");
continue;
}
if (lemmas.size() > 1) {
sbMultLemmas.append("MULTIPLE LEMMAS: " + word + " " + setToString(lemmas) + "\n");
continue;
}
String lemma = lemmas.iterator().next();
Set<String> reducedTags = lemmaReducedTagCounter.getCounter(lemma).keySet();
if (reducedTags.size() > 1) {
System.out.printf("%s --> %s%n", word, lemma);
for (String reducedTag : reducedTags) {
int count = lemmaReducedTagCounter.getCount(lemma, reducedTag);
String posTags =
setToString(reducedTagTagCounter.getCounter(lemma + reducedTag).keySet());
System.out.printf("\t%s\t%d\t%s%n", reducedTag, count, posTags);
}
System.out.println();
}
}
System.out.println("==================");
System.out.println(sbNoLemma.toString());
System.out.println(sbMultLemmas.toString());
System.out.println("==================");
List<String> tags = new ArrayList<>(tagReducedTagCounter.firstKeySet());
Collections.sort(tags);
for (String tag : tags) {
System.out.println(tag);
Set<String> reducedTags = tagReducedTagCounter.getCounter(tag).keySet();
for (String reducedTag : reducedTags) {
int count = tagReducedTagCounter.getCount(tag, reducedTag);
// reducedTag = reducedTag.equals("") ? "NONE" : reducedTag;
System.out.printf("\t%s\t%d%n", reducedTag, count);
}
System.out.println();
}
System.out.println("==================");
}
// when finished = false; break; is called, it means I successfully matched.
@SuppressWarnings("null")
private void goToNextNodeMatch() {
decommitVariableGroups(); // make sure variable groups are free.
decommitNamedNodes();
decommitNamedRelations();
finished = true;
Matcher m = null;
while (nodeMatchCandidateIterator.hasNext()) {
if (myNode.reln.getName() != null) {
String foundReln = namesToRelations.get(myNode.reln.getName());
nextMatchReln = ((GraphRelation.SearchNodeIterator) nodeMatchCandidateIterator).getReln();
if ((foundReln != null) && (!nextMatchReln.equals(foundReln))) {
nextMatch = nodeMatchCandidateIterator.next();
continue;
}
}
nextMatch = nodeMatchCandidateIterator.next();
// System.err.println("going to next match: " + nextMatch.word() + " " +
// myNode.descString + " " + myNode.isLink);
if (myNode.descString.equals("{}") && myNode.isLink) {
IndexedWord otherNode = namesToNodes.get(myNode.name);
if (otherNode != null) {
if (otherNode.equals(nextMatch)) {
if (!myNode.negDesc) {
finished = false;
break;
}
} else {
if (myNode.negDesc) {
finished = false;
break;
}
}
} else {
boolean found = myNode.nodeAttrMatch(nextMatch, hyp ? sg : sg_aligned, ignoreCase);
if (found) {
for (Pair<Integer, String> varGroup : myNode.variableGroups) {
// if variables have been captured from a regex, they
// must match any previous matchings
String thisVariable = varGroup.second();
String thisVarString = variableStrings.getString(thisVariable);
if (thisVarString != null && !thisVarString.equals(m.group(varGroup.first()))) {
// failed to match a variable
found = false;
break;
}
}
// nodeAttrMatch already checks negDesc, so no need to
// check for that here
finished = false;
break;
}
}
} else { // try to match the description pattern.
boolean found = myNode.nodeAttrMatch(nextMatch, hyp ? sg : sg_aligned, ignoreCase);
if (found) {
for (Pair<Integer, String> varGroup : myNode.variableGroups) {
// if variables have been captured from a regex, they
// must match any previous matchings
String thisVariable = varGroup.second();
String thisVarString = variableStrings.getString(thisVariable);
if (thisVarString != null && !thisVarString.equals(m.group(varGroup.first()))) {
// failed to match a variable
found = false;
break;
}
}
// nodeAttrMatch already checks negDesc, so no need to
// check for that here
finished = false;
break;
}
}
} // end while
if (!finished) { // I successfully matched.
resetChild();
if (myNode.name != null) {
// note: have to fill in the map as we go for backreferencing
if (!namesToNodes.containsKey(myNode.name)) {
// System.err.println("making namedFirst");
namedFirst = true;
}
// System.err.println("adding named node: " + myNode.name + "=" +
// nextMatch.word());
namesToNodes.put(myNode.name, nextMatch);
}
if (myNode.reln.getName() != null) {
if (!namesToRelations.containsKey(myNode.reln.getName())) relnNamedFirst = true;
namesToRelations.put(myNode.reln.getName(), nextMatchReln);
}
commitVariableGroups(m); // commit my variable groups.
}
// finished is false exiting this if and only if nextChild exists
// and has a label or backreference that matches
// (also it will just have been reset)
}