@SuppressWarnings("unchecked")
public Parser(Grammar<NT, T> grammar) {
this.table = new Table<NT, T, List<?>>(grammar);
this.start = grammar.getStartSymbol();
this.eof = grammar.getEOFToken();
reset();
}
private Object parseRecursive(StringTokenizer st) {
List<Object> currentSequence = new ArrayList<>();
Object lastParsed = null;
while (st.hasMoreElements()) {
String s = st.nextToken();
switch (s) {
case " ": // split on blanks
currentSequence.add(lastParsed);
break;
case "?":
lastParsed = grammar.zeroOrOne(lastParsed);
break;
case "*":
lastParsed = grammar.zeroOrMore(lastParsed);
break;
case "(":
lastParsed = parseRecursive(st);
break;
case ")": // current group finished
return finish(currentSequence, lastParsed);
case "'":
lastParsed = Token.getFromDescription(st.nextToken("'"));
st.nextToken("'?*() "); // '
break;
default:
lastParsed = s; // rule name
break;
}
}
return finish(currentSequence, lastParsed);
}
/**
* Create NFA, DFA and generate code for grammar. Create NFA for any delegates first. Once all NFA
* are created, it's ok to create DFA, which must check for left-recursion. That check is done by
* walking the full NFA, which therefore must be complete. After all NFA, comes DFA conversion for
* root grammar then code gen for root grammar. DFA and code gen for delegates comes next.
*/
protected void generateRecognizer(Grammar grammar) {
String language = (String) grammar.getOption("language");
if (language != null) {
CodeGenerator generator = new CodeGenerator(this, grammar, language);
grammar.setCodeGenerator(generator);
generator.setDebug(isDebug());
generator.setProfile(isProfile());
generator.setTrace(isTrace());
// generate NFA early in case of crash later (for debugging)
if (isGenerate_NFA_dot()) {
generateNFAs(grammar);
}
// GENERATE CODE
generator.genRecognizer();
if (isGenerate_DFA_dot()) {
generateDFAs(grammar);
}
List<Grammar> delegates = grammar.getDirectDelegates();
for (int i = 0; delegates != null && i < delegates.size(); i++) {
Grammar delegate = (Grammar) delegates.get(i);
if (delegate != grammar) { // already processing this one
generateRecognizer(delegate);
}
}
}
}
/* Checks that the method storeLengthProduction works fine. */
@Test(timeout = 100)
public void storeLengthProductionTest() {
Collection<Production> cp;
int res;
try {
l.storeLengthProductions(g);
cp = g.getProductions();
System.out.println();
System.out.println("*******************");
System.out.println();
System.out.println("TEST StoreLengthProduction");
System.out.println();
System.out.println("*******************");
for (Production p : cp) {
res = g.getLenghtProduction(p);
System.out.println();
System.out.println("The length of the Production " + p.getSymbol() + " is: " + res);
System.out.println();
}
} catch (GrammarExceptionImpl e) {
System.out.println(e.getMessage());
}
}
/* Checks that the method storeLengthNonTerminal works fine. */
@Test(timeout = 100)
public void storeLengthNonTerminalTest() {
Collection<NonTerminal> cnt;
int res;
try {
l.storeLengthNonTerminals(g);
cnt = g.getNonTerminals();
System.out.println();
System.out.println("*******************");
System.out.println();
System.out.println("TEST StoreLengthNonTerminal");
System.out.println();
System.out.println("*******************");
for (NonTerminal nt : cnt) {
res = g.getLenghtNonTerminal(nt);
System.out.println();
System.out.println("The length of the NonTerminal " + nt.getSymbol() + " is: " + res);
System.out.println();
}
} catch (GrammarExceptionImpl e) {
System.out.println(e.getMessage());
}
}
@Test
public void negativeForwardAssertion1() {
String[] rules = {
"ROOT = 'a' ! /\\B/", //
};
Grammar g = new Grammar(new Grammar(rules).describe());
assertNotNull(g.matches("a").match());
}
@Test
public void forwardAssertion2() {
String[] rules = {
"ROOT = 'a' ~+ /\\b/", //
};
Grammar g = new Grammar(new Grammar(rules).describe());
assertNotNull(g.matches("a").match());
}
@Test
public void literal() {
String[] rules = {
"ROOT = 'a'", //
};
Grammar g = new Grammar(new Grammar(rules).describe());
assertNotNull(g.matches("a").match());
}
@Test
public void backreference() {
String[] rules = {
"ROOT = 'a' 'b' 1", //
};
Grammar g = new Grammar(new Grammar(rules).describe());
assertNotNull(g.matches("aba").match());
}
@Test
public void tag() {
String[] rules = {
"ROOT = [{foo} 'a']", //
};
Grammar g = new Grammar(new Grammar(rules).describe());
assertNotNull(g.matches("a").match().first("foo"));
}
@Test
public void alternation() {
String[] rules = {
"ROOT = 'a' | 'b'", //
};
Grammar g = new Grammar(new Grammar(rules).describe());
assertNotNull(g.matches("b").match());
}
@Test
public void regex() {
String[] rules = {
"ROOT = /a/i", //
};
Grammar g = new Grammar(new Grammar(rules).describe());
assertNotNull(g.matches("A").match());
}
@Test
public void negativeBackwardAssertion() {
String[] rules = {
"ROOT = !- 'b' 'a'", //
};
Grammar g = new Grammar(new Grammar(rules).describe());
assertNotNull(g.find("a").match());
}
Symbol getNextSymbol(StringTokenizer tokenizer, Grammar grammar) {
String token = tokenizer.nextToken().trim();
Symbol symbol = grammar.getSymbolForToken(token);
if (symbol == null) {
symbol = Symbol.createNonterminalSymbol(token);
grammar.addSymbol(symbol);
}
return symbol;
}
public static void main(String[] args) throws IOException {
for (String filename : args) {
GrammarReader gr = new GrammarReader(true);
gr.read(Paths.get(filename).toFile().toURI().toURL());
Grammar grammar = gr.getGrammar();
grammar.addRule("Goal", "CompilationUnit");
grammar.validateRules();
for (Rule r : grammar.getRules()) System.out.println(r);
}
}
private static Builder makeBuilder(String grammarPath) {
Grammar g = new Grammar();
g.read(grammarPath);
Builder b = new Builder();
b.grammar = g;
b.executor = new FormulaMatchExecutor();
b.buildUnspecified();
return b;
}
@Test
public void cycle() {
String[] rules = {
"ROOT = 'a' | <ROOT> 'b'", //
};
Grammar g = new Grammar(rules);
assertNotNull(g.matches("ab").match());
String s = g.describe();
g = new Grammar(s);
assertNotNull(g.matches("ab").match());
}
@Test
public void basicTest() throws GrammarException, IOException {
String[] rules = {
//
"<ROOT> = <a> <s> <b>", //
"<a> = 'a'", //
"<b> = 'b'", //
"<s> = /\\s++/", //
};
Grammar g = new Grammar(rules);
String s = "a\nb";
Matcher m = g.find(s);
assertNotNull("found joe", m.match());
}
protected Grammar internalFindGrammar(GrammarProvider grammarProvider) {
Grammar grammar = grammarProvider.getGrammar(this);
while (grammar != null) {
if ("org.xtext.scripting.Scripting".equals(grammar.getName())) {
return grammar;
}
List<Grammar> grammars = grammar.getUsedGrammars();
if (!grammars.isEmpty()) {
grammar = grammars.iterator().next();
} else {
return null;
}
}
return grammar;
}
protected Grammar internalFindGrammar(GrammarProvider grammarProvider) {
Grammar grammar = grammarProvider.getGrammar(this);
while (grammar != null) {
if ("org.eclipse.ocl.examples.xtext.markup.Markup".equals(grammar.getName())) {
return grammar;
}
List<Grammar> grammars = grammar.getUsedGrammars();
if (!grammars.isEmpty()) {
grammar = grammars.iterator().next();
} else {
return null;
}
}
return grammar;
}
protected Grammar internalFindGrammar(GrammarProvider grammarProvider) {
Grammar grammar = grammarProvider.getGrammar(this);
while (grammar != null) {
if ("fr.inria.diverse.puzzle.language.Binding".equals(grammar.getName())) {
return grammar;
}
List<Grammar> grammars = grammar.getUsedGrammars();
if (!grammars.isEmpty()) {
grammar = grammars.iterator().next();
} else {
return null;
}
}
return grammar;
}
protected Grammar internalFindGrammar(GrammarProvider grammarProvider) {
Grammar grammar = grammarProvider.getGrammar(this);
while (grammar != null) {
if ("org.eclipse.xtext.parser.encoding.EncodingTestLanguage".equals(grammar.getName())) {
return grammar;
}
List<Grammar> grammars = grammar.getUsedGrammars();
if (!grammars.isEmpty()) {
grammar = grammars.iterator().next();
} else {
return null;
}
}
return grammar;
}
protected Grammar internalFindGrammar(GrammarProvider grammarProvider) {
Grammar grammar = grammarProvider.getGrammar(this);
while (grammar != null) {
if ("com.parallels.aps.ide.rqleditor.xtext.rql.RQL".equals(grammar.getName())) {
return grammar;
}
List<Grammar> grammars = grammar.getUsedGrammars();
if (!grammars.isEmpty()) {
grammar = grammars.iterator().next();
} else {
return null;
}
}
return grammar;
}
/*
* (non-Javadoc)
*
* @see org.eclipse.vtp.framework.spi.voice.output.Grammar#writeAttributes(
* org.xml.sax.helpers.AttributesImpl)
*/
protected void writeAttributes(AttributesImpl attributes) {
super.writeAttributes(attributes);
if (grammarType != null)
writeAttribute(attributes, null, null, NAME_TYPE, TYPE_CDATA, grammarType);
else writeAttribute(attributes, null, null, NAME_TYPE, TYPE_CDATA, detectGrammarType());
writeAttribute(attributes, null, null, NAME_SRC, TYPE_CDATA, grammarURI);
}
@Override
public String toString() {
StringBuffer result = new StringBuffer();
result.append("{[LR0ItemCollection]" + System.getProperty("line.separator"));
LR0ItemSet[] sets = sets();
Symbol[] symbols = Symbol.symbols();
for (int i = 0; i < sets.length; i++) {
result.append(i + ":" + Grammar.CLOSURE(sets[i]));
result.append(System.getProperty("line.separator"));
for (int j = 0; j < symbols.length; j++) {
if (GOTO(i, symbols[j]) != null) {
result.append("[");
result.append(symbols[j]);
result.append(":");
result.append(GOTO(i, symbols[j]));
result.append("]");
}
}
result.append(System.getProperty("line.separator"));
}
result.append("}");
return result.toString();
}
public void finishOutput() throws IOException {
currentOutput.close();
if (grammar != null) {
PrintWriter smapWriter;
smapWriter = tool.openOutputFile(grammar.getClassName() + ".smap");
String grammarFile = grammar.getFilename();
grammarFile = grammarFile.replace('\\', '/');
int lastSlash = grammarFile.lastIndexOf('/');
if (lastSlash != -1) {
grammarFile = grammarFile.substring(lastSlash + 1);
}
smapOutput.dump(smapWriter, grammar.getClassName(), grammarFile);
sourceMaps.put(currentFileName, smapOutput.getSourceMap());
}
currentOutput = null;
}
/**
* This method is used by all code generators to create new output files. If the outputDir set by
* -o is not present it will be created. The final filename is sensitive to the output directory
* and the directory where the grammar file was found. If -o is /tmp and the original grammar file
* was foo/t.g then output files go in /tmp/foo.
*
* <p>The output dir -o spec takes precedence if it's absolute. E.g., if the grammar file dir is
* absolute the output dir is given precendence. "-o /tmp /usr/lib/t.g" results in "/tmp/T.java"
* as output (assuming t.g holds T.java).
*
* <p>If no -o is specified, then just write to the directory where the grammar file was found.
*
* <p>If outputDirectory==null then write a String.
*/
public Writer getOutputFile(Grammar g, String fileName) throws IOException {
if (getOutputDirectory() == null) {
return new StringWriter();
}
// output directory is a function of where the grammar file lives
// for subdir/T.g, you get subdir here. Well, depends on -o etc...
// But, if this is a .tokens file, then we force the output to
// be the base output directory (or current directory if there is not a -o)
//
File outputDir;
if (fileName.endsWith(CodeGenerator.VOCAB_FILE_EXTENSION)) {
if (haveOutputDir) {
outputDir = new File(getOutputDirectory());
} else {
outputDir = new File(".");
}
} else {
outputDir = getOutputDirectory(g.getFileName());
}
File outputFile = new File(outputDir, fileName);
if (!outputDir.exists()) {
outputDir.mkdirs();
}
FileWriter fw = new FileWriter(outputFile);
return new BufferedWriter(fw);
}
protected Grammar internalFindGrammar(GrammarProvider grammarProvider) {
Grammar grammar = grammarProvider.getGrammar(this);
while (grammar != null) {
if ("org.eclipse.xtext.ui.tests.editor.contentassist.DatatypeRuleTestLanguage"
.equals(grammar.getName())) {
return grammar;
}
List<Grammar> grammars = grammar.getUsedGrammars();
if (!grammars.isEmpty()) {
grammar = grammars.iterator().next();
} else {
return null;
}
}
return grammar;
}
public String toString() {
return "GrammarCorpus, based on "
+ grammar.getClass()
+ "-grammar, "
+ sentences
+ " sentences ";
}
protected Grammar internalFindGrammar(GrammarProvider grammarProvider) {
Grammar grammar = grammarProvider.getGrammar(this);
while (grammar != null) {
if ("org.eclipse.xtext.formatting2.regionaccess.internal.RegionAccessTestLanguage"
.equals(grammar.getName())) {
return grammar;
}
List<Grammar> grammars = grammar.getUsedGrammars();
if (!grammars.isEmpty()) {
grammar = grammars.iterator().next();
} else {
return null;
}
}
return grammar;
}
