public void sortGrammarFiles() throws IOException {
// System.out.println("Grammar names "+getGrammarFileNames());
Graph g = new Graph();
List<String> missingFiles = new ArrayList<String>();
for (String gfile : grammarFileNames) {
try {
GrammarSpelunker grammar = new GrammarSpelunker(inputDirectory, gfile);
grammar.parse();
String vocabName = grammar.getTokenVocab();
String grammarName = grammar.getGrammarName();
// Make all grammars depend on any tokenVocab options
if (vocabName != null) g.addEdge(gfile, vocabName + CodeGenerator.VOCAB_FILE_EXTENSION);
// Make all generated tokens files depend on their grammars
g.addEdge(grammarName + CodeGenerator.VOCAB_FILE_EXTENSION, gfile);
} catch (FileNotFoundException fnfe) {
ErrorManager.error(ErrorManager.MSG_CANNOT_OPEN_FILE, gfile);
missingFiles.add(gfile);
}
}
List<Object> sorted = g.sort();
// System.out.println("sorted="+sorted);
grammarFileNames.clear(); // wipe so we can give new ordered list
for (int i = 0; i < sorted.size(); i++) {
String f = (String) sorted.get(i);
if (missingFiles.contains(f)) continue;
if (!(f.endsWith(".g") || f.endsWith(".g3"))) continue;
grammarFileNames.add(f);
}
// System.out.println("new grammars="+grammarFileNames);
}
/**
* A list of dependency generators that are accumulated aaaas (and if) the tool is required to
* sort the provided grammars into build dependency order. protected Map<String,
* BuildDependencyGenerator> buildDependencyGenerators;
*/
public static void main(String[] args) {
Tool antlr = new Tool(args);
if (!exitNow) {
antlr.process();
if (ErrorManager.getNumErrors() > 0) {
System.exit(1);
}
System.exit(0);
}
}
@Before
public void setUp() throws Exception {
lastTestFailed = false; // hope for the best, but set to true in asserts that fail
// new output dir for each test
tmpdir =
new File(
System.getProperty("java.io.tmpdir"),
"antlr-" + getClass().getName() + "-" + System.currentTimeMillis())
.getAbsolutePath();
ErrorManager.resetErrorState();
}
protected void generateNFAs(Grammar g) {
DOTGenerator dotGenerator = new DOTGenerator(g);
Collection rules = g.getAllImportedRules();
rules.addAll(g.getRules());
for (Iterator itr = rules.iterator(); itr.hasNext(); ) {
Rule r = (Rule) itr.next();
try {
String dot = dotGenerator.getDOT(r.startState);
if (dot != null) {
writeDOTFile(g, r, dot);
}
} catch (IOException ioe) {
ErrorManager.error(ErrorManager.MSG_CANNOT_WRITE_FILE, ioe);
}
}
}
/** Return true if all is ok, no errors */
protected boolean antlr(
String fileName, String grammarFileName, String grammarStr, boolean debug) {
boolean allIsWell = true;
mkdir(tmpdir);
writeFile(tmpdir, fileName, grammarStr);
try {
final List options = new ArrayList();
if (debug) {
options.add("-debug");
}
options.add("-o");
options.add(tmpdir);
options.add("-lib");
options.add(tmpdir);
options.add(new File(tmpdir, grammarFileName).toString());
final String[] optionsA = new String[options.size()];
options.toArray(optionsA);
/*
final ErrorQueue equeue = new ErrorQueue();
ErrorManager.setErrorListener(equeue);
*/
Tool antlr = newTool(optionsA);
antlr.process();
ANTLRErrorListener listener = ErrorManager.getErrorListener();
if (listener instanceof ErrorQueue) {
ErrorQueue equeue = (ErrorQueue) listener;
if (equeue.errors.size() > 0) {
allIsWell = false;
System.err.println("antlr reports errors from " + options);
for (int i = 0; i < equeue.errors.size(); i++) {
Message msg = (Message) equeue.errors.get(i);
System.err.println(msg);
}
System.out.println("!!!\ngrammar:");
System.out.println(grammarStr);
System.out.println("###");
}
}
} catch (Exception e) {
allIsWell = false;
System.err.println("problems building grammar: " + e);
e.printStackTrace(System.err);
}
return allIsWell;
}
public void generateDFAs(Grammar g) {
for (int d = 1; d <= g.getNumberOfDecisions(); d++) {
DFA dfa = g.getLookaheadDFA(d);
if (dfa == null) {
continue; // not there for some reason, ignore
}
DOTGenerator dotGenerator = new DOTGenerator(g);
String dot = dotGenerator.getDOT(dfa.startState);
String dotFileName = g.name + "." + "dec-" + d;
if (g.implicitLexer) {
dotFileName = g.name + Grammar.grammarTypeToFileNameSuffix[g.type] + "." + "dec-" + d;
}
try {
writeDOTFile(g, dotFileName, dot);
} catch (IOException ioe) {
ErrorManager.error(ErrorManager.MSG_CANNOT_GEN_DOT_FILE, dotFileName, ioe);
}
}
}
protected void checkError(ErrorQueue equeue, Message expectedMessage) throws Exception {
// System.out.println("errors="+equeue);
Message foundMsg = null;
for (int i = 0; i < equeue.errors.size(); i++) {
Message m = (Message) equeue.errors.get(i);
if (m.msgID == expectedMessage.msgID) {
foundMsg = m;
}
}
assertTrue("no error; " + expectedMessage.msgID + " expected", equeue.errors.size() > 0);
assertTrue("too many errors; " + equeue.errors, equeue.errors.size() <= 1);
assertNotNull("couldn't find expected error: " + expectedMessage.msgID, foundMsg);
/*
assertTrue("error is not a GrammarSemanticsMessage",
foundMsg instanceof GrammarSemanticsMessage);
*/
assertEquals(expectedMessage.arg, foundMsg.arg);
assertEquals(expectedMessage.arg2, foundMsg.arg2);
ErrorManager.resetErrorState(); // wack errors for next test
}
private static void Xhelp() {
ErrorManager.info("ANTLR Parser Generator Version " + new Tool().VERSION);
System.err.println(" -Xgrtree print the grammar AST");
System.err.println(" -Xdfa print DFA as text ");
System.err.println(" -Xnoprune test lookahead against EBNF block exit branches");
System.err.println(" -Xnocollapse collapse incident edges into DFA states");
System.err.println(" -Xdbgconversion dump lots of info during NFA conversion");
System.err.println(" -Xmultithreaded run the analysis in 2 threads");
System.err.println(" -Xnomergestopstates do not merge stop states");
System.err.println(" -Xdfaverbose generate DFA states in DOT with NFA configs");
System.err.println(" -Xwatchconversion print a message for each NFA before converting");
System.err.println(
" -XdbgST put tags at start/stop of all templates in output");
System.err.println(
" -Xnfastates for nondeterminisms, list NFA states for each path");
System.err.println(
" -Xm m max number of rule invocations during conversion ["
+ NFAContext.MAX_SAME_RULE_INVOCATIONS_PER_NFA_CONFIG_STACK
+ "]");
System.err.println(
" -Xmaxdfaedges m max \"comfortable\" number of edges for single DFA state ["
+ DFA.MAX_STATE_TRANSITIONS_FOR_TABLE
+ "]");
System.err.println(
" -Xconversiontimeout t set NFA conversion timeout (ms) for each decision ["
+ DFA.MAX_TIME_PER_DFA_CREATION
+ "]");
System.err.println(
" -Xmaxinlinedfastates m max DFA states before table used rather than inlining ["
+ CodeGenerator.MADSI_DEFAULT
+ "]");
System.err.println(
" -Xmaxswitchcaselabels m don't generate switch() statements for dfas bigger than m ["
+ CodeGenerator.MSCL_DEFAULT
+ "]");
System.err.println(
" -Xminswitchalts m don't generate switch() statements for dfas smaller than m ["
+ CodeGenerator.MSA_DEFAULT
+ "]");
}
private static void help() {
ErrorManager.info("ANTLR Parser Generator Version " + new Tool().VERSION);
System.err.println("usage: java org.antlr.Tool [args] file.g [file2.g file3.g ...]");
System.err.println(
" -o outputDir specify output directory where all output is generated");
System.err.println(
" -fo outputDir same as -o but force even files with relative paths to dir");
System.err.println(" -lib dir specify location of token files");
System.err.println(" -depend generate file dependencies");
System.err.println(" -report print out a report about the grammar(s) processed");
System.err.println(" -print print out the grammar without actions");
System.err.println(" -debug generate a parser that emits debugging events");
System.err.println(
" -profile generate a parser that computes profiling information");
System.err.println(" -nfa generate an NFA for each rule");
System.err.println(" -dfa generate a DFA for each decision point");
System.err.println(" -message-format name specify output style for messages");
System.err.println(" -verbose generate ANTLR version and other information");
System.err.println(" -make only build if generated files older than grammar");
System.err.println(" -version print the version of ANTLR and exit.");
System.err.println(" -X display extended argument list");
}
private static void version() {
ErrorManager.info("ANTLR Parser Generator Version " + new Tool().VERSION);
}
public void process() {
boolean exceptionWhenWritingLexerFile = false;
String lexerGrammarFileName = null; // necessary at this scope to have access in the catch below
// Have to be tricky here when Maven or build tools call in and must new Tool()
// before setting options. The banner won't display that way!
if (isVerbose() && showBanner) {
ErrorManager.info("ANTLR Parser Generator Version " + VERSION);
showBanner = false;
}
try {
sortGrammarFiles(); // update grammarFileNames
} catch (Exception e) {
ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e);
} catch (Error e) {
ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e);
}
for (String grammarFileName : grammarFileNames) {
// If we are in make mode (to support build tools like Maven) and the
// file is already up to date, then we do not build it (and in verbose mode
// we will say so).
if (make) {
try {
if (!buildRequired(grammarFileName)) continue;
} catch (Exception e) {
ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e);
}
}
if (isVerbose() && !isDepend()) {
System.out.println(grammarFileName);
}
try {
if (isDepend()) {
BuildDependencyGenerator dep = new BuildDependencyGenerator(this, grammarFileName);
/*
List outputFiles = dep.getGeneratedFileList();
List dependents = dep.getDependenciesFileList();
System.out.println("output: "+outputFiles);
System.out.println("dependents: "+dependents);
*/
System.out.println(dep.getDependencies());
continue;
}
Grammar grammar = getRootGrammar(grammarFileName);
// we now have all grammars read in as ASTs
// (i.e., root and all delegates)
grammar.composite.assignTokenTypes();
grammar.composite.defineGrammarSymbols();
grammar.composite.createNFAs();
generateRecognizer(grammar);
if (isPrintGrammar()) {
grammar.printGrammar(System.out);
}
if (isReport()) {
GrammarReport greport = new GrammarReport(grammar);
System.out.println(greport.toString());
// print out a backtracking report too (that is not encoded into log)
System.out.println(greport.getBacktrackingReport());
// same for aborted NFA->DFA conversions
System.out.println(greport.getAnalysisTimeoutReport());
}
if (isProfile()) {
GrammarReport greport = new GrammarReport(grammar);
Stats.writeReport(GrammarReport.GRAMMAR_STATS_FILENAME, greport.toNotifyString());
}
// now handle the lexer if one was created for a merged spec
String lexerGrammarStr = grammar.getLexerGrammar();
// System.out.println("lexer grammar:\n"+lexerGrammarStr);
if (grammar.type == Grammar.COMBINED && lexerGrammarStr != null) {
lexerGrammarFileName = grammar.getImplicitlyGeneratedLexerFileName();
try {
Writer w = getOutputFile(grammar, lexerGrammarFileName);
w.write(lexerGrammarStr);
w.close();
} catch (IOException e) {
// emit different error message when creating the implicit lexer fails
// due to write permission error
exceptionWhenWritingLexerFile = true;
throw e;
}
try {
StringReader sr = new StringReader(lexerGrammarStr);
Grammar lexerGrammar = new Grammar();
lexerGrammar.composite.watchNFAConversion = internalOption_watchNFAConversion;
lexerGrammar.implicitLexer = true;
lexerGrammar.setTool(this);
File lexerGrammarFullFile =
new File(getFileDirectory(lexerGrammarFileName), lexerGrammarFileName);
lexerGrammar.setFileName(lexerGrammarFullFile.toString());
lexerGrammar.importTokenVocabulary(grammar);
lexerGrammar.parseAndBuildAST(sr);
sr.close();
lexerGrammar.composite.assignTokenTypes();
lexerGrammar.composite.defineGrammarSymbols();
lexerGrammar.composite.createNFAs();
generateRecognizer(lexerGrammar);
} finally {
// make sure we clean up
if (deleteTempLexer) {
File outputDir = getOutputDirectory(lexerGrammarFileName);
File outputFile = new File(outputDir, lexerGrammarFileName);
outputFile.delete();
}
}
}
} catch (IOException e) {
if (exceptionWhenWritingLexerFile) {
ErrorManager.error(ErrorManager.MSG_CANNOT_WRITE_FILE, lexerGrammarFileName, e);
} else {
ErrorManager.error(ErrorManager.MSG_CANNOT_OPEN_FILE, grammarFileName);
}
} catch (Exception e) {
ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, grammarFileName, e);
}
/*
finally {
System.out.println("creates="+ Interval.creates);
System.out.println("hits="+ Interval.hits);
System.out.println("misses="+ Interval.misses);
System.out.println("outOfRange="+ Interval.outOfRange);
}
*/
}
}
public void processArgs(String[] args) {
if (isVerbose()) {
ErrorManager.info("ANTLR Parser Generator Version " + VERSION);
showBanner = false;
}
if (args == null || args.length == 0) {
help();
return;
}
for (int i = 0; i < args.length; i++) {
if (args[i].equals("-o") || args[i].equals("-fo")) {
if (i + 1 >= args.length) {
System.err.println("missing output directory with -fo/-o option; ignoring");
} else {
if (args[i].equals("-fo")) { // force output into dir
setForceAllFilesToOutputDir(true);
}
i++;
outputDirectory = args[i];
if (outputDirectory.endsWith("/") || outputDirectory.endsWith("\\")) {
outputDirectory = outputDirectory.substring(0, getOutputDirectory().length() - 1);
}
File outDir = new File(outputDirectory);
haveOutputDir = true;
if (outDir.exists() && !outDir.isDirectory()) {
ErrorManager.error(ErrorManager.MSG_OUTPUT_DIR_IS_FILE, outputDirectory);
setLibDirectory(".");
}
}
} else if (args[i].equals("-lib")) {
if (i + 1 >= args.length) {
System.err.println("missing library directory with -lib option; ignoring");
} else {
i++;
setLibDirectory(args[i]);
if (getLibraryDirectory().endsWith("/") || getLibraryDirectory().endsWith("\\")) {
setLibDirectory(getLibraryDirectory().substring(0, getLibraryDirectory().length() - 1));
}
File outDir = new File(getLibraryDirectory());
if (!outDir.exists()) {
ErrorManager.error(ErrorManager.MSG_DIR_NOT_FOUND, getLibraryDirectory());
setLibDirectory(".");
}
}
} else if (args[i].equals("-nfa")) {
setGenerate_NFA_dot(true);
} else if (args[i].equals("-dfa")) {
setGenerate_DFA_dot(true);
} else if (args[i].equals("-debug")) {
setDebug(true);
} else if (args[i].equals("-trace")) {
setTrace(true);
} else if (args[i].equals("-report")) {
setReport(true);
} else if (args[i].equals("-profile")) {
setProfile(true);
} else if (args[i].equals("-print")) {
setPrintGrammar(true);
} else if (args[i].equals("-depend")) {
setDepend(true);
} else if (args[i].equals("-verbose")) {
setVerbose(true);
} else if (args[i].equals("-version")) {
version();
exitNow = true;
} else if (args[i].equals("-make")) {
setMake(true);
} else if (args[i].equals("-message-format")) {
if (i + 1 >= args.length) {
System.err.println("missing output format with -message-format option; using default");
} else {
i++;
ErrorManager.setFormat(args[i]);
}
} else if (args[i].equals("-Xgrtree")) {
internalOption_PrintGrammarTree = true; // print grammar tree
} else if (args[i].equals("-Xdfa")) {
internalOption_PrintDFA = true;
} else if (args[i].equals("-Xnoprune")) {
DFAOptimizer.PRUNE_EBNF_EXIT_BRANCHES = false;
} else if (args[i].equals("-Xnocollapse")) {
DFAOptimizer.COLLAPSE_ALL_PARALLEL_EDGES = false;
} else if (args[i].equals("-Xdbgconversion")) {
NFAToDFAConverter.debug = true;
} else if (args[i].equals("-Xmultithreaded")) {
NFAToDFAConverter.SINGLE_THREADED_NFA_CONVERSION = false;
} else if (args[i].equals("-Xnomergestopstates")) {
DFAOptimizer.MERGE_STOP_STATES = false;
} else if (args[i].equals("-Xdfaverbose")) {
internalOption_ShowNFAConfigsInDFA = true;
} else if (args[i].equals("-Xwatchconversion")) {
internalOption_watchNFAConversion = true;
} else if (args[i].equals("-XdbgST")) {
CodeGenerator.EMIT_TEMPLATE_DELIMITERS = true;
} else if (args[i].equals("-Xmaxinlinedfastates")) {
if (i + 1 >= args.length) {
System.err.println("missing max inline dfa states -Xmaxinlinedfastates option; ignoring");
} else {
i++;
CodeGenerator.MAX_ACYCLIC_DFA_STATES_INLINE = Integer.parseInt(args[i]);
}
} else if (args[i].equals("-Xmaxswitchcaselabels")) {
if (i + 1 >= args.length) {
System.err.println(
"missing max switch case labels -Xmaxswitchcaselabels option; ignoring");
} else {
i++;
CodeGenerator.MAX_SWITCH_CASE_LABELS = Integer.parseInt(args[i]);
}
} else if (args[i].equals("-Xminswitchalts")) {
if (i + 1 >= args.length) {
System.err.println("missing min switch alternatives -Xminswitchalts option; ignoring");
} else {
i++;
CodeGenerator.MIN_SWITCH_ALTS = Integer.parseInt(args[i]);
}
} else if (args[i].equals("-Xm")) {
if (i + 1 >= args.length) {
System.err.println("missing max recursion with -Xm option; ignoring");
} else {
i++;
NFAContext.MAX_SAME_RULE_INVOCATIONS_PER_NFA_CONFIG_STACK = Integer.parseInt(args[i]);
}
} else if (args[i].equals("-Xmaxdfaedges")) {
if (i + 1 >= args.length) {
System.err.println("missing max number of edges with -Xmaxdfaedges option; ignoring");
} else {
i++;
DFA.MAX_STATE_TRANSITIONS_FOR_TABLE = Integer.parseInt(args[i]);
}
} else if (args[i].equals("-Xconversiontimeout")) {
if (i + 1 >= args.length) {
System.err.println("missing max time in ms -Xconversiontimeout option; ignoring");
} else {
i++;
DFA.MAX_TIME_PER_DFA_CREATION = Integer.parseInt(args[i]);
}
} else if (args[i].equals("-Xnfastates")) {
DecisionProbe.verbose = true;
} else if (args[i].equals("-X")) {
Xhelp();
} else {
if (args[i].charAt(0) != '-') {
// Must be the grammar file
addGrammarFile(args[i]);
}
}
}
}
/**
* Set the message format to one of ANTLR, gnu, vs2005
*
* @param format
*/
public void setMessageFormat(String format) {
ErrorManager.setFormat(format);
}
/**
* Returns the number of errors that the analysis/processing threw up.
*
* @return Error count
*/
public int getNumErrors() {
return ErrorManager.getNumErrors();
}
/**
* Returns the current setting of the message format descriptor
*
* @return Current message format
*/
public String getMessageFormat() {
return ErrorManager.getMessageFormat().toString();
}