public void sortGrammarFiles() throws IOException { // System.out.println("Grammar names "+getGrammarFileNames()); Graph g = new Graph(); List<String> missingFiles = new ArrayList<String>(); for (String gfile : grammarFileNames) { try { GrammarSpelunker grammar = new GrammarSpelunker(inputDirectory, gfile); grammar.parse(); String vocabName = grammar.getTokenVocab(); String grammarName = grammar.getGrammarName(); // Make all grammars depend on any tokenVocab options if (vocabName != null) g.addEdge(gfile, vocabName + CodeGenerator.VOCAB_FILE_EXTENSION); // Make all generated tokens files depend on their grammars g.addEdge(grammarName + CodeGenerator.VOCAB_FILE_EXTENSION, gfile); } catch (FileNotFoundException fnfe) { ErrorManager.error(ErrorManager.MSG_CANNOT_OPEN_FILE, gfile); missingFiles.add(gfile); } } List<Object> sorted = g.sort(); // System.out.println("sorted="+sorted); grammarFileNames.clear(); // wipe so we can give new ordered list for (int i = 0; i < sorted.size(); i++) { String f = (String) sorted.get(i); if (missingFiles.contains(f)) continue; if (!(f.endsWith(".g") || f.endsWith(".g3"))) continue; grammarFileNames.add(f); } // System.out.println("new grammars="+grammarFileNames); }
/** * A list of dependency generators that are accumulated aaaas (and if) the tool is required to * sort the provided grammars into build dependency order. protected Map<String, * BuildDependencyGenerator> buildDependencyGenerators; */ public static void main(String[] args) { Tool antlr = new Tool(args); if (!exitNow) { antlr.process(); if (ErrorManager.getNumErrors() > 0) { System.exit(1); } System.exit(0); } }
@Before public void setUp() throws Exception { lastTestFailed = false; // hope for the best, but set to true in asserts that fail // new output dir for each test tmpdir = new File( System.getProperty("java.io.tmpdir"), "antlr-" + getClass().getName() + "-" + System.currentTimeMillis()) .getAbsolutePath(); ErrorManager.resetErrorState(); }
protected void generateNFAs(Grammar g) { DOTGenerator dotGenerator = new DOTGenerator(g); Collection rules = g.getAllImportedRules(); rules.addAll(g.getRules()); for (Iterator itr = rules.iterator(); itr.hasNext(); ) { Rule r = (Rule) itr.next(); try { String dot = dotGenerator.getDOT(r.startState); if (dot != null) { writeDOTFile(g, r, dot); } } catch (IOException ioe) { ErrorManager.error(ErrorManager.MSG_CANNOT_WRITE_FILE, ioe); } } }
/** Return true if all is ok, no errors */ protected boolean antlr( String fileName, String grammarFileName, String grammarStr, boolean debug) { boolean allIsWell = true; mkdir(tmpdir); writeFile(tmpdir, fileName, grammarStr); try { final List options = new ArrayList(); if (debug) { options.add("-debug"); } options.add("-o"); options.add(tmpdir); options.add("-lib"); options.add(tmpdir); options.add(new File(tmpdir, grammarFileName).toString()); final String[] optionsA = new String[options.size()]; options.toArray(optionsA); /* final ErrorQueue equeue = new ErrorQueue(); ErrorManager.setErrorListener(equeue); */ Tool antlr = newTool(optionsA); antlr.process(); ANTLRErrorListener listener = ErrorManager.getErrorListener(); if (listener instanceof ErrorQueue) { ErrorQueue equeue = (ErrorQueue) listener; if (equeue.errors.size() > 0) { allIsWell = false; System.err.println("antlr reports errors from " + options); for (int i = 0; i < equeue.errors.size(); i++) { Message msg = (Message) equeue.errors.get(i); System.err.println(msg); } System.out.println("!!!\ngrammar:"); System.out.println(grammarStr); System.out.println("###"); } } } catch (Exception e) { allIsWell = false; System.err.println("problems building grammar: " + e); e.printStackTrace(System.err); } return allIsWell; }
public void generateDFAs(Grammar g) { for (int d = 1; d <= g.getNumberOfDecisions(); d++) { DFA dfa = g.getLookaheadDFA(d); if (dfa == null) { continue; // not there for some reason, ignore } DOTGenerator dotGenerator = new DOTGenerator(g); String dot = dotGenerator.getDOT(dfa.startState); String dotFileName = g.name + "." + "dec-" + d; if (g.implicitLexer) { dotFileName = g.name + Grammar.grammarTypeToFileNameSuffix[g.type] + "." + "dec-" + d; } try { writeDOTFile(g, dotFileName, dot); } catch (IOException ioe) { ErrorManager.error(ErrorManager.MSG_CANNOT_GEN_DOT_FILE, dotFileName, ioe); } } }
protected void checkError(ErrorQueue equeue, Message expectedMessage) throws Exception { // System.out.println("errors="+equeue); Message foundMsg = null; for (int i = 0; i < equeue.errors.size(); i++) { Message m = (Message) equeue.errors.get(i); if (m.msgID == expectedMessage.msgID) { foundMsg = m; } } assertTrue("no error; " + expectedMessage.msgID + " expected", equeue.errors.size() > 0); assertTrue("too many errors; " + equeue.errors, equeue.errors.size() <= 1); assertNotNull("couldn't find expected error: " + expectedMessage.msgID, foundMsg); /* assertTrue("error is not a GrammarSemanticsMessage", foundMsg instanceof GrammarSemanticsMessage); */ assertEquals(expectedMessage.arg, foundMsg.arg); assertEquals(expectedMessage.arg2, foundMsg.arg2); ErrorManager.resetErrorState(); // wack errors for next test }
private static void Xhelp() { ErrorManager.info("ANTLR Parser Generator Version " + new Tool().VERSION); System.err.println(" -Xgrtree print the grammar AST"); System.err.println(" -Xdfa print DFA as text "); System.err.println(" -Xnoprune test lookahead against EBNF block exit branches"); System.err.println(" -Xnocollapse collapse incident edges into DFA states"); System.err.println(" -Xdbgconversion dump lots of info during NFA conversion"); System.err.println(" -Xmultithreaded run the analysis in 2 threads"); System.err.println(" -Xnomergestopstates do not merge stop states"); System.err.println(" -Xdfaverbose generate DFA states in DOT with NFA configs"); System.err.println(" -Xwatchconversion print a message for each NFA before converting"); System.err.println( " -XdbgST put tags at start/stop of all templates in output"); System.err.println( " -Xnfastates for nondeterminisms, list NFA states for each path"); System.err.println( " -Xm m max number of rule invocations during conversion [" + NFAContext.MAX_SAME_RULE_INVOCATIONS_PER_NFA_CONFIG_STACK + "]"); System.err.println( " -Xmaxdfaedges m max \"comfortable\" number of edges for single DFA state [" + DFA.MAX_STATE_TRANSITIONS_FOR_TABLE + "]"); System.err.println( " -Xconversiontimeout t set NFA conversion timeout (ms) for each decision [" + DFA.MAX_TIME_PER_DFA_CREATION + "]"); System.err.println( " -Xmaxinlinedfastates m max DFA states before table used rather than inlining [" + CodeGenerator.MADSI_DEFAULT + "]"); System.err.println( " -Xmaxswitchcaselabels m don't generate switch() statements for dfas bigger than m [" + CodeGenerator.MSCL_DEFAULT + "]"); System.err.println( " -Xminswitchalts m don't generate switch() statements for dfas smaller than m [" + CodeGenerator.MSA_DEFAULT + "]"); }
private static void help() { ErrorManager.info("ANTLR Parser Generator Version " + new Tool().VERSION); System.err.println("usage: java org.antlr.Tool [args] file.g [file2.g file3.g ...]"); System.err.println( " -o outputDir specify output directory where all output is generated"); System.err.println( " -fo outputDir same as -o but force even files with relative paths to dir"); System.err.println(" -lib dir specify location of token files"); System.err.println(" -depend generate file dependencies"); System.err.println(" -report print out a report about the grammar(s) processed"); System.err.println(" -print print out the grammar without actions"); System.err.println(" -debug generate a parser that emits debugging events"); System.err.println( " -profile generate a parser that computes profiling information"); System.err.println(" -nfa generate an NFA for each rule"); System.err.println(" -dfa generate a DFA for each decision point"); System.err.println(" -message-format name specify output style for messages"); System.err.println(" -verbose generate ANTLR version and other information"); System.err.println(" -make only build if generated files older than grammar"); System.err.println(" -version print the version of ANTLR and exit."); System.err.println(" -X display extended argument list"); }
private static void version() { ErrorManager.info("ANTLR Parser Generator Version " + new Tool().VERSION); }
public void process() { boolean exceptionWhenWritingLexerFile = false; String lexerGrammarFileName = null; // necessary at this scope to have access in the catch below // Have to be tricky here when Maven or build tools call in and must new Tool() // before setting options. The banner won't display that way! if (isVerbose() && showBanner) { ErrorManager.info("ANTLR Parser Generator Version " + VERSION); showBanner = false; } try { sortGrammarFiles(); // update grammarFileNames } catch (Exception e) { ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e); } catch (Error e) { ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e); } for (String grammarFileName : grammarFileNames) { // If we are in make mode (to support build tools like Maven) and the // file is already up to date, then we do not build it (and in verbose mode // we will say so). if (make) { try { if (!buildRequired(grammarFileName)) continue; } catch (Exception e) { ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e); } } if (isVerbose() && !isDepend()) { System.out.println(grammarFileName); } try { if (isDepend()) { BuildDependencyGenerator dep = new BuildDependencyGenerator(this, grammarFileName); /* List outputFiles = dep.getGeneratedFileList(); List dependents = dep.getDependenciesFileList(); System.out.println("output: "+outputFiles); System.out.println("dependents: "+dependents); */ System.out.println(dep.getDependencies()); continue; } Grammar grammar = getRootGrammar(grammarFileName); // we now have all grammars read in as ASTs // (i.e., root and all delegates) grammar.composite.assignTokenTypes(); grammar.composite.defineGrammarSymbols(); grammar.composite.createNFAs(); generateRecognizer(grammar); if (isPrintGrammar()) { grammar.printGrammar(System.out); } if (isReport()) { GrammarReport greport = new GrammarReport(grammar); System.out.println(greport.toString()); // print out a backtracking report too (that is not encoded into log) System.out.println(greport.getBacktrackingReport()); // same for aborted NFA->DFA conversions System.out.println(greport.getAnalysisTimeoutReport()); } if (isProfile()) { GrammarReport greport = new GrammarReport(grammar); Stats.writeReport(GrammarReport.GRAMMAR_STATS_FILENAME, greport.toNotifyString()); } // now handle the lexer if one was created for a merged spec String lexerGrammarStr = grammar.getLexerGrammar(); // System.out.println("lexer grammar:\n"+lexerGrammarStr); if (grammar.type == Grammar.COMBINED && lexerGrammarStr != null) { lexerGrammarFileName = grammar.getImplicitlyGeneratedLexerFileName(); try { Writer w = getOutputFile(grammar, lexerGrammarFileName); w.write(lexerGrammarStr); w.close(); } catch (IOException e) { // emit different error message when creating the implicit lexer fails // due to write permission error exceptionWhenWritingLexerFile = true; throw e; } try { StringReader sr = new StringReader(lexerGrammarStr); Grammar lexerGrammar = new Grammar(); lexerGrammar.composite.watchNFAConversion = internalOption_watchNFAConversion; lexerGrammar.implicitLexer = true; lexerGrammar.setTool(this); File lexerGrammarFullFile = new File(getFileDirectory(lexerGrammarFileName), lexerGrammarFileName); lexerGrammar.setFileName(lexerGrammarFullFile.toString()); lexerGrammar.importTokenVocabulary(grammar); lexerGrammar.parseAndBuildAST(sr); sr.close(); lexerGrammar.composite.assignTokenTypes(); lexerGrammar.composite.defineGrammarSymbols(); lexerGrammar.composite.createNFAs(); generateRecognizer(lexerGrammar); } finally { // make sure we clean up if (deleteTempLexer) { File outputDir = getOutputDirectory(lexerGrammarFileName); File outputFile = new File(outputDir, lexerGrammarFileName); outputFile.delete(); } } } } catch (IOException e) { if (exceptionWhenWritingLexerFile) { ErrorManager.error(ErrorManager.MSG_CANNOT_WRITE_FILE, lexerGrammarFileName, e); } else { ErrorManager.error(ErrorManager.MSG_CANNOT_OPEN_FILE, grammarFileName); } } catch (Exception e) { ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, grammarFileName, e); } /* finally { System.out.println("creates="+ Interval.creates); System.out.println("hits="+ Interval.hits); System.out.println("misses="+ Interval.misses); System.out.println("outOfRange="+ Interval.outOfRange); } */ } }
public void processArgs(String[] args) { if (isVerbose()) { ErrorManager.info("ANTLR Parser Generator Version " + VERSION); showBanner = false; } if (args == null || args.length == 0) { help(); return; } for (int i = 0; i < args.length; i++) { if (args[i].equals("-o") || args[i].equals("-fo")) { if (i + 1 >= args.length) { System.err.println("missing output directory with -fo/-o option; ignoring"); } else { if (args[i].equals("-fo")) { // force output into dir setForceAllFilesToOutputDir(true); } i++; outputDirectory = args[i]; if (outputDirectory.endsWith("/") || outputDirectory.endsWith("\\")) { outputDirectory = outputDirectory.substring(0, getOutputDirectory().length() - 1); } File outDir = new File(outputDirectory); haveOutputDir = true; if (outDir.exists() && !outDir.isDirectory()) { ErrorManager.error(ErrorManager.MSG_OUTPUT_DIR_IS_FILE, outputDirectory); setLibDirectory("."); } } } else if (args[i].equals("-lib")) { if (i + 1 >= args.length) { System.err.println("missing library directory with -lib option; ignoring"); } else { i++; setLibDirectory(args[i]); if (getLibraryDirectory().endsWith("/") || getLibraryDirectory().endsWith("\\")) { setLibDirectory(getLibraryDirectory().substring(0, getLibraryDirectory().length() - 1)); } File outDir = new File(getLibraryDirectory()); if (!outDir.exists()) { ErrorManager.error(ErrorManager.MSG_DIR_NOT_FOUND, getLibraryDirectory()); setLibDirectory("."); } } } else if (args[i].equals("-nfa")) { setGenerate_NFA_dot(true); } else if (args[i].equals("-dfa")) { setGenerate_DFA_dot(true); } else if (args[i].equals("-debug")) { setDebug(true); } else if (args[i].equals("-trace")) { setTrace(true); } else if (args[i].equals("-report")) { setReport(true); } else if (args[i].equals("-profile")) { setProfile(true); } else if (args[i].equals("-print")) { setPrintGrammar(true); } else if (args[i].equals("-depend")) { setDepend(true); } else if (args[i].equals("-verbose")) { setVerbose(true); } else if (args[i].equals("-version")) { version(); exitNow = true; } else if (args[i].equals("-make")) { setMake(true); } else if (args[i].equals("-message-format")) { if (i + 1 >= args.length) { System.err.println("missing output format with -message-format option; using default"); } else { i++; ErrorManager.setFormat(args[i]); } } else if (args[i].equals("-Xgrtree")) { internalOption_PrintGrammarTree = true; // print grammar tree } else if (args[i].equals("-Xdfa")) { internalOption_PrintDFA = true; } else if (args[i].equals("-Xnoprune")) { DFAOptimizer.PRUNE_EBNF_EXIT_BRANCHES = false; } else if (args[i].equals("-Xnocollapse")) { DFAOptimizer.COLLAPSE_ALL_PARALLEL_EDGES = false; } else if (args[i].equals("-Xdbgconversion")) { NFAToDFAConverter.debug = true; } else if (args[i].equals("-Xmultithreaded")) { NFAToDFAConverter.SINGLE_THREADED_NFA_CONVERSION = false; } else if (args[i].equals("-Xnomergestopstates")) { DFAOptimizer.MERGE_STOP_STATES = false; } else if (args[i].equals("-Xdfaverbose")) { internalOption_ShowNFAConfigsInDFA = true; } else if (args[i].equals("-Xwatchconversion")) { internalOption_watchNFAConversion = true; } else if (args[i].equals("-XdbgST")) { CodeGenerator.EMIT_TEMPLATE_DELIMITERS = true; } else if (args[i].equals("-Xmaxinlinedfastates")) { if (i + 1 >= args.length) { System.err.println("missing max inline dfa states -Xmaxinlinedfastates option; ignoring"); } else { i++; CodeGenerator.MAX_ACYCLIC_DFA_STATES_INLINE = Integer.parseInt(args[i]); } } else if (args[i].equals("-Xmaxswitchcaselabels")) { if (i + 1 >= args.length) { System.err.println( "missing max switch case labels -Xmaxswitchcaselabels option; ignoring"); } else { i++; CodeGenerator.MAX_SWITCH_CASE_LABELS = Integer.parseInt(args[i]); } } else if (args[i].equals("-Xminswitchalts")) { if (i + 1 >= args.length) { System.err.println("missing min switch alternatives -Xminswitchalts option; ignoring"); } else { i++; CodeGenerator.MIN_SWITCH_ALTS = Integer.parseInt(args[i]); } } else if (args[i].equals("-Xm")) { if (i + 1 >= args.length) { System.err.println("missing max recursion with -Xm option; ignoring"); } else { i++; NFAContext.MAX_SAME_RULE_INVOCATIONS_PER_NFA_CONFIG_STACK = Integer.parseInt(args[i]); } } else if (args[i].equals("-Xmaxdfaedges")) { if (i + 1 >= args.length) { System.err.println("missing max number of edges with -Xmaxdfaedges option; ignoring"); } else { i++; DFA.MAX_STATE_TRANSITIONS_FOR_TABLE = Integer.parseInt(args[i]); } } else if (args[i].equals("-Xconversiontimeout")) { if (i + 1 >= args.length) { System.err.println("missing max time in ms -Xconversiontimeout option; ignoring"); } else { i++; DFA.MAX_TIME_PER_DFA_CREATION = Integer.parseInt(args[i]); } } else if (args[i].equals("-Xnfastates")) { DecisionProbe.verbose = true; } else if (args[i].equals("-X")) { Xhelp(); } else { if (args[i].charAt(0) != '-') { // Must be the grammar file addGrammarFile(args[i]); } } } }
/** * Set the message format to one of ANTLR, gnu, vs2005 * * @param format */ public void setMessageFormat(String format) { ErrorManager.setFormat(format); }
/** * Returns the number of errors that the analysis/processing threw up. * * @return Error count */ public int getNumErrors() { return ErrorManager.getNumErrors(); }
/** * Returns the current setting of the message format descriptor * * @return Current message format */ public String getMessageFormat() { return ErrorManager.getMessageFormat().toString(); }