Example #1
0
 public void sortGrammarFiles() throws IOException {
   // System.out.println("Grammar names "+getGrammarFileNames());
   Graph g = new Graph();
   List<String> missingFiles = new ArrayList<String>();
   for (String gfile : grammarFileNames) {
     try {
       GrammarSpelunker grammar = new GrammarSpelunker(inputDirectory, gfile);
       grammar.parse();
       String vocabName = grammar.getTokenVocab();
       String grammarName = grammar.getGrammarName();
       // Make all grammars depend on any tokenVocab options
       if (vocabName != null) g.addEdge(gfile, vocabName + CodeGenerator.VOCAB_FILE_EXTENSION);
       // Make all generated tokens files depend on their grammars
       g.addEdge(grammarName + CodeGenerator.VOCAB_FILE_EXTENSION, gfile);
     } catch (FileNotFoundException fnfe) {
       ErrorManager.error(ErrorManager.MSG_CANNOT_OPEN_FILE, gfile);
       missingFiles.add(gfile);
     }
   }
   List<Object> sorted = g.sort();
   // System.out.println("sorted="+sorted);
   grammarFileNames.clear(); // wipe so we can give new ordered list
   for (int i = 0; i < sorted.size(); i++) {
     String f = (String) sorted.get(i);
     if (missingFiles.contains(f)) continue;
     if (!(f.endsWith(".g") || f.endsWith(".g3"))) continue;
     grammarFileNames.add(f);
   }
   // System.out.println("new grammars="+grammarFileNames);
 }
Example #2
0
  /**
   * A list of dependency generators that are accumulated aaaas (and if) the tool is required to
   * sort the provided grammars into build dependency order. protected Map<String,
   * BuildDependencyGenerator> buildDependencyGenerators;
   */
  public static void main(String[] args) {
    Tool antlr = new Tool(args);

    if (!exitNow) {
      antlr.process();
      if (ErrorManager.getNumErrors() > 0) {
        System.exit(1);
      }
      System.exit(0);
    }
  }
 @Before
 public void setUp() throws Exception {
   lastTestFailed = false; // hope for the best, but set to true in asserts that fail
   // new output dir for each test
   tmpdir =
       new File(
               System.getProperty("java.io.tmpdir"),
               "antlr-" + getClass().getName() + "-" + System.currentTimeMillis())
           .getAbsolutePath();
   ErrorManager.resetErrorState();
 }
Example #4
0
  protected void generateNFAs(Grammar g) {
    DOTGenerator dotGenerator = new DOTGenerator(g);
    Collection rules = g.getAllImportedRules();
    rules.addAll(g.getRules());

    for (Iterator itr = rules.iterator(); itr.hasNext(); ) {
      Rule r = (Rule) itr.next();
      try {
        String dot = dotGenerator.getDOT(r.startState);
        if (dot != null) {
          writeDOTFile(g, r, dot);
        }
      } catch (IOException ioe) {
        ErrorManager.error(ErrorManager.MSG_CANNOT_WRITE_FILE, ioe);
      }
    }
  }
 /** Return true if all is ok, no errors */
 protected boolean antlr(
     String fileName, String grammarFileName, String grammarStr, boolean debug) {
   boolean allIsWell = true;
   mkdir(tmpdir);
   writeFile(tmpdir, fileName, grammarStr);
   try {
     final List options = new ArrayList();
     if (debug) {
       options.add("-debug");
     }
     options.add("-o");
     options.add(tmpdir);
     options.add("-lib");
     options.add(tmpdir);
     options.add(new File(tmpdir, grammarFileName).toString());
     final String[] optionsA = new String[options.size()];
     options.toArray(optionsA);
     /*
     final ErrorQueue equeue = new ErrorQueue();
     ErrorManager.setErrorListener(equeue);
     */
     Tool antlr = newTool(optionsA);
     antlr.process();
     ANTLRErrorListener listener = ErrorManager.getErrorListener();
     if (listener instanceof ErrorQueue) {
       ErrorQueue equeue = (ErrorQueue) listener;
       if (equeue.errors.size() > 0) {
         allIsWell = false;
         System.err.println("antlr reports errors from " + options);
         for (int i = 0; i < equeue.errors.size(); i++) {
           Message msg = (Message) equeue.errors.get(i);
           System.err.println(msg);
         }
         System.out.println("!!!\ngrammar:");
         System.out.println(grammarStr);
         System.out.println("###");
       }
     }
   } catch (Exception e) {
     allIsWell = false;
     System.err.println("problems building grammar: " + e);
     e.printStackTrace(System.err);
   }
   return allIsWell;
 }
Example #6
0
 public void generateDFAs(Grammar g) {
   for (int d = 1; d <= g.getNumberOfDecisions(); d++) {
     DFA dfa = g.getLookaheadDFA(d);
     if (dfa == null) {
       continue; // not there for some reason, ignore
     }
     DOTGenerator dotGenerator = new DOTGenerator(g);
     String dot = dotGenerator.getDOT(dfa.startState);
     String dotFileName = g.name + "." + "dec-" + d;
     if (g.implicitLexer) {
       dotFileName = g.name + Grammar.grammarTypeToFileNameSuffix[g.type] + "." + "dec-" + d;
     }
     try {
       writeDOTFile(g, dotFileName, dot);
     } catch (IOException ioe) {
       ErrorManager.error(ErrorManager.MSG_CANNOT_GEN_DOT_FILE, dotFileName, ioe);
     }
   }
 }
 protected void checkError(ErrorQueue equeue, Message expectedMessage) throws Exception {
   // System.out.println("errors="+equeue);
   Message foundMsg = null;
   for (int i = 0; i < equeue.errors.size(); i++) {
     Message m = (Message) equeue.errors.get(i);
     if (m.msgID == expectedMessage.msgID) {
       foundMsg = m;
     }
   }
   assertTrue("no error; " + expectedMessage.msgID + " expected", equeue.errors.size() > 0);
   assertTrue("too many errors; " + equeue.errors, equeue.errors.size() <= 1);
   assertNotNull("couldn't find expected error: " + expectedMessage.msgID, foundMsg);
   /*
   assertTrue("error is not a GrammarSemanticsMessage",
              foundMsg instanceof GrammarSemanticsMessage);
    */
   assertEquals(expectedMessage.arg, foundMsg.arg);
   assertEquals(expectedMessage.arg2, foundMsg.arg2);
   ErrorManager.resetErrorState(); // wack errors for next test
 }
Example #8
0
 private static void Xhelp() {
   ErrorManager.info("ANTLR Parser Generator  Version " + new Tool().VERSION);
   System.err.println("  -Xgrtree                print the grammar AST");
   System.err.println("  -Xdfa                   print DFA as text ");
   System.err.println("  -Xnoprune               test lookahead against EBNF block exit branches");
   System.err.println("  -Xnocollapse            collapse incident edges into DFA states");
   System.err.println("  -Xdbgconversion         dump lots of info during NFA conversion");
   System.err.println("  -Xmultithreaded         run the analysis in 2 threads");
   System.err.println("  -Xnomergestopstates     do not merge stop states");
   System.err.println("  -Xdfaverbose            generate DFA states in DOT with NFA configs");
   System.err.println("  -Xwatchconversion       print a message for each NFA before converting");
   System.err.println(
       "  -XdbgST                 put tags at start/stop of all templates in output");
   System.err.println(
       "  -Xnfastates             for nondeterminisms, list NFA states for each path");
   System.err.println(
       "  -Xm m                   max number of rule invocations during conversion           ["
           + NFAContext.MAX_SAME_RULE_INVOCATIONS_PER_NFA_CONFIG_STACK
           + "]");
   System.err.println(
       "  -Xmaxdfaedges m         max \"comfortable\" number of edges for single DFA state     ["
           + DFA.MAX_STATE_TRANSITIONS_FOR_TABLE
           + "]");
   System.err.println(
       "  -Xconversiontimeout t   set NFA conversion timeout (ms) for each decision          ["
           + DFA.MAX_TIME_PER_DFA_CREATION
           + "]");
   System.err.println(
       "  -Xmaxinlinedfastates m  max DFA states before table used rather than inlining      ["
           + CodeGenerator.MADSI_DEFAULT
           + "]");
   System.err.println(
       "  -Xmaxswitchcaselabels m don't generate switch() statements for dfas bigger  than m ["
           + CodeGenerator.MSCL_DEFAULT
           + "]");
   System.err.println(
       "  -Xminswitchalts m       don't generate switch() statements for dfas smaller than m ["
           + CodeGenerator.MSA_DEFAULT
           + "]");
 }
Example #9
0
 private static void help() {
   ErrorManager.info("ANTLR Parser Generator  Version " + new Tool().VERSION);
   System.err.println("usage: java org.antlr.Tool [args] file.g [file2.g file3.g ...]");
   System.err.println(
       "  -o outputDir          specify output directory where all output is generated");
   System.err.println(
       "  -fo outputDir         same as -o but force even files with relative paths to dir");
   System.err.println("  -lib dir              specify location of token files");
   System.err.println("  -depend               generate file dependencies");
   System.err.println("  -report               print out a report about the grammar(s) processed");
   System.err.println("  -print                print out the grammar without actions");
   System.err.println("  -debug                generate a parser that emits debugging events");
   System.err.println(
       "  -profile              generate a parser that computes profiling information");
   System.err.println("  -nfa                  generate an NFA for each rule");
   System.err.println("  -dfa                  generate a DFA for each decision point");
   System.err.println("  -message-format name  specify output style for messages");
   System.err.println("  -verbose              generate ANTLR version and other information");
   System.err.println("  -make                 only build if generated files older than grammar");
   System.err.println("  -version              print the version of ANTLR and exit.");
   System.err.println("  -X                    display extended argument list");
 }
Example #10
0
 private static void version() {
   ErrorManager.info("ANTLR Parser Generator  Version " + new Tool().VERSION);
 }
Example #11
0
  public void process() {
    boolean exceptionWhenWritingLexerFile = false;
    String lexerGrammarFileName = null; // necessary at this scope to have access in the catch below

    // Have to be tricky here when Maven or build tools call in and must new Tool()
    // before setting options. The banner won't display that way!
    if (isVerbose() && showBanner) {
      ErrorManager.info("ANTLR Parser Generator  Version " + VERSION);
      showBanner = false;
    }

    try {
      sortGrammarFiles(); // update grammarFileNames
    } catch (Exception e) {
      ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e);
    } catch (Error e) {
      ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e);
    }

    for (String grammarFileName : grammarFileNames) {
      // If we are in make mode (to support build tools like Maven) and the
      // file is already up to date, then we do not build it (and in verbose mode
      // we will say so).
      if (make) {
        try {
          if (!buildRequired(grammarFileName)) continue;
        } catch (Exception e) {
          ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e);
        }
      }

      if (isVerbose() && !isDepend()) {
        System.out.println(grammarFileName);
      }
      try {
        if (isDepend()) {
          BuildDependencyGenerator dep = new BuildDependencyGenerator(this, grammarFileName);
          /*
          List outputFiles = dep.getGeneratedFileList();
          List dependents = dep.getDependenciesFileList();
          System.out.println("output: "+outputFiles);
          System.out.println("dependents: "+dependents);
           */
          System.out.println(dep.getDependencies());
          continue;
        }

        Grammar grammar = getRootGrammar(grammarFileName);
        // we now have all grammars read in as ASTs
        // (i.e., root and all delegates)
        grammar.composite.assignTokenTypes();
        grammar.composite.defineGrammarSymbols();
        grammar.composite.createNFAs();

        generateRecognizer(grammar);

        if (isPrintGrammar()) {
          grammar.printGrammar(System.out);
        }

        if (isReport()) {
          GrammarReport greport = new GrammarReport(grammar);
          System.out.println(greport.toString());
          // print out a backtracking report too (that is not encoded into log)
          System.out.println(greport.getBacktrackingReport());
          // same for aborted NFA->DFA conversions
          System.out.println(greport.getAnalysisTimeoutReport());
        }
        if (isProfile()) {
          GrammarReport greport = new GrammarReport(grammar);
          Stats.writeReport(GrammarReport.GRAMMAR_STATS_FILENAME, greport.toNotifyString());
        }

        // now handle the lexer if one was created for a merged spec
        String lexerGrammarStr = grammar.getLexerGrammar();
        // System.out.println("lexer grammar:\n"+lexerGrammarStr);
        if (grammar.type == Grammar.COMBINED && lexerGrammarStr != null) {
          lexerGrammarFileName = grammar.getImplicitlyGeneratedLexerFileName();
          try {
            Writer w = getOutputFile(grammar, lexerGrammarFileName);
            w.write(lexerGrammarStr);
            w.close();
          } catch (IOException e) {
            // emit different error message when creating the implicit lexer fails
            // due to write permission error
            exceptionWhenWritingLexerFile = true;
            throw e;
          }
          try {
            StringReader sr = new StringReader(lexerGrammarStr);
            Grammar lexerGrammar = new Grammar();
            lexerGrammar.composite.watchNFAConversion = internalOption_watchNFAConversion;
            lexerGrammar.implicitLexer = true;
            lexerGrammar.setTool(this);
            File lexerGrammarFullFile =
                new File(getFileDirectory(lexerGrammarFileName), lexerGrammarFileName);
            lexerGrammar.setFileName(lexerGrammarFullFile.toString());

            lexerGrammar.importTokenVocabulary(grammar);
            lexerGrammar.parseAndBuildAST(sr);

            sr.close();

            lexerGrammar.composite.assignTokenTypes();
            lexerGrammar.composite.defineGrammarSymbols();
            lexerGrammar.composite.createNFAs();

            generateRecognizer(lexerGrammar);
          } finally {
            // make sure we clean up
            if (deleteTempLexer) {
              File outputDir = getOutputDirectory(lexerGrammarFileName);
              File outputFile = new File(outputDir, lexerGrammarFileName);
              outputFile.delete();
            }
          }
        }
      } catch (IOException e) {
        if (exceptionWhenWritingLexerFile) {
          ErrorManager.error(ErrorManager.MSG_CANNOT_WRITE_FILE, lexerGrammarFileName, e);
        } else {
          ErrorManager.error(ErrorManager.MSG_CANNOT_OPEN_FILE, grammarFileName);
        }
      } catch (Exception e) {
        ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, grammarFileName, e);
      }
      /*
      finally {
      System.out.println("creates="+ Interval.creates);
      System.out.println("hits="+ Interval.hits);
      System.out.println("misses="+ Interval.misses);
      System.out.println("outOfRange="+ Interval.outOfRange);
      }
       */
    }
  }
Example #12
0
  public void processArgs(String[] args) {

    if (isVerbose()) {
      ErrorManager.info("ANTLR Parser Generator  Version " + VERSION);
      showBanner = false;
    }

    if (args == null || args.length == 0) {
      help();
      return;
    }
    for (int i = 0; i < args.length; i++) {
      if (args[i].equals("-o") || args[i].equals("-fo")) {
        if (i + 1 >= args.length) {
          System.err.println("missing output directory with -fo/-o option; ignoring");
        } else {
          if (args[i].equals("-fo")) { // force output into dir
            setForceAllFilesToOutputDir(true);
          }
          i++;
          outputDirectory = args[i];
          if (outputDirectory.endsWith("/") || outputDirectory.endsWith("\\")) {
            outputDirectory = outputDirectory.substring(0, getOutputDirectory().length() - 1);
          }
          File outDir = new File(outputDirectory);
          haveOutputDir = true;
          if (outDir.exists() && !outDir.isDirectory()) {
            ErrorManager.error(ErrorManager.MSG_OUTPUT_DIR_IS_FILE, outputDirectory);
            setLibDirectory(".");
          }
        }
      } else if (args[i].equals("-lib")) {
        if (i + 1 >= args.length) {
          System.err.println("missing library directory with -lib option; ignoring");
        } else {
          i++;
          setLibDirectory(args[i]);
          if (getLibraryDirectory().endsWith("/") || getLibraryDirectory().endsWith("\\")) {
            setLibDirectory(getLibraryDirectory().substring(0, getLibraryDirectory().length() - 1));
          }
          File outDir = new File(getLibraryDirectory());
          if (!outDir.exists()) {
            ErrorManager.error(ErrorManager.MSG_DIR_NOT_FOUND, getLibraryDirectory());
            setLibDirectory(".");
          }
        }
      } else if (args[i].equals("-nfa")) {
        setGenerate_NFA_dot(true);
      } else if (args[i].equals("-dfa")) {
        setGenerate_DFA_dot(true);
      } else if (args[i].equals("-debug")) {
        setDebug(true);
      } else if (args[i].equals("-trace")) {
        setTrace(true);
      } else if (args[i].equals("-report")) {
        setReport(true);
      } else if (args[i].equals("-profile")) {
        setProfile(true);
      } else if (args[i].equals("-print")) {
        setPrintGrammar(true);
      } else if (args[i].equals("-depend")) {
        setDepend(true);
      } else if (args[i].equals("-verbose")) {
        setVerbose(true);
      } else if (args[i].equals("-version")) {
        version();
        exitNow = true;
      } else if (args[i].equals("-make")) {
        setMake(true);
      } else if (args[i].equals("-message-format")) {
        if (i + 1 >= args.length) {
          System.err.println("missing output format with -message-format option; using default");
        } else {
          i++;
          ErrorManager.setFormat(args[i]);
        }
      } else if (args[i].equals("-Xgrtree")) {
        internalOption_PrintGrammarTree = true; // print grammar tree
      } else if (args[i].equals("-Xdfa")) {
        internalOption_PrintDFA = true;
      } else if (args[i].equals("-Xnoprune")) {
        DFAOptimizer.PRUNE_EBNF_EXIT_BRANCHES = false;
      } else if (args[i].equals("-Xnocollapse")) {
        DFAOptimizer.COLLAPSE_ALL_PARALLEL_EDGES = false;
      } else if (args[i].equals("-Xdbgconversion")) {
        NFAToDFAConverter.debug = true;
      } else if (args[i].equals("-Xmultithreaded")) {
        NFAToDFAConverter.SINGLE_THREADED_NFA_CONVERSION = false;
      } else if (args[i].equals("-Xnomergestopstates")) {
        DFAOptimizer.MERGE_STOP_STATES = false;
      } else if (args[i].equals("-Xdfaverbose")) {
        internalOption_ShowNFAConfigsInDFA = true;
      } else if (args[i].equals("-Xwatchconversion")) {
        internalOption_watchNFAConversion = true;
      } else if (args[i].equals("-XdbgST")) {
        CodeGenerator.EMIT_TEMPLATE_DELIMITERS = true;
      } else if (args[i].equals("-Xmaxinlinedfastates")) {
        if (i + 1 >= args.length) {
          System.err.println("missing max inline dfa states -Xmaxinlinedfastates option; ignoring");
        } else {
          i++;
          CodeGenerator.MAX_ACYCLIC_DFA_STATES_INLINE = Integer.parseInt(args[i]);
        }
      } else if (args[i].equals("-Xmaxswitchcaselabels")) {
        if (i + 1 >= args.length) {
          System.err.println(
              "missing max switch case labels -Xmaxswitchcaselabels option; ignoring");
        } else {
          i++;
          CodeGenerator.MAX_SWITCH_CASE_LABELS = Integer.parseInt(args[i]);
        }
      } else if (args[i].equals("-Xminswitchalts")) {
        if (i + 1 >= args.length) {
          System.err.println("missing min switch alternatives -Xminswitchalts option; ignoring");
        } else {
          i++;
          CodeGenerator.MIN_SWITCH_ALTS = Integer.parseInt(args[i]);
        }
      } else if (args[i].equals("-Xm")) {
        if (i + 1 >= args.length) {
          System.err.println("missing max recursion with -Xm option; ignoring");
        } else {
          i++;
          NFAContext.MAX_SAME_RULE_INVOCATIONS_PER_NFA_CONFIG_STACK = Integer.parseInt(args[i]);
        }
      } else if (args[i].equals("-Xmaxdfaedges")) {
        if (i + 1 >= args.length) {
          System.err.println("missing max number of edges with -Xmaxdfaedges option; ignoring");
        } else {
          i++;
          DFA.MAX_STATE_TRANSITIONS_FOR_TABLE = Integer.parseInt(args[i]);
        }
      } else if (args[i].equals("-Xconversiontimeout")) {
        if (i + 1 >= args.length) {
          System.err.println("missing max time in ms -Xconversiontimeout option; ignoring");
        } else {
          i++;
          DFA.MAX_TIME_PER_DFA_CREATION = Integer.parseInt(args[i]);
        }
      } else if (args[i].equals("-Xnfastates")) {
        DecisionProbe.verbose = true;
      } else if (args[i].equals("-X")) {
        Xhelp();
      } else {
        if (args[i].charAt(0) != '-') {
          // Must be the grammar file
          addGrammarFile(args[i]);
        }
      }
    }
  }
Example #13
0
 /**
  * Set the message format to one of ANTLR, gnu, vs2005
  *
  * @param format
  */
 public void setMessageFormat(String format) {
   ErrorManager.setFormat(format);
 }
Example #14
0
 /**
  * Returns the number of errors that the analysis/processing threw up.
  *
  * @return Error count
  */
 public int getNumErrors() {
   return ErrorManager.getNumErrors();
 }
Example #15
0
 /**
  * Returns the current setting of the message format descriptor
  *
  * @return Current message format
  */
 public String getMessageFormat() {
   return ErrorManager.getMessageFormat().toString();
 }