Exemple #1
0
  protected void performGrammarAnalysis(CodeGenerator generator, Grammar grammar) {
    // Build NFAs from the grammar AST
    grammar.buildNFA();

    // Create the DFA predictors for each decision
    grammar.createLookaheadDFAs();
  }
Exemple #2
0
  /**
   * Create NFA, DFA and generate code for grammar. Create NFA for any delegates first. Once all NFA
   * are created, it's ok to create DFA, which must check for left-recursion. That check is done by
   * walking the full NFA, which therefore must be complete. After all NFA, comes DFA conversion for
   * root grammar then code gen for root grammar. DFA and code gen for delegates comes next.
   */
  protected void generateRecognizer(Grammar grammar) {
    String language = (String) grammar.getOption("language");
    if (language != null) {
      CodeGenerator generator = new CodeGenerator(this, grammar, language);
      grammar.setCodeGenerator(generator);
      generator.setDebug(isDebug());
      generator.setProfile(isProfile());
      generator.setTrace(isTrace());

      // generate NFA early in case of crash later (for debugging)
      if (isGenerate_NFA_dot()) {
        generateNFAs(grammar);
      }

      // GENERATE CODE
      generator.genRecognizer();

      if (isGenerate_DFA_dot()) {
        generateDFAs(grammar);
      }

      List<Grammar> delegates = grammar.getDirectDelegates();
      for (int i = 0; delegates != null && i < delegates.size(); i++) {
        Grammar delegate = (Grammar) delegates.get(i);
        if (delegate != grammar) { // already processing this one
          generateRecognizer(delegate);
        }
      }
    }
  }
Exemple #3
0
  /**
   * This method is used by all code generators to create new output files. If the outputDir set by
   * -o is not present it will be created. The final filename is sensitive to the output directory
   * and the directory where the grammar file was found. If -o is /tmp and the original grammar file
   * was foo/t.g then output files go in /tmp/foo.
   *
   * <p>The output dir -o spec takes precedence if it's absolute. E.g., if the grammar file dir is
   * absolute the output dir is given precendence. "-o /tmp /usr/lib/t.g" results in "/tmp/T.java"
   * as output (assuming t.g holds T.java).
   *
   * <p>If no -o is specified, then just write to the directory where the grammar file was found.
   *
   * <p>If outputDirectory==null then write a String.
   */
  public Writer getOutputFile(Grammar g, String fileName) throws IOException {
    if (getOutputDirectory() == null) {
      return new StringWriter();
    }
    // output directory is a function of where the grammar file lives
    // for subdir/T.g, you get subdir here.  Well, depends on -o etc...
    // But, if this is a .tokens file, then we force the output to
    // be the base output directory (or current directory if there is not a -o)
    //
    File outputDir;
    if (fileName.endsWith(CodeGenerator.VOCAB_FILE_EXTENSION)) {
      if (haveOutputDir) {
        outputDir = new File(getOutputDirectory());
      } else {
        outputDir = new File(".");
      }
    } else {
      outputDir = getOutputDirectory(g.getFileName());
    }
    File outputFile = new File(outputDir, fileName);

    if (!outputDir.exists()) {
      outputDir.mkdirs();
    }
    FileWriter fw = new FileWriter(outputFile);
    return new BufferedWriter(fw);
  }
Exemple #4
0
  /**
   * Convert from an ANTLR char literal found in a grammar file to an equivalent char literal in the
   * target language. For most languages, this means leaving 'x' as 'x'. Actually, we need to escape
   * '\u000A' so that it doesn't get converted to \n by the compiler. Convert the literal to the
   * char value and then to an appropriate target char literal.
   *
   * <p>Expect single quotes around the incoming literal.
   */
  public String getTargetCharLiteralFromANTLRCharLiteral(CodeGenerator generator, String literal) {
    StringBuffer buf = new StringBuffer();
    buf.append('\'');
    int c = Grammar.getCharValueFromGrammarCharLiteral(literal);
    if (c < Label.MIN_CHAR_VALUE) {
      return "'\u0000'";
    }
    if (c < targetCharValueEscape.length && targetCharValueEscape[c] != null) {
      buf.append(targetCharValueEscape[c]);
    } else if (Character.UnicodeBlock.of((char) c) == Character.UnicodeBlock.BASIC_LATIN
        && !Character.isISOControl((char) c)) {
      // normal char
      buf.append((char) c);
    } else {
      // must be something unprintable...use \\uXXXX
      // turn on the bit above max "\\uFFFF" value so that we pad with zeros
      // then only take last 4 digits
      String hex = Integer.toHexString(c | 0x10000).toUpperCase().substring(1, 5);
      buf.append("\\u");
      buf.append(hex);
    }

    buf.append('\'');
    return buf.toString();
  }
Exemple #5
0
  protected void generateNFAs(Grammar g) {
    DOTGenerator dotGenerator = new DOTGenerator(g);
    Collection rules = g.getAllImportedRules();
    rules.addAll(g.getRules());

    for (Iterator itr = rules.iterator(); itr.hasNext(); ) {
      Rule r = (Rule) itr.next();
      try {
        String dot = dotGenerator.getDOT(r.startState);
        if (dot != null) {
          writeDOTFile(g, r, dot);
        }
      } catch (IOException ioe) {
        ErrorManager.error(ErrorManager.MSG_CANNOT_WRITE_FILE, ioe);
      }
    }
  }
Exemple #6
0
 public void generateDFAs(Grammar g) {
   for (int d = 1; d <= g.getNumberOfDecisions(); d++) {
     DFA dfa = g.getLookaheadDFA(d);
     if (dfa == null) {
       continue; // not there for some reason, ignore
     }
     DOTGenerator dotGenerator = new DOTGenerator(g);
     String dot = dotGenerator.getDOT(dfa.startState);
     String dotFileName = g.name + "." + "dec-" + d;
     if (g.implicitLexer) {
       dotFileName = g.name + Grammar.grammarTypeToFileNameSuffix[g.type] + "." + "dec-" + d;
     }
     try {
       writeDOTFile(g, dotFileName, dot);
     } catch (IOException ioe) {
       ErrorManager.error(ErrorManager.MSG_CANNOT_GEN_DOT_FILE, dotFileName, ioe);
     }
   }
 }
Exemple #7
0
  /** Get a grammar mentioned on the command-line and any delegates */
  public Grammar getRootGrammar(String grammarFileName) throws IOException {
    // StringTemplate.setLintMode(true);
    // grammars mentioned on command line are either roots or single grammars.
    // create the necessary composite in case it's got delegates; even
    // single grammar needs it to get token types.
    CompositeGrammar composite = new CompositeGrammar();
    Grammar grammar = new Grammar(this, grammarFileName, composite);
    composite.setDelegationRoot(grammar);
    FileReader fr = null;
    File f = null;

    if (haveInputDir) {
      f = new File(inputDirectory, grammarFileName);
    } else {
      f = new File(grammarFileName);
    }

    // Store the location of this grammar as if we import files, we can then
    // search for imports in the same location as the original grammar as well as in
    // the lib directory.
    //
    parentGrammarDirectory = f.getParent();

    if (grammarFileName.lastIndexOf(File.separatorChar) == -1) {
      grammarOutputDirectory = ".";
    } else {
      grammarOutputDirectory =
          grammarFileName.substring(0, grammarFileName.lastIndexOf(File.separatorChar));
    }
    fr = new FileReader(f);
    BufferedReader br = new BufferedReader(fr);
    grammar.parseAndBuildAST(br);
    composite.watchNFAConversion = internalOption_watchNFAConversion;
    br.close();
    fr.close();
    return grammar;
  }
Exemple #8
0
  @Override
  public String getTargetCharLiteralFromANTLRCharLiteral(
      final CodeGenerator generator, final String literal) {
    final StringBuffer buf = new StringBuffer(10);

    final int c = Grammar.getCharValueFromGrammarCharLiteral(literal);
    if (c < Label.MIN_CHAR_VALUE) {
      buf.append("\\x{0000}");
    } else if (c < targetCharValueEscape.length && targetCharValueEscape[c] != null) {
      buf.append(targetCharValueEscape[c]);
    } else if (Character.UnicodeBlock.of((char) c) == Character.UnicodeBlock.BASIC_LATIN
        && !Character.isISOControl((char) c)) {
      // normal char
      buf.append((char) c);
    } else {
      // must be something unprintable...use \\uXXXX
      // turn on the bit above max "\\uFFFF" value so that we pad with zeros
      // then only take last 4 digits
      String hex = Integer.toHexString(c | 0x10000).toUpperCase().substring(1, 5);
      buf.append("\\x{");
      buf.append(hex);
      buf.append("}");
    }

    if (buf.indexOf("\\") == -1) {
      // no need for interpolation, use single quotes
      buf.insert(0, '\'');
      buf.append('\'');
    } else {
      // need string interpolation
      buf.insert(0, '\"');
      buf.append('\"');
    }

    return buf.toString();
  }
Exemple #9
0
  public void process() {
    boolean exceptionWhenWritingLexerFile = false;
    String lexerGrammarFileName = null; // necessary at this scope to have access in the catch below

    // Have to be tricky here when Maven or build tools call in and must new Tool()
    // before setting options. The banner won't display that way!
    if (isVerbose() && showBanner) {
      ErrorManager.info("ANTLR Parser Generator  Version " + VERSION);
      showBanner = false;
    }

    try {
      sortGrammarFiles(); // update grammarFileNames
    } catch (Exception e) {
      ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e);
    } catch (Error e) {
      ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e);
    }

    for (String grammarFileName : grammarFileNames) {
      // If we are in make mode (to support build tools like Maven) and the
      // file is already up to date, then we do not build it (and in verbose mode
      // we will say so).
      if (make) {
        try {
          if (!buildRequired(grammarFileName)) continue;
        } catch (Exception e) {
          ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, e);
        }
      }

      if (isVerbose() && !isDepend()) {
        System.out.println(grammarFileName);
      }
      try {
        if (isDepend()) {
          BuildDependencyGenerator dep = new BuildDependencyGenerator(this, grammarFileName);
          /*
          List outputFiles = dep.getGeneratedFileList();
          List dependents = dep.getDependenciesFileList();
          System.out.println("output: "+outputFiles);
          System.out.println("dependents: "+dependents);
           */
          System.out.println(dep.getDependencies());
          continue;
        }

        Grammar grammar = getRootGrammar(grammarFileName);
        // we now have all grammars read in as ASTs
        // (i.e., root and all delegates)
        grammar.composite.assignTokenTypes();
        grammar.composite.defineGrammarSymbols();
        grammar.composite.createNFAs();

        generateRecognizer(grammar);

        if (isPrintGrammar()) {
          grammar.printGrammar(System.out);
        }

        if (isReport()) {
          GrammarReport greport = new GrammarReport(grammar);
          System.out.println(greport.toString());
          // print out a backtracking report too (that is not encoded into log)
          System.out.println(greport.getBacktrackingReport());
          // same for aborted NFA->DFA conversions
          System.out.println(greport.getAnalysisTimeoutReport());
        }
        if (isProfile()) {
          GrammarReport greport = new GrammarReport(grammar);
          Stats.writeReport(GrammarReport.GRAMMAR_STATS_FILENAME, greport.toNotifyString());
        }

        // now handle the lexer if one was created for a merged spec
        String lexerGrammarStr = grammar.getLexerGrammar();
        // System.out.println("lexer grammar:\n"+lexerGrammarStr);
        if (grammar.type == Grammar.COMBINED && lexerGrammarStr != null) {
          lexerGrammarFileName = grammar.getImplicitlyGeneratedLexerFileName();
          try {
            Writer w = getOutputFile(grammar, lexerGrammarFileName);
            w.write(lexerGrammarStr);
            w.close();
          } catch (IOException e) {
            // emit different error message when creating the implicit lexer fails
            // due to write permission error
            exceptionWhenWritingLexerFile = true;
            throw e;
          }
          try {
            StringReader sr = new StringReader(lexerGrammarStr);
            Grammar lexerGrammar = new Grammar();
            lexerGrammar.composite.watchNFAConversion = internalOption_watchNFAConversion;
            lexerGrammar.implicitLexer = true;
            lexerGrammar.setTool(this);
            File lexerGrammarFullFile =
                new File(getFileDirectory(lexerGrammarFileName), lexerGrammarFileName);
            lexerGrammar.setFileName(lexerGrammarFullFile.toString());

            lexerGrammar.importTokenVocabulary(grammar);
            lexerGrammar.parseAndBuildAST(sr);

            sr.close();

            lexerGrammar.composite.assignTokenTypes();
            lexerGrammar.composite.defineGrammarSymbols();
            lexerGrammar.composite.createNFAs();

            generateRecognizer(lexerGrammar);
          } finally {
            // make sure we clean up
            if (deleteTempLexer) {
              File outputDir = getOutputDirectory(lexerGrammarFileName);
              File outputFile = new File(outputDir, lexerGrammarFileName);
              outputFile.delete();
            }
          }
        }
      } catch (IOException e) {
        if (exceptionWhenWritingLexerFile) {
          ErrorManager.error(ErrorManager.MSG_CANNOT_WRITE_FILE, lexerGrammarFileName, e);
        } else {
          ErrorManager.error(ErrorManager.MSG_CANNOT_OPEN_FILE, grammarFileName);
        }
      } catch (Exception e) {
        ErrorManager.error(ErrorManager.MSG_INTERNAL_ERROR, grammarFileName, e);
      }
      /*
      finally {
      System.out.println("creates="+ Interval.creates);
      System.out.println("hits="+ Interval.hits);
      System.out.println("misses="+ Interval.misses);
      System.out.println("outOfRange="+ Interval.outOfRange);
      }
       */
    }
  }
Exemple #10
0
 public String getTargetCharLiteralFromANTLRCharLiteral(CodeGenerator generator, String literal) {
   int c = Grammar.getCharValueFromGrammarCharLiteral(literal);
   return String.valueOf(c);
 }