Example #1
0
  /**
   * Called right after the creation of any token.
   *
   * <p>We may need to add the special tokens found while parsing to it and add dedents (otherwise
   * the grammar won't finish successfully if we don't have a new line in the end).
   */
  protected final void CommonTokenAction(final Token initial) {
    Token t = initial;

    int i = specialTokens.size();
    while (t.specialToken != null) {
      this.specialTokens.add(i, t.specialToken);
      t = t.specialToken;
    }

    // Now, we must check the actual token here for EOF.
    t = initial;

    // This is the place we check if we have to add dedents so that the parsing ends 'gracefully'
    // when
    // we find and EOF.
    if (t.kind == getEofId()) {
      // Store it because if we backtrack we have to restore it!!
      this.levelBeforeEof = level;
      if (getCurLexState() == getLexerDefaultId()) {
        t.kind = getNewlineId();
      } else {
        t.kind = getDedentId();
        if (level >= 0) level -= 1;
      }
      while (level >= 0) {
        level--;
        t = addDedent(t);
      }
      t.kind = getEofId();
      t.image = "<EOF>";
    }
  }
Example #2
0
 /**
  * Creates a new token of the given kind (with the given image) in the same position as the
  * previous token passed (note that even endline and endColumn are in the same position... even if
  * that'd be strange, that's what we want because those positions can be used later and we need
  * that specific point in the grammar)
  */
 protected final Token createFrom(Token previous, int kind, String image) {
   Token t = new Token();
   t.kind = kind;
   t.beginLine = previous.beginLine;
   t.endLine = previous.endLine;
   t.beginColumn = previous.beginColumn;
   t.endColumn = previous.endColumn;
   t.image = image;
   t.specialToken = null;
   t.next = null;
   return t;
 }