/**
* Called right after the creation of any token.
*
* <p>We may need to add the special tokens found while parsing to it and add dedents (otherwise
* the grammar won't finish successfully if we don't have a new line in the end).
*/
protected final void CommonTokenAction(final Token initial) {
Token t = initial;
int i = specialTokens.size();
while (t.specialToken != null) {
this.specialTokens.add(i, t.specialToken);
t = t.specialToken;
}
// Now, we must check the actual token here for EOF.
t = initial;
// This is the place we check if we have to add dedents so that the parsing ends 'gracefully'
// when
// we find and EOF.
if (t.kind == getEofId()) {
// Store it because if we backtrack we have to restore it!!
this.levelBeforeEof = level;
if (getCurLexState() == getLexerDefaultId()) {
t.kind = getNewlineId();
} else {
t.kind = getDedentId();
if (level >= 0) level -= 1;
}
while (level >= 0) {
level--;
t = addDedent(t);
}
t.kind = getEofId();
t.image = "<EOF>";
}
}
/**
* Creates a new token of the given kind (with the given image) in the same position as the
* previous token passed (note that even endline and endColumn are in the same position... even if
* that'd be strange, that's what we want because those positions can be used later and we need
* that specific point in the grammar)
*/
protected final Token createFrom(Token previous, int kind, String image) {
Token t = new Token();
t.kind = kind;
t.beginLine = previous.beginLine;
t.endLine = previous.endLine;
t.beginColumn = previous.beginColumn;
t.endColumn = previous.endColumn;
t.image = image;
t.specialToken = null;
t.next = null;
return t;
}