public IToken nextToken() {
fCurrentOffset += fCurrentLength;
if (fLexerToken == null) {
fLexerToken = fLexer.next();
}
fCurrentLength = fLexer.getOffset() - fCurrentOffset;
if (fCurrentLength != 0) {
return fDefaultToken;
}
fCurrentLength = fLexer.getLength();
return getTokenFromScannerToken();
}
@Override
public void setRange(final IDocument document, final int offset, final int length) {
super.setRange(document, offset, length);
fCurrentOffset = offset;
fCurrentLength = 0;
fLexer.reset(this);
}
protected IToken getTokenFromScannerToken() {
IToken token;
if (fLexerToken == RTerminal.SYMBOL) {
final String text = fLexer.getText();
if (text != null) {
token = fSpecialSymbols.get(text);
if (token != null) {
fLexerToken = null;
return token;
}
}
fLexerToken = null;
return fDefaultToken;
}
token = fTokens.get(fLexerToken);
fLexerToken = null;
return token;
}
