public String filterMethod(String compilationUnit, String className, String methodNameAndParams) {
final String methodName = extractMethodName(methodNameAndParams);
final String[] params = extractParams(methodNameAndParams);
ANTLRInputStream in = new ANTLRInputStream(compilationUnit);
TokenStream tokenStream = new CommonTokenStream(new Java8Lexer(in));
Java8Parser parser = new Java8Parser(tokenStream);
MethodFilterListener filterListener = new MethodFilterListener(className, methodName, params);
parser.addParseListener(filterListener);
parser.setBuildParseTree(true);
parser.compilationUnit();
if (filterListener.getInterval() == null) {
throw new IllegalArgumentException("Method '" + methodName + "' not found!");
}
final Token startToken = tokenStream.get(filterListener.getInterval().a);
final Token endToken = tokenStream.get(filterListener.getInterval().b);
final int startLine = startToken.getLine();
final int endLine = endToken.getLine();
return getLines(compilationUnit, startLine, endLine);
}
@Test
public void testCompleteBufferAfterConsuming() throws Exception {
LexerGrammar g =
new LexerGrammar(
"lexer grammar t;\n"
+ "ID : 'a'..'z'+;\n"
+ "INT : '0'..'9'+;\n"
+ "SEMI : ';';\n"
+ "ASSIGN : '=';\n"
+ "PLUS : '+';\n"
+ "MULT : '*';\n"
+ "WS : ' '+;\n");
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
LexerInterpreter lexEngine = g.createLexerInterpreter(input);
TokenStream tokens = createTokenStream(lexEngine);
Token t = tokens.LT(1);
while (t.getType() != Token.EOF) {
tokens.consume();
t = tokens.LT(1);
}
String result = tokens.getText();
String expecting = "x = 3 * 0 + 2 * 0;";
assertEquals(expecting, result);
}
protected String getErrorMessage(RecognitionException re) {
String message = "";
Parser recognizer = (Parser) re.getRecognizer();
TokenStream tokens = recognizer.getInputStream();
if (re instanceof NoViableAltException) {
NoViableAltException e = (NoViableAltException) re;
Token startToken = e.getStartToken();
String input =
(startToken.getType() == Token.EOF)
? "end of text"
: quote(tokens.getText(startToken, e.getOffendingToken()));
message = "no viable date format found at " + input;
} else if (re instanceof InputMismatchException) {
InputMismatchException e = (InputMismatchException) re;
message =
"did not expect "
+ getTokenDisplayString(e.getOffendingToken())
+ " while looking for "
+ e.getExpectedTokens().toString(recognizer.getTokenNames());
} else if (re instanceof FailedPredicateException) {
FailedPredicateException e = (FailedPredicateException) re;
String ruleName = recognizer.getRuleNames()[recognizer.getContext().getRuleIndex()];
message = "failed predicate " + ruleName + ": " + e.getMessage();
}
return message;
}
@Test
public void testLookback() throws Exception {
LexerGrammar g =
new LexerGrammar(
"lexer grammar t;\n"
+ "ID : 'a'..'z'+;\n"
+ "INT : '0'..'9'+;\n"
+ "SEMI : ';';\n"
+ "ASSIGN : '=';\n"
+ "PLUS : '+';\n"
+ "MULT : '*';\n"
+ "WS : ' '+;\n");
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
LexerInterpreter lexEngine = g.createLexerInterpreter(input);
TokenStream tokens = createTokenStream(lexEngine);
tokens.consume(); // get x into buffer
Token t = tokens.LT(-1);
assertEquals("x", t.getText());
tokens.consume();
tokens.consume(); // consume '='
t = tokens.LT(-3);
assertEquals("x", t.getText());
t = tokens.LT(-2);
assertEquals(" ", t.getText());
t = tokens.LT(-1);
assertEquals("=", t.getText());
}
/**
* Convert word like {@code *} or {@code ID} or {@code expr} to a path element. {@code anywhere}
* is {@code true} if {@code //} precedes the word.
*/
protected XPathElement getXPathElement(Token wordToken, boolean anywhere) {
if (wordToken.getType() == Token.EOF) {
throw new IllegalArgumentException("Missing path element at end of path");
}
String word = wordToken.getText();
int ttype = parser.getTokenType(word);
int ruleIndex = parser.getRuleIndex(word);
switch (wordToken.getType()) {
case XPathLexer.WILDCARD:
return anywhere ? new XPathWildcardAnywhereElement() : new XPathWildcardElement();
case XPathLexer.TOKEN_REF:
case XPathLexer.STRING:
if (ttype == Token.INVALID_TYPE) {
throw new IllegalArgumentException(
word + " at index " + wordToken.getStartIndex() + " isn't a valid token name");
}
return anywhere
? new XPathTokenAnywhereElement(word, ttype)
: new XPathTokenElement(word, ttype);
default:
if (ruleIndex == -1) {
throw new IllegalArgumentException(
word + " at index " + wordToken.getStartIndex() + " isn't a valid rule name");
}
return anywhere
? new XPathRuleAnywhereElement(word, ruleIndex)
: new XPathRuleElement(word, ruleIndex);
}
}
@Override
public void addToHighlighting(int annexOffset, Token token, String id) {
int offset = ((CommonToken) token).getStartIndex();
int length = token.getText().length();
int column = token.getCharPositionInLine();
_elementToHighlight.add(new AadlBaLocationReference(annexOffset, offset, length, column, id));
}
protected boolean isMultiLineToken(TokenSourceWithStateV4<TState> lexer, Token token) {
/*if (lexer != null && lexer.getLine() > token.getLine()) {
return true;
}*/
int startLine = snapshot.findLineNumber(token.getStartIndex());
int stopLine = snapshot.findLineNumber(token.getStopIndex() + 1);
return startLine != stopLine;
}
public static boolean isEmptyStatement(String sql) {
TokenSource tokens = getLexer(sql, ImmutableSet.<String>of());
while (true) {
Token token = tokens.nextToken();
if (token.getType() == Token.EOF) {
return true;
}
if (token.getChannel() != Token.HIDDEN_CHANNEL) {
return false;
}
}
}
@Override
protected boolean sync(int i) {
if (!super.sync(i)) {
return false;
}
Token t = get(i);
if (hide.contains(t.getType())) {
((WritableToken) t).setChannel(Token.HIDDEN_CHANNEL);
}
return true;
}
protected Collection<TaggedPositionRegion<TokenTag<Token>>> getTagsForToken(Token token) {
TokenTag<Token> tag = highlightToken(token);
if (tag != null) {
return Collections.<TaggedPositionRegion<TokenTag<Token>>>singleton(
new BaseTaggedPositionRegion<>(
new SnapshotPositionRegion(
snapshot,
OffsetRegion.fromBounds(token.getStartIndex(), token.getStopIndex() + 1)),
tag));
}
return Collections.emptyList();
}
private void addElement(Token token, SyntaxElementType type) {
syntaxElements.add(
SyntaxElement.create(
token.getText(),
token.getStartIndex(),
token.getStopIndex(),
token.getTokenIndex(),
type));
if (debug) {
SyntaxElement e = syntaxElements.get(syntaxElements.size() - 1);
System.out.println(String.format("%d-%d %s %s", e.from, e.to, e.type.name(), e.value));
}
}
@Override
@RuleDependency(
recognizer = TemplateParser.class,
rule = TemplateParser.RULE_anonymousTemplateParameters,
version = 0,
dependents = Dependents.PARENTS)
public void enterAnonymousTemplateParameters(AnonymousTemplateParametersContext ctx) {
if (ctx.names != null) {
parameterDeclarations.addAll(ctx.names);
Set<String> currentParameters = parameters.peek();
for (Token token : ctx.names) {
currentParameters.add(token.getText());
}
}
}
public static String squeezeStatement(String sql) {
TokenSource tokens = getLexer(sql, ImmutableSet.<String>of());
StringBuilder sb = new StringBuilder();
while (true) {
Token token = tokens.nextToken();
if (token.getType() == Token.EOF) {
break;
}
if (token.getType() == SqlBaseLexer.WS) {
sb.append(' ');
} else {
sb.append(token.getText());
}
}
return sb.toString().trim();
}
public void syntaxError(
Recognizer<?, ?> recognizer,
Object offendingSymbol,
int line,
int charPositionInLine,
String msg,
RecognitionException e) {
CommonTokenStream tokens = (CommonTokenStream) recognizer.getInputStream();
String input = tokens.getTokenSource().getInputStream().toString();
Token token = (Token) offendingSymbol;
String[] lines = StringUtils.splitPreserveAllTokens(input, '\n');
String errorLine = lines[line - 1];
String simpleMessage = "syntax error at or near \"" + token.getText() + "\"";
throw new LangParserException(token, line, charPositionInLine, simpleMessage, errorLine);
}
@Override
public void createPresentation(TextPresentation presentation, ITypedRegion region) {
// Use tokens provided by the lexer to highlight keywords, etc...
// Seems fast enough to skip Eclipse partitioning. Infact, the Eclipse
// partitioner seems to slow everything down...
TSLKGrammarLexer lexer = new TSLKGrammarLexer(new ANTLRInputStream(document.get()));
Token t = null;
while ((t = lexer.nextToken()).getType() != Token.EOF) {
if (t.getStartIndex() > region.getOffset() + region.getLength()) break;
int start = t.getStartIndex();
int end = t.getStopIndex();
RGB foreground = null;
RGB background = null;
int style = SWT.NORMAL;
switch (t.getType()) { // TODO: Make keywords customisable
case TSLKGrammarLexer.WHILE:
case TSLKGrammarLexer.FOR:
case TSLKGrammarLexer.FUNC:
case TSLKGrammarLexer.IF:
case TSLKGrammarLexer.THEN:
case TSLKGrammarLexer.DO:
case TSLKGrammarLexer.END:
foreground = ColorManager.KEYWORD;
style = SWT.BOLD;
break;
case TSLKGrammarLexer.STRING:
foreground = ColorManager.STRING;
break;
case TSLKGrammarLexer.SLCOMMENT:
foreground = ColorManager.SINGLE_LINE_COMMENT;
break;
case TSLKGrammarLexer.MLCOMMENT:
foreground = ColorManager.MULTI_LINE_COMMENT;
break;
default:
foreground = ColorManager.DEFAULT;
break;
}
presentation.addStyleRange(
new StyleRange(
start,
end - start + 1,
colorManager.getColor(foreground),
colorManager.getColor(background),
style));
}
}
private static int getArgumentCount(final List<Token> tokens) {
if (tokens.isEmpty()) {
return 0;
}
int count = 0;
for (Token token : tokens) {
if (token.getType() == AntlrLexer.ARGUMENT) {
++count;
} else {
break;
}
}
return count;
}
/**
* Gets whether or not {@code token} is the first non-whitespace symbol on a line.
*
* @param token The token to test.
* @return {@code true} if the only characters appearing before {@code token} on the same line are
* whitespace characters according to {@link Character#isWhitespace}.
*/
public static boolean elementStartsLine(Token token) {
String beginningOfLineText =
token
.getTokenSource()
.getInputStream()
.getText(
new Interval(
token.getStartIndex() - token.getCharPositionInLine(),
token.getStartIndex() - 1));
for (int i = 0; i < beginningOfLineText.length(); i++) {
if (!Character.isWhitespace(beginningOfLineText.charAt(i))) {
return false;
}
}
return true;
}
public static Interval getSourceInterval(@NonNull ParseTree context) {
Parameters.notNull("context", context);
if (context instanceof TerminalNode) {
TerminalNode terminalNode = (TerminalNode) context;
Token token = terminalNode.getSymbol();
return new Interval(token.getStartIndex(), token.getStopIndex());
} else if (context instanceof RuleNode) {
RuleNode ruleNode = (RuleNode) context;
RuleContext ruleContext = ruleNode.getRuleContext();
if (ruleContext instanceof ParserRuleContext) {
return getSourceInterval((ParserRuleContext) ruleContext);
} else {
Token startSymbol = getStartSymbol(context);
Token stopSymbol = getStopSymbol(context);
if (startSymbol == null || stopSymbol == null) {
return Interval.INVALID;
}
return new Interval(startSymbol.getStartIndex(), stopSymbol.getStopIndex());
}
} else {
return Interval.INVALID;
}
}
@Override
public int getLen(
RuleNode parent, Token token, List<Token> hiddenTokensToRight, TokenStream tokenStream) {
int len = 0;
for (org.antlr.v4.runtime.Token hiddenToken : hiddenTokensToRight) {
final String text = hiddenToken.getText();
if (TokenUtils.isWs(text)) {
len++;
} else {
if (hiddenToken.getCharPositionInLine() > token.getCharPositionInLine()) {
len++;
} else {
break;
}
}
}
return len;
}
@Override
public void exitInst(InstContext ctx) {
String s = getValue(ctx.getChild(0));
setValue(ctx, s);
int a = ctx.getStart().getTokenIndex();
List<Token> tt = tokens.getHiddenTokensToLeft(a, 1);
List<Token> ttnl = tokens.getHiddenTokensToLeft(a, 2);
String ttt = "";
if (ttnl != null && ttnl.size() > 1) ttt = "\n\n";
if (tt != null) {
for (Token t : tt) {
ttt = ttt + t.getText();
// System.out.println(t.getType() + " - " + t.getChannel());
}
// System.out.println(ttt);
}
setValue(ctx, ttt + getValue(ctx));
}
/**
* Turns a token stream into a string, makes for easy debugging of token errors.
*
* @param lexer
* @return
*/
public static String tokensToString(final Lexer lexer) {
final StringBuilder build = new StringBuilder();
Token t;
final String[] names = lexer.getTokenNames();
while ((t = lexer.nextToken()) != null) {
build.append("|");
build.append(t.getText());
build.append(" -> ");
if (t.getType() >= 0) {
build.append(names[t.getType()]);
} else if (t.getType() == Token.EOF) {
build.append("EOF");
break;
} else {
build.append("???");
}
}
return build.toString();
}
protected String getTokenDisplayString(Token token) {
String string;
if (token == null) {
string = "[no token]";
} else {
String text = token.getText();
if (text == null) {
if (token.getType() == Token.EOF) {
string = "end of text";
} else {
string = "[" + token.getType() + "]";
}
} else {
string = quote(text);
}
}
return string;
}
protected boolean tokenEndsAtEndOfLine(TokenSourceWithStateV4<TState> lexer, Token token) {
CharStream charStream = lexer.getInputStream();
if (charStream != null) {
int nextCharIndex = token.getStopIndex() + 1;
if (nextCharIndex >= charStream.size()) {
return true;
}
int c =
charStream
.getText(Interval.of(token.getStopIndex() + 1, token.getStopIndex() + 1))
.charAt(0);
return c == '\r' || c == '\n';
}
if (token.getStopIndex() + 1 >= snapshot.length()) {
return true;
}
char c = snapshot.subSequence(token.getStopIndex() + 1, token.getStopIndex() + 1).charAt(0);
return c == '\r' || c == '\n';
/*int line = snapshot.findLineNumber(token.getStopIndex() + 1);
int lineStart = snapshot.findLineFromOffset(line).getStart().getOffset();
int nextLineStart = snapshot.findLineFromOffset(line + 1).getStart().getOffset();
int lineEnd = nextLineStart - 1;
if (lineEnd > 0 && lineEnd > lineStart) {
char c = snapshot.charAt(lineEnd - 1);
if (c == '\r' || c == '\n') {
lineEnd--;
}
}
return lineEnd <= token.getStopIndex() + 1 && nextLineStart >= token.getStopIndex() + 1;*/
}
public static void displayErrorOrWarning(
File compilationUnit,
String errorOrWarning,
CommonTokenStream tokens,
Token offendingTokenStart,
Token offendingTokenEnd,
String msg) {
System.err.println(
errorOrWarning
+ compilationUnit.getAbsolutePath()
+ ", line "
+ offendingTokenStart.getLine()
+ ":"
+ offendingTokenStart.getCharPositionInLine()
+ " "
+ msg);
String input = tokens.getTokenSource().getInputStream().toString();
String[] lines = input.split("\n");
String errorLine = lines[offendingTokenStart.getLine() - 1];
System.err.println(errorLine);
for (int i = 0; i < offendingTokenStart.getCharPositionInLine(); i++)
if (errorLine.substring(i, i + 1).equals("\t")) System.err.print("\t");
else System.err.print(" ");
int start = offendingTokenStart.getStartIndex();
int stop = offendingTokenEnd.getStopIndex();
if (start >= 0 && stop >= 0) for (int i = start; i <= stop; i++) System.err.print("^");
System.err.println();
}
public StatementSplitter(String sql, Set<String> delimiters) {
TokenSource tokens = getLexer(sql, delimiters);
ImmutableList.Builder<Statement> list = ImmutableList.builder();
StringBuilder sb = new StringBuilder();
while (true) {
Token token = tokens.nextToken();
if (token.getType() == Token.EOF) {
break;
}
if (token.getType() == SqlBaseParser.DELIMITER) {
String statement = sb.toString().trim();
if (!statement.isEmpty()) {
list.add(new Statement(statement, token.getText()));
}
sb = new StringBuilder();
} else {
sb.append(token.getText());
}
}
this.completeStatements = list.build();
this.partialStatement = sb.toString().trim();
}
/**
* Logs parser errors in Checkstyle manner. Parser can generate error messages. There is special
* error that parser can generate. It is missed close HTML tag. This case is special because
* parser prints error like {@code "no viable alternative at input 'b \n *\n'"} and it is not
* clear that error is about missed close HTML tag. Other error messages are not special and
* logged simply as "Parse Error...".
*
* <p>{@inheritDoc}
*/
@Override
public void syntaxError(
Recognizer<?, ?> recognizer,
Object offendingSymbol,
int line,
int charPositionInLine,
String msg,
RecognitionException ex) {
final int lineNumber = offset + line;
final Token token = (Token) offendingSymbol;
if (MSG_JAVADOC_MISSED_HTML_CLOSE.equals(msg)) {
errorMessage =
new ParseErrorMessage(
lineNumber, MSG_JAVADOC_MISSED_HTML_CLOSE, charPositionInLine, token.getText());
throw new ParseCancellationException(msg);
} else if (MSG_JAVADOC_WRONG_SINGLETON_TAG.equals(msg)) {
errorMessage =
new ParseErrorMessage(
lineNumber, MSG_JAVADOC_WRONG_SINGLETON_TAG, charPositionInLine, token.getText());
throw new ParseCancellationException(msg);
} else {
final int ruleIndex = ex.getCtx().getRuleIndex();
final String ruleName = recognizer.getRuleNames()[ruleIndex];
final String upperCaseRuleName =
CaseFormat.UPPER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, ruleName);
errorMessage =
new ParseErrorMessage(
lineNumber,
MSG_JAVADOC_PARSE_RULE_ERROR,
charPositionInLine,
msg,
upperCaseRuleName);
}
}
public static Interval getSourceInterval(@NonNull ParserRuleContext context) {
Parameters.notNull("context", context);
int startIndex = context.start.getStartIndex();
Token stopSymbol = getStopSymbol(context);
if (stopSymbol == null) {
return new Interval(startIndex, startIndex - 1);
}
int stopIndex;
if (stopSymbol.getType() != Token.EOF) {
stopIndex = stopSymbol.getStopIndex();
} else {
TokenSource tokenSource = context.getStart().getTokenSource();
CharStream inputStream = tokenSource != null ? tokenSource.getInputStream() : null;
if (inputStream != null) {
stopIndex = inputStream.size() - 1;
} else {
stopIndex = context.start.getStartIndex() - 1;
}
}
stopIndex = Math.max(stopIndex, startIndex - 1);
return new Interval(startIndex, stopIndex);
}
@CheckForNull
public static TerminalNode findTerminalNode(@NonNull ParseTree node, Token symbol) {
if (symbol == null) {
return null;
}
if (node instanceof TerminalNode) {
TerminalNode terminalNode = (TerminalNode) node;
if (Utils.equals(terminalNode.getSymbol(), symbol)) {
return terminalNode;
}
return null;
}
for (int i = 0; i < node.getChildCount(); i++) {
ParseTree child = node.getChild(i);
TerminalNode stopNode = ParseTrees.getStopNode(child);
if (stopNode == null) {
continue;
}
Token stopSymbol = stopNode.getSymbol();
if (stopSymbol.getStopIndex() < symbol.getStartIndex()) {
continue;
}
TerminalNode startNode = ParseTrees.getStartNode(child);
assert startNode != null;
stopSymbol = startNode.getSymbol();
if (stopSymbol == null || stopSymbol.getStartIndex() > symbol.getStopIndex()) {
break;
}
if (stopSymbol.equals(symbol)) {
return startNode;
}
TerminalNode terminalNode = findTerminalNode(child, symbol);
if (terminalNode != null) {
return terminalNode;
}
}
return null;
}
private static boolean is(Token token, int type) {
return token != null && token.getType() == type;
}
public PSymbolBuilder qualifier(Token q) {
return qualifier(q != null ? q.getText() : null);
}