/**
* Convert word like {@code *} or {@code ID} or {@code expr} to a path element. {@code anywhere}
* is {@code true} if {@code //} precedes the word.
*/
protected XPathElement getXPathElement(Token wordToken, boolean anywhere) {
if (wordToken.getType() == Token.EOF) {
throw new IllegalArgumentException("Missing path element at end of path");
}
String word = wordToken.getText();
int ttype = parser.getTokenType(word);
int ruleIndex = parser.getRuleIndex(word);
switch (wordToken.getType()) {
case XPathLexer.WILDCARD:
return anywhere ? new XPathWildcardAnywhereElement() : new XPathWildcardElement();
case XPathLexer.TOKEN_REF:
case XPathLexer.STRING:
if (ttype == Token.INVALID_TYPE) {
throw new IllegalArgumentException(
word + " at index " + wordToken.getStartIndex() + " isn't a valid token name");
}
return anywhere
? new XPathTokenAnywhereElement(word, ttype)
: new XPathTokenElement(word, ttype);
default:
if (ruleIndex == -1) {
throw new IllegalArgumentException(
word + " at index " + wordToken.getStartIndex() + " isn't a valid rule name");
}
return anywhere
? new XPathRuleAnywhereElement(word, ruleIndex)
: new XPathRuleElement(word, ruleIndex);
}
}
protected String getErrorMessage(RecognitionException re) {
String message = "";
Parser recognizer = (Parser) re.getRecognizer();
TokenStream tokens = recognizer.getInputStream();
if (re instanceof NoViableAltException) {
NoViableAltException e = (NoViableAltException) re;
Token startToken = e.getStartToken();
String input =
(startToken.getType() == Token.EOF)
? "end of text"
: quote(tokens.getText(startToken, e.getOffendingToken()));
message = "no viable date format found at " + input;
} else if (re instanceof InputMismatchException) {
InputMismatchException e = (InputMismatchException) re;
message =
"did not expect "
+ getTokenDisplayString(e.getOffendingToken())
+ " while looking for "
+ e.getExpectedTokens().toString(recognizer.getTokenNames());
} else if (re instanceof FailedPredicateException) {
FailedPredicateException e = (FailedPredicateException) re;
String ruleName = recognizer.getRuleNames()[recognizer.getContext().getRuleIndex()];
message = "failed predicate " + ruleName + ": " + e.getMessage();
}
return message;
}
public static String squeezeStatement(String sql) {
TokenSource tokens = getLexer(sql, ImmutableSet.<String>of());
StringBuilder sb = new StringBuilder();
while (true) {
Token token = tokens.nextToken();
if (token.getType() == Token.EOF) {
break;
}
if (token.getType() == SqlBaseLexer.WS) {
sb.append(' ');
} else {
sb.append(token.getText());
}
}
return sb.toString().trim();
}
@Test
public void testCompleteBufferAfterConsuming() throws Exception {
LexerGrammar g =
new LexerGrammar(
"lexer grammar t;\n"
+ "ID : 'a'..'z'+;\n"
+ "INT : '0'..'9'+;\n"
+ "SEMI : ';';\n"
+ "ASSIGN : '=';\n"
+ "PLUS : '+';\n"
+ "MULT : '*';\n"
+ "WS : ' '+;\n");
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
LexerInterpreter lexEngine = g.createLexerInterpreter(input);
TokenStream tokens = createTokenStream(lexEngine);
Token t = tokens.LT(1);
while (t.getType() != Token.EOF) {
tokens.consume();
t = tokens.LT(1);
}
String result = tokens.getText();
String expecting = "x = 3 * 0 + 2 * 0;";
assertEquals(expecting, result);
}
/**
* Turns a token stream into a string, makes for easy debugging of token errors.
*
* @param lexer
* @return
*/
public static String tokensToString(final Lexer lexer) {
final StringBuilder build = new StringBuilder();
Token t;
final String[] names = lexer.getTokenNames();
while ((t = lexer.nextToken()) != null) {
build.append("|");
build.append(t.getText());
build.append(" -> ");
if (t.getType() >= 0) {
build.append(names[t.getType()]);
} else if (t.getType() == Token.EOF) {
build.append("EOF");
break;
} else {
build.append("???");
}
}
return build.toString();
}
protected String getTokenDisplayString(Token token) {
String string;
if (token == null) {
string = "[no token]";
} else {
String text = token.getText();
if (text == null) {
if (token.getType() == Token.EOF) {
string = "end of text";
} else {
string = "[" + token.getType() + "]";
}
} else {
string = quote(text);
}
}
return string;
}
public static boolean isEmptyStatement(String sql) {
TokenSource tokens = getLexer(sql, ImmutableSet.<String>of());
while (true) {
Token token = tokens.nextToken();
if (token.getType() == Token.EOF) {
return true;
}
if (token.getChannel() != Token.HIDDEN_CHANNEL) {
return false;
}
}
}
public StatementSplitter(String sql, Set<String> delimiters) {
TokenSource tokens = getLexer(sql, delimiters);
ImmutableList.Builder<Statement> list = ImmutableList.builder();
StringBuilder sb = new StringBuilder();
while (true) {
Token token = tokens.nextToken();
if (token.getType() == Token.EOF) {
break;
}
if (token.getType() == SqlBaseParser.DELIMITER) {
String statement = sb.toString().trim();
if (!statement.isEmpty()) {
list.add(new Statement(statement, token.getText()));
}
sb = new StringBuilder();
} else {
sb.append(token.getText());
}
}
this.completeStatements = list.build();
this.partialStatement = sb.toString().trim();
}
@Override
protected boolean sync(int i) {
if (!super.sync(i)) {
return false;
}
Token t = get(i);
if (hide.contains(t.getType())) {
((WritableToken) t).setChannel(Token.HIDDEN_CHANNEL);
}
return true;
}
@Override
public void createPresentation(TextPresentation presentation, ITypedRegion region) {
// Use tokens provided by the lexer to highlight keywords, etc...
// Seems fast enough to skip Eclipse partitioning. Infact, the Eclipse
// partitioner seems to slow everything down...
TSLKGrammarLexer lexer = new TSLKGrammarLexer(new ANTLRInputStream(document.get()));
Token t = null;
while ((t = lexer.nextToken()).getType() != Token.EOF) {
if (t.getStartIndex() > region.getOffset() + region.getLength()) break;
int start = t.getStartIndex();
int end = t.getStopIndex();
RGB foreground = null;
RGB background = null;
int style = SWT.NORMAL;
switch (t.getType()) { // TODO: Make keywords customisable
case TSLKGrammarLexer.WHILE:
case TSLKGrammarLexer.FOR:
case TSLKGrammarLexer.FUNC:
case TSLKGrammarLexer.IF:
case TSLKGrammarLexer.THEN:
case TSLKGrammarLexer.DO:
case TSLKGrammarLexer.END:
foreground = ColorManager.KEYWORD;
style = SWT.BOLD;
break;
case TSLKGrammarLexer.STRING:
foreground = ColorManager.STRING;
break;
case TSLKGrammarLexer.SLCOMMENT:
foreground = ColorManager.SINGLE_LINE_COMMENT;
break;
case TSLKGrammarLexer.MLCOMMENT:
foreground = ColorManager.MULTI_LINE_COMMENT;
break;
default:
foreground = ColorManager.DEFAULT;
break;
}
presentation.addStyleRange(
new StyleRange(
start,
end - start + 1,
colorManager.getColor(foreground),
colorManager.getColor(background),
style));
}
}
private static int getArgumentCount(final List<Token> tokens) {
if (tokens.isEmpty()) {
return 0;
}
int count = 0;
for (Token token : tokens) {
if (token.getType() == AntlrLexer.ARGUMENT) {
++count;
} else {
break;
}
}
return count;
}
public static Interval getSourceInterval(@NonNull ParserRuleContext context) {
Parameters.notNull("context", context);
int startIndex = context.start.getStartIndex();
Token stopSymbol = getStopSymbol(context);
if (stopSymbol == null) {
return new Interval(startIndex, startIndex - 1);
}
int stopIndex;
if (stopSymbol.getType() != Token.EOF) {
stopIndex = stopSymbol.getStopIndex();
} else {
TokenSource tokenSource = context.getStart().getTokenSource();
CharStream inputStream = tokenSource != null ? tokenSource.getInputStream() : null;
if (inputStream != null) {
stopIndex = inputStream.size() - 1;
} else {
stopIndex = context.start.getStartIndex() - 1;
}
}
stopIndex = Math.max(stopIndex, startIndex - 1);
return new Interval(startIndex, stopIndex);
}
private static boolean is(Token token, int type) {
return token != null && token.getType() == type;
}
public List<TaggedPositionRegion<TokenTag<Token>>> getHighlights(int startOffset, int endOffset) {
List<TaggedPositionRegion<TokenTag<Token>>> tags = new ArrayList<>();
boolean updateOffsets = true;
if (endOffset == Integer.MAX_VALUE) {
endOffset = snapshot.length();
}
OffsetRegion span = OffsetRegion.fromBounds(startOffset, endOffset);
if (failedTimeout) {
return tags;
}
boolean spanExtended = false;
int extendMultiLineSpanToLine = 0;
OffsetRegion extendedSpan = span;
synchronized (lock) {
OffsetRegion requestedSpan = span;
ParseRequest<TState> request = adjustParseSpan(span);
TState startState = request.getState();
span = request.getRegion();
CharStream input;
try {
input = createInputStream(span);
} catch (BadLocationException ex) {
LOGGER.log(Level.WARNING, ex.getMessage(), ex);
return tags;
}
TokenSourceWithStateV4<TState> lexer = createLexer(input, startState);
lexer.setTokenFactory(new DocumentSnapshotTokenFactory(getEffectiveTokenSource(lexer)));
Token previousToken = null;
boolean previousTokenEndsLine = false;
/* this is held outside the loop because only tokens which end at the end of a line
* impact its value.
*/
boolean lineStateChanged = false;
while (true) {
// TODO: perform this under a read lock
Token token = lexer.nextToken();
// The latter is true for EOF token with span.getEnd() at the end of the document
boolean inBounds =
token.getStartIndex() < span.getEnd() || token.getStopIndex() < span.getEnd();
if (updateOffsets) {
int startLineCurrent;
if (token.getType() == Token.EOF) startLineCurrent = snapshot.getLineCount();
else startLineCurrent = snapshot.findLineNumber(token.getStartIndex());
// endLinePrevious is the line number the previous token ended on
int endLinePrevious;
if (previousToken != null)
endLinePrevious = snapshot.findLineNumber(previousToken.getStopIndex() + 1);
else endLinePrevious = snapshot.findLineNumber(span.getStart()) - 1;
if (startLineCurrent > endLinePrevious + 1
|| (startLineCurrent == endLinePrevious + 1 && !previousTokenEndsLine)) {
int firstMultilineLine = endLinePrevious;
if (previousToken == null || previousTokenEndsLine) firstMultilineLine++;
for (int i = firstMultilineLine; i < startLineCurrent; i++) {
if (!lineStates.get(i).getIsMultiLineToken() || lineStateChanged)
extendMultiLineSpanToLine = i + 1;
if (inBounds) setLineState(i, lineStates.get(i).createMultiLineState());
}
}
}
if (token.getType() == Token.EOF) break;
if (updateOffsets && isMultiLineToken(lexer, token)) {
int startLine = snapshot.findLineNumber(token.getStartIndex());
int stopLine = snapshot.findLineNumber(token.getStopIndex() + 1);
for (int i = startLine; i < stopLine; i++) {
if (!lineStates.get(i).getIsMultiLineToken()) extendMultiLineSpanToLine = i + 1;
if (inBounds) setLineState(i, lineStates.get(i).createMultiLineState());
}
}
boolean tokenEndsLine = tokenEndsAtEndOfLine(lexer, token);
if (updateOffsets && tokenEndsLine) {
TState stateAtEndOfLine = lexer.getCurrentState();
int line = snapshot.findLineNumber(token.getStopIndex() + 1);
lineStateChanged =
lineStates.get(line).getIsMultiLineToken()
|| !lineStates.get(line).equals(stateAtEndOfLine);
// even if the state didn't change, we call SetLineState to make sure the
// _first/_lastChangedLine values get updated.
// have to check bounds for this one or the editor might not get an update (if the token
// ends a line)
if (updateOffsets && inBounds) setLineState(line, stateAtEndOfLine);
if (lineStateChanged) {
if (line < snapshot.getLineCount() - 1) {
/* update the span's end position or the line state change won't be reflected
* in the editor
*/
int endPosition =
line < snapshot.getLineCount() - 2
? snapshot.findLineFromLineNumber(line + 2).getStart().getOffset()
: snapshot.length();
if (endPosition > extendedSpan.getEnd()) {
spanExtended = true;
extendedSpan = OffsetRegion.fromBounds(extendedSpan.getStart(), endPosition);
}
}
}
}
if (token.getStartIndex() >= span.getEnd()) {
break;
}
previousToken = token;
previousTokenEndsLine = tokenEndsLine;
if (token.getStopIndex() < requestedSpan.getStart()) {
continue;
}
Collection<TaggedPositionRegion<TokenTag<Token>>> tokenClassificationSpans =
getTagsForToken(token);
if (tokenClassificationSpans != null) {
tags.addAll(tokenClassificationSpans);
}
if (!inBounds) {
break;
}
}
}
if (updateOffsets && extendMultiLineSpanToLine > 0) {
int endPosition =
extendMultiLineSpanToLine < snapshot.getLineCount() - 1
? snapshot
.findLineFromLineNumber(extendMultiLineSpanToLine + 1)
.getStart()
.getOffset()
: snapshot.length();
if (endPosition > extendedSpan.getEnd()) {
spanExtended = true;
extendedSpan = OffsetRegion.fromBounds(extendedSpan.getStart(), endPosition);
}
}
if (updateOffsets && spanExtended) {
/* Subtract 1 from each of these because the spans include the line break on their last
* line, forcing it to appear as the first position on the following line.
*/
assert extendedSpan.getEnd() > span.getEnd();
int firstLine = snapshot.findLineNumber(span.getEnd());
int lastLine = snapshot.findLineNumber(extendedSpan.getEnd()) - 1;
// when considering the last line of a document, span and extendedSpan may end on the same
// line
forceRehighlightLines(firstLine, Math.max(firstLine, lastLine));
}
return tags;
}
public XPathElement[] split(String path) {
ANTLRInputStream in;
try {
in = new ANTLRInputStream(new StringReader(path));
} catch (IOException ioe) {
throw new IllegalArgumentException("Could not read path: " + path, ioe);
}
XPathLexer lexer =
new XPathLexer(in) {
@Override
public void recover(LexerNoViableAltException e) {
throw e;
}
};
lexer.removeErrorListeners();
lexer.addErrorListener(new XPathLexerErrorListener());
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
try {
tokenStream.fill();
} catch (LexerNoViableAltException e) {
int pos = lexer.getCharPositionInLine();
String msg = "Invalid tokens or characters at index " + pos + " in path '" + path + "'";
throw new IllegalArgumentException(msg, e);
}
List<Token> tokens = tokenStream.getTokens();
// System.out.println("path="+path+"=>"+tokens);
List<XPathElement> elements = new ArrayList<XPathElement>();
int n = tokens.size();
int i = 0;
loop:
while (i < n) {
Token el = tokens.get(i);
Token next = null;
switch (el.getType()) {
case XPathLexer.ROOT:
case XPathLexer.ANYWHERE:
boolean anywhere = el.getType() == XPathLexer.ANYWHERE;
i++;
next = tokens.get(i);
boolean invert = next.getType() == XPathLexer.BANG;
if (invert) {
i++;
next = tokens.get(i);
}
XPathElement pathElement = getXPathElement(next, anywhere);
pathElement.invert = invert;
elements.add(pathElement);
i++;
break;
case XPathLexer.TOKEN_REF:
case XPathLexer.RULE_REF:
case XPathLexer.WILDCARD:
elements.add(getXPathElement(el, false));
i++;
break;
case Token.EOF:
break loop;
default:
throw new IllegalArgumentException("Unknowth path element " + el);
}
}
return elements.toArray(new XPathElement[0]);
}