/** * Convert word like {@code *} or {@code ID} or {@code expr} to a path element. {@code anywhere} * is {@code true} if {@code //} precedes the word. */ protected XPathElement getXPathElement(Token wordToken, boolean anywhere) { if (wordToken.getType() == Token.EOF) { throw new IllegalArgumentException("Missing path element at end of path"); } String word = wordToken.getText(); int ttype = parser.getTokenType(word); int ruleIndex = parser.getRuleIndex(word); switch (wordToken.getType()) { case XPathLexer.WILDCARD: return anywhere ? new XPathWildcardAnywhereElement() : new XPathWildcardElement(); case XPathLexer.TOKEN_REF: case XPathLexer.STRING: if (ttype == Token.INVALID_TYPE) { throw new IllegalArgumentException( word + " at index " + wordToken.getStartIndex() + " isn't a valid token name"); } return anywhere ? new XPathTokenAnywhereElement(word, ttype) : new XPathTokenElement(word, ttype); default: if (ruleIndex == -1) { throw new IllegalArgumentException( word + " at index " + wordToken.getStartIndex() + " isn't a valid rule name"); } return anywhere ? new XPathRuleAnywhereElement(word, ruleIndex) : new XPathRuleElement(word, ruleIndex); } }
protected String getErrorMessage(RecognitionException re) { String message = ""; Parser recognizer = (Parser) re.getRecognizer(); TokenStream tokens = recognizer.getInputStream(); if (re instanceof NoViableAltException) { NoViableAltException e = (NoViableAltException) re; Token startToken = e.getStartToken(); String input = (startToken.getType() == Token.EOF) ? "end of text" : quote(tokens.getText(startToken, e.getOffendingToken())); message = "no viable date format found at " + input; } else if (re instanceof InputMismatchException) { InputMismatchException e = (InputMismatchException) re; message = "did not expect " + getTokenDisplayString(e.getOffendingToken()) + " while looking for " + e.getExpectedTokens().toString(recognizer.getTokenNames()); } else if (re instanceof FailedPredicateException) { FailedPredicateException e = (FailedPredicateException) re; String ruleName = recognizer.getRuleNames()[recognizer.getContext().getRuleIndex()]; message = "failed predicate " + ruleName + ": " + e.getMessage(); } return message; }
public static String squeezeStatement(String sql) { TokenSource tokens = getLexer(sql, ImmutableSet.<String>of()); StringBuilder sb = new StringBuilder(); while (true) { Token token = tokens.nextToken(); if (token.getType() == Token.EOF) { break; } if (token.getType() == SqlBaseLexer.WS) { sb.append(' '); } else { sb.append(token.getText()); } } return sb.toString().trim(); }
@Test public void testCompleteBufferAfterConsuming() throws Exception { LexerGrammar g = new LexerGrammar( "lexer grammar t;\n" + "ID : 'a'..'z'+;\n" + "INT : '0'..'9'+;\n" + "SEMI : ';';\n" + "ASSIGN : '=';\n" + "PLUS : '+';\n" + "MULT : '*';\n" + "WS : ' '+;\n"); // Tokens: 012345678901234567 // Input: x = 3 * 0 + 2 * 0; CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;"); LexerInterpreter lexEngine = g.createLexerInterpreter(input); TokenStream tokens = createTokenStream(lexEngine); Token t = tokens.LT(1); while (t.getType() != Token.EOF) { tokens.consume(); t = tokens.LT(1); } String result = tokens.getText(); String expecting = "x = 3 * 0 + 2 * 0;"; assertEquals(expecting, result); }
/** * Turns a token stream into a string, makes for easy debugging of token errors. * * @param lexer * @return */ public static String tokensToString(final Lexer lexer) { final StringBuilder build = new StringBuilder(); Token t; final String[] names = lexer.getTokenNames(); while ((t = lexer.nextToken()) != null) { build.append("|"); build.append(t.getText()); build.append(" -> "); if (t.getType() >= 0) { build.append(names[t.getType()]); } else if (t.getType() == Token.EOF) { build.append("EOF"); break; } else { build.append("???"); } } return build.toString(); }
protected String getTokenDisplayString(Token token) { String string; if (token == null) { string = "[no token]"; } else { String text = token.getText(); if (text == null) { if (token.getType() == Token.EOF) { string = "end of text"; } else { string = "[" + token.getType() + "]"; } } else { string = quote(text); } } return string; }
public static boolean isEmptyStatement(String sql) { TokenSource tokens = getLexer(sql, ImmutableSet.<String>of()); while (true) { Token token = tokens.nextToken(); if (token.getType() == Token.EOF) { return true; } if (token.getChannel() != Token.HIDDEN_CHANNEL) { return false; } } }
public StatementSplitter(String sql, Set<String> delimiters) { TokenSource tokens = getLexer(sql, delimiters); ImmutableList.Builder<Statement> list = ImmutableList.builder(); StringBuilder sb = new StringBuilder(); while (true) { Token token = tokens.nextToken(); if (token.getType() == Token.EOF) { break; } if (token.getType() == SqlBaseParser.DELIMITER) { String statement = sb.toString().trim(); if (!statement.isEmpty()) { list.add(new Statement(statement, token.getText())); } sb = new StringBuilder(); } else { sb.append(token.getText()); } } this.completeStatements = list.build(); this.partialStatement = sb.toString().trim(); }
@Override protected boolean sync(int i) { if (!super.sync(i)) { return false; } Token t = get(i); if (hide.contains(t.getType())) { ((WritableToken) t).setChannel(Token.HIDDEN_CHANNEL); } return true; }
@Override public void createPresentation(TextPresentation presentation, ITypedRegion region) { // Use tokens provided by the lexer to highlight keywords, etc... // Seems fast enough to skip Eclipse partitioning. Infact, the Eclipse // partitioner seems to slow everything down... TSLKGrammarLexer lexer = new TSLKGrammarLexer(new ANTLRInputStream(document.get())); Token t = null; while ((t = lexer.nextToken()).getType() != Token.EOF) { if (t.getStartIndex() > region.getOffset() + region.getLength()) break; int start = t.getStartIndex(); int end = t.getStopIndex(); RGB foreground = null; RGB background = null; int style = SWT.NORMAL; switch (t.getType()) { // TODO: Make keywords customisable case TSLKGrammarLexer.WHILE: case TSLKGrammarLexer.FOR: case TSLKGrammarLexer.FUNC: case TSLKGrammarLexer.IF: case TSLKGrammarLexer.THEN: case TSLKGrammarLexer.DO: case TSLKGrammarLexer.END: foreground = ColorManager.KEYWORD; style = SWT.BOLD; break; case TSLKGrammarLexer.STRING: foreground = ColorManager.STRING; break; case TSLKGrammarLexer.SLCOMMENT: foreground = ColorManager.SINGLE_LINE_COMMENT; break; case TSLKGrammarLexer.MLCOMMENT: foreground = ColorManager.MULTI_LINE_COMMENT; break; default: foreground = ColorManager.DEFAULT; break; } presentation.addStyleRange( new StyleRange( start, end - start + 1, colorManager.getColor(foreground), colorManager.getColor(background), style)); } }
private static int getArgumentCount(final List<Token> tokens) { if (tokens.isEmpty()) { return 0; } int count = 0; for (Token token : tokens) { if (token.getType() == AntlrLexer.ARGUMENT) { ++count; } else { break; } } return count; }
public static Interval getSourceInterval(@NonNull ParserRuleContext context) { Parameters.notNull("context", context); int startIndex = context.start.getStartIndex(); Token stopSymbol = getStopSymbol(context); if (stopSymbol == null) { return new Interval(startIndex, startIndex - 1); } int stopIndex; if (stopSymbol.getType() != Token.EOF) { stopIndex = stopSymbol.getStopIndex(); } else { TokenSource tokenSource = context.getStart().getTokenSource(); CharStream inputStream = tokenSource != null ? tokenSource.getInputStream() : null; if (inputStream != null) { stopIndex = inputStream.size() - 1; } else { stopIndex = context.start.getStartIndex() - 1; } } stopIndex = Math.max(stopIndex, startIndex - 1); return new Interval(startIndex, stopIndex); }
private static boolean is(Token token, int type) { return token != null && token.getType() == type; }
public List<TaggedPositionRegion<TokenTag<Token>>> getHighlights(int startOffset, int endOffset) { List<TaggedPositionRegion<TokenTag<Token>>> tags = new ArrayList<>(); boolean updateOffsets = true; if (endOffset == Integer.MAX_VALUE) { endOffset = snapshot.length(); } OffsetRegion span = OffsetRegion.fromBounds(startOffset, endOffset); if (failedTimeout) { return tags; } boolean spanExtended = false; int extendMultiLineSpanToLine = 0; OffsetRegion extendedSpan = span; synchronized (lock) { OffsetRegion requestedSpan = span; ParseRequest<TState> request = adjustParseSpan(span); TState startState = request.getState(); span = request.getRegion(); CharStream input; try { input = createInputStream(span); } catch (BadLocationException ex) { LOGGER.log(Level.WARNING, ex.getMessage(), ex); return tags; } TokenSourceWithStateV4<TState> lexer = createLexer(input, startState); lexer.setTokenFactory(new DocumentSnapshotTokenFactory(getEffectiveTokenSource(lexer))); Token previousToken = null; boolean previousTokenEndsLine = false; /* this is held outside the loop because only tokens which end at the end of a line * impact its value. */ boolean lineStateChanged = false; while (true) { // TODO: perform this under a read lock Token token = lexer.nextToken(); // The latter is true for EOF token with span.getEnd() at the end of the document boolean inBounds = token.getStartIndex() < span.getEnd() || token.getStopIndex() < span.getEnd(); if (updateOffsets) { int startLineCurrent; if (token.getType() == Token.EOF) startLineCurrent = snapshot.getLineCount(); else startLineCurrent = snapshot.findLineNumber(token.getStartIndex()); // endLinePrevious is the line number the previous token ended on int endLinePrevious; if (previousToken != null) endLinePrevious = snapshot.findLineNumber(previousToken.getStopIndex() + 1); else endLinePrevious = snapshot.findLineNumber(span.getStart()) - 1; if (startLineCurrent > endLinePrevious + 1 || (startLineCurrent == endLinePrevious + 1 && !previousTokenEndsLine)) { int firstMultilineLine = endLinePrevious; if (previousToken == null || previousTokenEndsLine) firstMultilineLine++; for (int i = firstMultilineLine; i < startLineCurrent; i++) { if (!lineStates.get(i).getIsMultiLineToken() || lineStateChanged) extendMultiLineSpanToLine = i + 1; if (inBounds) setLineState(i, lineStates.get(i).createMultiLineState()); } } } if (token.getType() == Token.EOF) break; if (updateOffsets && isMultiLineToken(lexer, token)) { int startLine = snapshot.findLineNumber(token.getStartIndex()); int stopLine = snapshot.findLineNumber(token.getStopIndex() + 1); for (int i = startLine; i < stopLine; i++) { if (!lineStates.get(i).getIsMultiLineToken()) extendMultiLineSpanToLine = i + 1; if (inBounds) setLineState(i, lineStates.get(i).createMultiLineState()); } } boolean tokenEndsLine = tokenEndsAtEndOfLine(lexer, token); if (updateOffsets && tokenEndsLine) { TState stateAtEndOfLine = lexer.getCurrentState(); int line = snapshot.findLineNumber(token.getStopIndex() + 1); lineStateChanged = lineStates.get(line).getIsMultiLineToken() || !lineStates.get(line).equals(stateAtEndOfLine); // even if the state didn't change, we call SetLineState to make sure the // _first/_lastChangedLine values get updated. // have to check bounds for this one or the editor might not get an update (if the token // ends a line) if (updateOffsets && inBounds) setLineState(line, stateAtEndOfLine); if (lineStateChanged) { if (line < snapshot.getLineCount() - 1) { /* update the span's end position or the line state change won't be reflected * in the editor */ int endPosition = line < snapshot.getLineCount() - 2 ? snapshot.findLineFromLineNumber(line + 2).getStart().getOffset() : snapshot.length(); if (endPosition > extendedSpan.getEnd()) { spanExtended = true; extendedSpan = OffsetRegion.fromBounds(extendedSpan.getStart(), endPosition); } } } } if (token.getStartIndex() >= span.getEnd()) { break; } previousToken = token; previousTokenEndsLine = tokenEndsLine; if (token.getStopIndex() < requestedSpan.getStart()) { continue; } Collection<TaggedPositionRegion<TokenTag<Token>>> tokenClassificationSpans = getTagsForToken(token); if (tokenClassificationSpans != null) { tags.addAll(tokenClassificationSpans); } if (!inBounds) { break; } } } if (updateOffsets && extendMultiLineSpanToLine > 0) { int endPosition = extendMultiLineSpanToLine < snapshot.getLineCount() - 1 ? snapshot .findLineFromLineNumber(extendMultiLineSpanToLine + 1) .getStart() .getOffset() : snapshot.length(); if (endPosition > extendedSpan.getEnd()) { spanExtended = true; extendedSpan = OffsetRegion.fromBounds(extendedSpan.getStart(), endPosition); } } if (updateOffsets && spanExtended) { /* Subtract 1 from each of these because the spans include the line break on their last * line, forcing it to appear as the first position on the following line. */ assert extendedSpan.getEnd() > span.getEnd(); int firstLine = snapshot.findLineNumber(span.getEnd()); int lastLine = snapshot.findLineNumber(extendedSpan.getEnd()) - 1; // when considering the last line of a document, span and extendedSpan may end on the same // line forceRehighlightLines(firstLine, Math.max(firstLine, lastLine)); } return tags; }
public XPathElement[] split(String path) { ANTLRInputStream in; try { in = new ANTLRInputStream(new StringReader(path)); } catch (IOException ioe) { throw new IllegalArgumentException("Could not read path: " + path, ioe); } XPathLexer lexer = new XPathLexer(in) { @Override public void recover(LexerNoViableAltException e) { throw e; } }; lexer.removeErrorListeners(); lexer.addErrorListener(new XPathLexerErrorListener()); CommonTokenStream tokenStream = new CommonTokenStream(lexer); try { tokenStream.fill(); } catch (LexerNoViableAltException e) { int pos = lexer.getCharPositionInLine(); String msg = "Invalid tokens or characters at index " + pos + " in path '" + path + "'"; throw new IllegalArgumentException(msg, e); } List<Token> tokens = tokenStream.getTokens(); // System.out.println("path="+path+"=>"+tokens); List<XPathElement> elements = new ArrayList<XPathElement>(); int n = tokens.size(); int i = 0; loop: while (i < n) { Token el = tokens.get(i); Token next = null; switch (el.getType()) { case XPathLexer.ROOT: case XPathLexer.ANYWHERE: boolean anywhere = el.getType() == XPathLexer.ANYWHERE; i++; next = tokens.get(i); boolean invert = next.getType() == XPathLexer.BANG; if (invert) { i++; next = tokens.get(i); } XPathElement pathElement = getXPathElement(next, anywhere); pathElement.invert = invert; elements.add(pathElement); i++; break; case XPathLexer.TOKEN_REF: case XPathLexer.RULE_REF: case XPathLexer.WILDCARD: elements.add(getXPathElement(el, false)); i++; break; case Token.EOF: break loop; default: throw new IllegalArgumentException("Unknowth path element " + el); } } return elements.toArray(new XPathElement[0]); }