public String filterMethod(String compilationUnit, String className, String methodNameAndParams) {

    final String methodName = extractMethodName(methodNameAndParams);
    final String[] params = extractParams(methodNameAndParams);

    ANTLRInputStream in = new ANTLRInputStream(compilationUnit);
    TokenStream tokenStream = new CommonTokenStream(new Java8Lexer(in));
    Java8Parser parser = new Java8Parser(tokenStream);

    MethodFilterListener filterListener = new MethodFilterListener(className, methodName, params);

    parser.addParseListener(filterListener);
    parser.setBuildParseTree(true);
    parser.compilationUnit();

    if (filterListener.getInterval() == null) {
      throw new IllegalArgumentException("Method '" + methodName + "' not found!");
    }

    final Token startToken = tokenStream.get(filterListener.getInterval().a);
    final Token endToken = tokenStream.get(filterListener.getInterval().b);

    final int startLine = startToken.getLine();
    final int endLine = endToken.getLine();

    return getLines(compilationUnit, startLine, endLine);
  }
  @Test
  public void testCompleteBufferAfterConsuming() throws Exception {
    LexerGrammar g =
        new LexerGrammar(
            "lexer grammar t;\n"
                + "ID : 'a'..'z'+;\n"
                + "INT : '0'..'9'+;\n"
                + "SEMI : ';';\n"
                + "ASSIGN : '=';\n"
                + "PLUS : '+';\n"
                + "MULT : '*';\n"
                + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 3 * 0 + 2 * 0;
    CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TokenStream tokens = createTokenStream(lexEngine);

    Token t = tokens.LT(1);
    while (t.getType() != Token.EOF) {
      tokens.consume();
      t = tokens.LT(1);
    }

    String result = tokens.getText();
    String expecting = "x = 3 * 0 + 2 * 0;";
    assertEquals(expecting, result);
  }
  protected String getErrorMessage(RecognitionException re) {
    String message = "";

    Parser recognizer = (Parser) re.getRecognizer();
    TokenStream tokens = recognizer.getInputStream();

    if (re instanceof NoViableAltException) {
      NoViableAltException e = (NoViableAltException) re;
      Token startToken = e.getStartToken();
      String input =
          (startToken.getType() == Token.EOF)
              ? "end of text"
              : quote(tokens.getText(startToken, e.getOffendingToken()));

      message = "no viable date format found at " + input;
    } else if (re instanceof InputMismatchException) {
      InputMismatchException e = (InputMismatchException) re;
      message =
          "did not expect "
              + getTokenDisplayString(e.getOffendingToken())
              + " while looking for "
              + e.getExpectedTokens().toString(recognizer.getTokenNames());
    } else if (re instanceof FailedPredicateException) {
      FailedPredicateException e = (FailedPredicateException) re;
      String ruleName = recognizer.getRuleNames()[recognizer.getContext().getRuleIndex()];

      message = "failed predicate " + ruleName + ": " + e.getMessage();
    }

    return message;
  }
  @Test
  public void testLookback() throws Exception {
    LexerGrammar g =
        new LexerGrammar(
            "lexer grammar t;\n"
                + "ID : 'a'..'z'+;\n"
                + "INT : '0'..'9'+;\n"
                + "SEMI : ';';\n"
                + "ASSIGN : '=';\n"
                + "PLUS : '+';\n"
                + "MULT : '*';\n"
                + "WS : ' '+;\n");
    // Tokens: 012345678901234567
    // Input:  x = 3 * 0 + 2 * 0;
    CharStream input = new ANTLRInputStream("x = 3 * 0 + 2 * 0;");
    LexerInterpreter lexEngine = g.createLexerInterpreter(input);
    TokenStream tokens = createTokenStream(lexEngine);

    tokens.consume(); // get x into buffer
    Token t = tokens.LT(-1);
    assertEquals("x", t.getText());

    tokens.consume();
    tokens.consume(); // consume '='
    t = tokens.LT(-3);
    assertEquals("x", t.getText());
    t = tokens.LT(-2);
    assertEquals(" ", t.getText());
    t = tokens.LT(-1);
    assertEquals("=", t.getText());
  }
Ejemplo n.º 5
0
 /**
  * Convert word like {@code *} or {@code ID} or {@code expr} to a path element. {@code anywhere}
  * is {@code true} if {@code //} precedes the word.
  */
 protected XPathElement getXPathElement(Token wordToken, boolean anywhere) {
   if (wordToken.getType() == Token.EOF) {
     throw new IllegalArgumentException("Missing path element at end of path");
   }
   String word = wordToken.getText();
   int ttype = parser.getTokenType(word);
   int ruleIndex = parser.getRuleIndex(word);
   switch (wordToken.getType()) {
     case XPathLexer.WILDCARD:
       return anywhere ? new XPathWildcardAnywhereElement() : new XPathWildcardElement();
     case XPathLexer.TOKEN_REF:
     case XPathLexer.STRING:
       if (ttype == Token.INVALID_TYPE) {
         throw new IllegalArgumentException(
             word + " at index " + wordToken.getStartIndex() + " isn't a valid token name");
       }
       return anywhere
           ? new XPathTokenAnywhereElement(word, ttype)
           : new XPathTokenElement(word, ttype);
     default:
       if (ruleIndex == -1) {
         throw new IllegalArgumentException(
             word + " at index " + wordToken.getStartIndex() + " isn't a valid rule name");
       }
       return anywhere
           ? new XPathRuleAnywhereElement(word, ruleIndex)
           : new XPathRuleElement(word, ruleIndex);
   }
 }
  @Override
  public void addToHighlighting(int annexOffset, Token token, String id) {
    int offset = ((CommonToken) token).getStartIndex();
    int length = token.getText().length();
    int column = token.getCharPositionInLine();

    _elementToHighlight.add(new AadlBaLocationReference(annexOffset, offset, length, column, id));
  }
  protected boolean isMultiLineToken(TokenSourceWithStateV4<TState> lexer, Token token) {
    /*if (lexer != null && lexer.getLine() > token.getLine()) {
        return true;
    }*/

    int startLine = snapshot.findLineNumber(token.getStartIndex());
    int stopLine = snapshot.findLineNumber(token.getStopIndex() + 1);
    return startLine != stopLine;
  }
Ejemplo n.º 8
0
 public static boolean isEmptyStatement(String sql) {
   TokenSource tokens = getLexer(sql, ImmutableSet.<String>of());
   while (true) {
     Token token = tokens.nextToken();
     if (token.getType() == Token.EOF) {
       return true;
     }
     if (token.getChannel() != Token.HIDDEN_CHANNEL) {
       return false;
     }
   }
 }
Ejemplo n.º 9
0
    @Override
    protected boolean sync(int i) {
      if (!super.sync(i)) {
        return false;
      }

      Token t = get(i);
      if (hide.contains(t.getType())) {
        ((WritableToken) t).setChannel(Token.HIDDEN_CHANNEL);
      }

      return true;
    }
  protected Collection<TaggedPositionRegion<TokenTag<Token>>> getTagsForToken(Token token) {
    TokenTag<Token> tag = highlightToken(token);
    if (tag != null) {
      return Collections.<TaggedPositionRegion<TokenTag<Token>>>singleton(
          new BaseTaggedPositionRegion<>(
              new SnapshotPositionRegion(
                  snapshot,
                  OffsetRegion.fromBounds(token.getStartIndex(), token.getStopIndex() + 1)),
              tag));
    }

    return Collections.emptyList();
  }
Ejemplo n.º 11
0
 private void addElement(Token token, SyntaxElementType type) {
   syntaxElements.add(
       SyntaxElement.create(
           token.getText(),
           token.getStartIndex(),
           token.getStopIndex(),
           token.getTokenIndex(),
           type));
   if (debug) {
     SyntaxElement e = syntaxElements.get(syntaxElements.size() - 1);
     System.out.println(String.format("%d-%d %s %s", e.from, e.to, e.type.name(), e.value));
   }
 }
    @Override
    @RuleDependency(
        recognizer = TemplateParser.class,
        rule = TemplateParser.RULE_anonymousTemplateParameters,
        version = 0,
        dependents = Dependents.PARENTS)
    public void enterAnonymousTemplateParameters(AnonymousTemplateParametersContext ctx) {
      if (ctx.names != null) {
        parameterDeclarations.addAll(ctx.names);

        Set<String> currentParameters = parameters.peek();
        for (Token token : ctx.names) {
          currentParameters.add(token.getText());
        }
      }
    }
Ejemplo n.º 13
0
 public static String squeezeStatement(String sql) {
   TokenSource tokens = getLexer(sql, ImmutableSet.<String>of());
   StringBuilder sb = new StringBuilder();
   while (true) {
     Token token = tokens.nextToken();
     if (token.getType() == Token.EOF) {
       break;
     }
     if (token.getType() == SqlBaseLexer.WS) {
       sb.append(' ');
     } else {
       sb.append(token.getText());
     }
   }
   return sb.toString().trim();
 }
Ejemplo n.º 14
0
  public void syntaxError(
      Recognizer<?, ?> recognizer,
      Object offendingSymbol,
      int line,
      int charPositionInLine,
      String msg,
      RecognitionException e) {
    CommonTokenStream tokens = (CommonTokenStream) recognizer.getInputStream();
    String input = tokens.getTokenSource().getInputStream().toString();
    Token token = (Token) offendingSymbol;
    String[] lines = StringUtils.splitPreserveAllTokens(input, '\n');
    String errorLine = lines[line - 1];

    String simpleMessage = "syntax error at or near \"" + token.getText() + "\"";
    throw new LangParserException(token, line, charPositionInLine, simpleMessage, errorLine);
  }
Ejemplo n.º 15
0
 @Override
 public void createPresentation(TextPresentation presentation, ITypedRegion region) {
   // Use tokens provided by the lexer to highlight keywords, etc...
   // Seems fast enough to skip Eclipse partitioning. Infact, the Eclipse
   // partitioner seems to slow everything down...
   TSLKGrammarLexer lexer = new TSLKGrammarLexer(new ANTLRInputStream(document.get()));
   Token t = null;
   while ((t = lexer.nextToken()).getType() != Token.EOF) {
     if (t.getStartIndex() > region.getOffset() + region.getLength()) break;
     int start = t.getStartIndex();
     int end = t.getStopIndex();
     RGB foreground = null;
     RGB background = null;
     int style = SWT.NORMAL;
     switch (t.getType()) { // TODO: Make keywords customisable
       case TSLKGrammarLexer.WHILE:
       case TSLKGrammarLexer.FOR:
       case TSLKGrammarLexer.FUNC:
       case TSLKGrammarLexer.IF:
       case TSLKGrammarLexer.THEN:
       case TSLKGrammarLexer.DO:
       case TSLKGrammarLexer.END:
         foreground = ColorManager.KEYWORD;
         style = SWT.BOLD;
         break;
       case TSLKGrammarLexer.STRING:
         foreground = ColorManager.STRING;
         break;
       case TSLKGrammarLexer.SLCOMMENT:
         foreground = ColorManager.SINGLE_LINE_COMMENT;
         break;
       case TSLKGrammarLexer.MLCOMMENT:
         foreground = ColorManager.MULTI_LINE_COMMENT;
         break;
       default:
         foreground = ColorManager.DEFAULT;
         break;
     }
     presentation.addStyleRange(
         new StyleRange(
             start,
             end - start + 1,
             colorManager.getColor(foreground),
             colorManager.getColor(background),
             style));
   }
 }
Ejemplo n.º 16
0
  private static int getArgumentCount(final List<Token> tokens) {
    if (tokens.isEmpty()) {
      return 0;
    }

    int count = 0;

    for (Token token : tokens) {
      if (token.getType() == AntlrLexer.ARGUMENT) {
        ++count;
      } else {
        break;
      }
    }

    return count;
  }
Ejemplo n.º 17
0
  /**
   * Gets whether or not {@code token} is the first non-whitespace symbol on a line.
   *
   * @param token The token to test.
   * @return {@code true} if the only characters appearing before {@code token} on the same line are
   *     whitespace characters according to {@link Character#isWhitespace}.
   */
  public static boolean elementStartsLine(Token token) {
    String beginningOfLineText =
        token
            .getTokenSource()
            .getInputStream()
            .getText(
                new Interval(
                    token.getStartIndex() - token.getCharPositionInLine(),
                    token.getStartIndex() - 1));
    for (int i = 0; i < beginningOfLineText.length(); i++) {
      if (!Character.isWhitespace(beginningOfLineText.charAt(i))) {
        return false;
      }
    }

    return true;
  }
Ejemplo n.º 18
0
  public static Interval getSourceInterval(@NonNull ParseTree context) {
    Parameters.notNull("context", context);

    if (context instanceof TerminalNode) {
      TerminalNode terminalNode = (TerminalNode) context;
      Token token = terminalNode.getSymbol();
      return new Interval(token.getStartIndex(), token.getStopIndex());
    } else if (context instanceof RuleNode) {
      RuleNode ruleNode = (RuleNode) context;
      RuleContext ruleContext = ruleNode.getRuleContext();
      if (ruleContext instanceof ParserRuleContext) {
        return getSourceInterval((ParserRuleContext) ruleContext);
      } else {
        Token startSymbol = getStartSymbol(context);
        Token stopSymbol = getStopSymbol(context);
        if (startSymbol == null || stopSymbol == null) {
          return Interval.INVALID;
        }

        return new Interval(startSymbol.getStartIndex(), stopSymbol.getStopIndex());
      }
    } else {
      return Interval.INVALID;
    }
  }
Ejemplo n.º 19
0
 @Override
 public int getLen(
     RuleNode parent, Token token, List<Token> hiddenTokensToRight, TokenStream tokenStream) {
   int len = 0;
   for (org.antlr.v4.runtime.Token hiddenToken : hiddenTokensToRight) {
     final String text = hiddenToken.getText();
     if (TokenUtils.isWs(text)) {
       len++;
     } else {
       if (hiddenToken.getCharPositionInLine() > token.getCharPositionInLine()) {
         len++;
       } else {
         break;
       }
     }
   }
   return len;
 }
 @Override
 public void exitInst(InstContext ctx) {
   String s = getValue(ctx.getChild(0));
   setValue(ctx, s);
   int a = ctx.getStart().getTokenIndex();
   List<Token> tt = tokens.getHiddenTokensToLeft(a, 1);
   List<Token> ttnl = tokens.getHiddenTokensToLeft(a, 2);
   String ttt = "";
   if (ttnl != null && ttnl.size() > 1) ttt = "\n\n";
   if (tt != null) {
     for (Token t : tt) {
       ttt = ttt + t.getText();
       // System.out.println(t.getType() + " - " + t.getChannel());
     }
     // System.out.println(ttt);
   }
   setValue(ctx, ttt + getValue(ctx));
 }
Ejemplo n.º 21
0
  /**
   * Turns a token stream into a string, makes for easy debugging of token errors.
   *
   * @param lexer
   * @return
   */
  public static String tokensToString(final Lexer lexer) {
    final StringBuilder build = new StringBuilder();
    Token t;
    final String[] names = lexer.getTokenNames();
    while ((t = lexer.nextToken()) != null) {
      build.append("|");
      build.append(t.getText());
      build.append(" -> ");
      if (t.getType() >= 0) {
        build.append(names[t.getType()]);
      } else if (t.getType() == Token.EOF) {
        build.append("EOF");
        break;
      } else {
        build.append("???");
      }
    }

    return build.toString();
  }
  protected String getTokenDisplayString(Token token) {
    String string;

    if (token == null) {
      string = "[no token]";
    } else {
      String text = token.getText();

      if (text == null) {
        if (token.getType() == Token.EOF) {
          string = "end of text";
        } else {
          string = "[" + token.getType() + "]";
        }
      } else {
        string = quote(text);
      }
    }

    return string;
  }
  protected boolean tokenEndsAtEndOfLine(TokenSourceWithStateV4<TState> lexer, Token token) {
    CharStream charStream = lexer.getInputStream();
    if (charStream != null) {
      int nextCharIndex = token.getStopIndex() + 1;
      if (nextCharIndex >= charStream.size()) {
        return true;
      }

      int c =
          charStream
              .getText(Interval.of(token.getStopIndex() + 1, token.getStopIndex() + 1))
              .charAt(0);
      return c == '\r' || c == '\n';
    }

    if (token.getStopIndex() + 1 >= snapshot.length()) {
      return true;
    }

    char c = snapshot.subSequence(token.getStopIndex() + 1, token.getStopIndex() + 1).charAt(0);
    return c == '\r' || c == '\n';

    /*int line = snapshot.findLineNumber(token.getStopIndex() + 1);
    int lineStart = snapshot.findLineFromOffset(line).getStart().getOffset();
    int nextLineStart = snapshot.findLineFromOffset(line + 1).getStart().getOffset();
    int lineEnd = nextLineStart - 1;
    if (lineEnd > 0 && lineEnd > lineStart) {
        char c = snapshot.charAt(lineEnd - 1);
        if (c == '\r' || c == '\n') {
            lineEnd--;
        }
    }

    return lineEnd <= token.getStopIndex() + 1 && nextLineStart >= token.getStopIndex() + 1;*/
  }
  public static void displayErrorOrWarning(
      File compilationUnit,
      String errorOrWarning,
      CommonTokenStream tokens,
      Token offendingTokenStart,
      Token offendingTokenEnd,
      String msg) {
    System.err.println(
        errorOrWarning
            + compilationUnit.getAbsolutePath()
            + ", line "
            + offendingTokenStart.getLine()
            + ":"
            + offendingTokenStart.getCharPositionInLine()
            + "   "
            + msg);

    String input = tokens.getTokenSource().getInputStream().toString();
    String[] lines = input.split("\n");
    String errorLine = lines[offendingTokenStart.getLine() - 1];
    System.err.println(errorLine);

    for (int i = 0; i < offendingTokenStart.getCharPositionInLine(); i++)
      if (errorLine.substring(i, i + 1).equals("\t")) System.err.print("\t");
      else System.err.print(" ");

    int start = offendingTokenStart.getStartIndex();
    int stop = offendingTokenEnd.getStopIndex();

    if (start >= 0 && stop >= 0) for (int i = start; i <= stop; i++) System.err.print("^");

    System.err.println();
  }
Ejemplo n.º 25
0
 public StatementSplitter(String sql, Set<String> delimiters) {
   TokenSource tokens = getLexer(sql, delimiters);
   ImmutableList.Builder<Statement> list = ImmutableList.builder();
   StringBuilder sb = new StringBuilder();
   while (true) {
     Token token = tokens.nextToken();
     if (token.getType() == Token.EOF) {
       break;
     }
     if (token.getType() == SqlBaseParser.DELIMITER) {
       String statement = sb.toString().trim();
       if (!statement.isEmpty()) {
         list.add(new Statement(statement, token.getText()));
       }
       sb = new StringBuilder();
     } else {
       sb.append(token.getText());
     }
   }
   this.completeStatements = list.build();
   this.partialStatement = sb.toString().trim();
 }
Ejemplo n.º 26
0
    /**
     * Logs parser errors in Checkstyle manner. Parser can generate error messages. There is special
     * error that parser can generate. It is missed close HTML tag. This case is special because
     * parser prints error like {@code "no viable alternative at input 'b \n *\n'"} and it is not
     * clear that error is about missed close HTML tag. Other error messages are not special and
     * logged simply as "Parse Error...".
     *
     * <p>{@inheritDoc}
     */
    @Override
    public void syntaxError(
        Recognizer<?, ?> recognizer,
        Object offendingSymbol,
        int line,
        int charPositionInLine,
        String msg,
        RecognitionException ex) {
      final int lineNumber = offset + line;
      final Token token = (Token) offendingSymbol;

      if (MSG_JAVADOC_MISSED_HTML_CLOSE.equals(msg)) {
        errorMessage =
            new ParseErrorMessage(
                lineNumber, MSG_JAVADOC_MISSED_HTML_CLOSE, charPositionInLine, token.getText());

        throw new ParseCancellationException(msg);
      } else if (MSG_JAVADOC_WRONG_SINGLETON_TAG.equals(msg)) {
        errorMessage =
            new ParseErrorMessage(
                lineNumber, MSG_JAVADOC_WRONG_SINGLETON_TAG, charPositionInLine, token.getText());

        throw new ParseCancellationException(msg);
      } else {
        final int ruleIndex = ex.getCtx().getRuleIndex();
        final String ruleName = recognizer.getRuleNames()[ruleIndex];
        final String upperCaseRuleName =
            CaseFormat.UPPER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, ruleName);

        errorMessage =
            new ParseErrorMessage(
                lineNumber,
                MSG_JAVADOC_PARSE_RULE_ERROR,
                charPositionInLine,
                msg,
                upperCaseRuleName);
      }
    }
Ejemplo n.º 27
0
  public static Interval getSourceInterval(@NonNull ParserRuleContext context) {
    Parameters.notNull("context", context);
    int startIndex = context.start.getStartIndex();
    Token stopSymbol = getStopSymbol(context);
    if (stopSymbol == null) {
      return new Interval(startIndex, startIndex - 1);
    }

    int stopIndex;
    if (stopSymbol.getType() != Token.EOF) {
      stopIndex = stopSymbol.getStopIndex();
    } else {
      TokenSource tokenSource = context.getStart().getTokenSource();
      CharStream inputStream = tokenSource != null ? tokenSource.getInputStream() : null;
      if (inputStream != null) {
        stopIndex = inputStream.size() - 1;
      } else {
        stopIndex = context.start.getStartIndex() - 1;
      }
    }

    stopIndex = Math.max(stopIndex, startIndex - 1);
    return new Interval(startIndex, stopIndex);
  }
Ejemplo n.º 28
0
  @CheckForNull
  public static TerminalNode findTerminalNode(@NonNull ParseTree node, Token symbol) {
    if (symbol == null) {
      return null;
    }

    if (node instanceof TerminalNode) {
      TerminalNode terminalNode = (TerminalNode) node;
      if (Utils.equals(terminalNode.getSymbol(), symbol)) {
        return terminalNode;
      }

      return null;
    }

    for (int i = 0; i < node.getChildCount(); i++) {
      ParseTree child = node.getChild(i);
      TerminalNode stopNode = ParseTrees.getStopNode(child);
      if (stopNode == null) {
        continue;
      }

      Token stopSymbol = stopNode.getSymbol();
      if (stopSymbol.getStopIndex() < symbol.getStartIndex()) {
        continue;
      }

      TerminalNode startNode = ParseTrees.getStartNode(child);
      assert startNode != null;

      stopSymbol = startNode.getSymbol();
      if (stopSymbol == null || stopSymbol.getStartIndex() > symbol.getStopIndex()) {
        break;
      }

      if (stopSymbol.equals(symbol)) {
        return startNode;
      }

      TerminalNode terminalNode = findTerminalNode(child, symbol);
      if (terminalNode != null) {
        return terminalNode;
      }
    }

    return null;
  }
Ejemplo n.º 29
0
 private static boolean is(Token token, int type) {
   return token != null && token.getType() == type;
 }
Ejemplo n.º 30
0
 public PSymbolBuilder qualifier(Token q) {
   return qualifier(q != null ? q.getText() : null);
 }