@Override
  public boolean consume(CodeReader code, Lexer lexer) {
    if (!Character.isJavaIdentifierStart(code.peek())) {
      return false;
    }

    int line = code.getCursor().getLine();
    int column = code.getCursor().getColumn();
    while (Character.isJavaIdentifierPart(code.peek())) {
      tmpBuilder.append((char) code.pop());
    }

    String word = tmpBuilder.toString();

    TokenType keywordType = keywordsMap.get(word);
    Token token =
        tokenBuilder
            .setType(keywordType == null ? GenericTokenType.IDENTIFIER : keywordType)
            .setValueAndOriginalValue(word, word)
            .setURI(lexer.getURI())
            .setLine(line)
            .setColumn(column)
            .build();

    lexer.addToken(token);

    tmpBuilder.delete(0, tmpBuilder.length());

    return true;
  }
Esempio n. 2
0
  @Override
  public boolean consume(CodeReader code, Lexer output) {
    int line = code.getLinePosition();
    int column = code.getColumnPosition();

    char ch = code.charAt(0);
    if ((ch != '#')) {
      return false;
    }

    String tokenValue = read(code);
    output.addToken(
        Token.builder()
            .setLine(line)
            .setColumn(column)
            .setURI(output.getURI())
            .setValueAndOriginalValue(tokenValue)
            .setType(CxxTokenType.PREPROCESSOR)
            .build());

    return true;
  }
  public static Lexer create(ObjectiveCConfiguration conf) {
    return Lexer.builder()
        .withCharset(conf.getCharset())
        .withFailIfNoChannelToConsumeOneCharacter(false)

        // Comments
        .withChannel(commentRegexp("//[^\\n\\r]*+"))
        .withChannel(commentRegexp("/\\*[\\s\\S]*?\\*/"))

        // All other tokens
        .withChannel(regexp(LITERAL, "[^\r\n\\s/]+"))
        .withChannel(new BlackHoleChannel("[\\s]"))
        .build();
  }
Esempio n. 4
0
 @Test
 public void hash_followed_by_word() {
   List<Token> tokens = lexer.lex("#a");
   assertThat(tokens, hasToken("#", CppPunctuator.HASH));
   assertThat(tokens, hasToken("a", IDENTIFIER));
 }
Esempio n. 5
0
 @Test
 public void cpp_operators() {
   assertThat(lexer.lex("#"), hasToken("#", CppPunctuator.HASH));
   assertThat(lexer.lex("##"), hasToken("##", CppPunctuator.HASHHASH));
 }
Esempio n. 6
0
 @Test
 public void cpp_identifiers() {
   assertThat(lexer.lex("lala"), hasToken("lala", IDENTIFIER));
 }
Esempio n. 7
0
 @Test
 public void cpp_keywords_indented() {
   assertThat(lexer.lex(" #define"), hasToken("#define", CppKeyword.DEFINE));
   assertThat(lexer.lex("\t#define"), hasToken("#define", CppKeyword.DEFINE));
 }
Esempio n. 8
0
 @Test
 public void cpp_keywords_with_whitespaces() {
   assertThat(lexer.lex("#  define"), hasToken("#define", CppKeyword.DEFINE));
   assertThat(lexer.lex("#\tinclude"), hasToken("#include", CppKeyword.INCLUDE));
 }